breakdown folder tree for epic R01
This commit is contained in:
@@ -0,0 +1,157 @@
|
||||
"""Characterization tests for core/chat_agent.py (run_chat and run_cowork runtime seams).
|
||||
|
||||
These tests capture existing behavior as an executable baseline specification,
|
||||
ensuring that future refactoring to ConversationApplicationService does not alter
|
||||
core turn semantics, event emissions, or file handling.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from cowork_local.core import chat_agent
|
||||
from cowork_local.tests.fakes.fake_provider import FakeProvider
|
||||
|
||||
|
||||
def test_run_chat_characterization() -> None:
|
||||
"""Capture baseline behavior of run_chat: system prompt insertion, streaming, and message persistence."""
|
||||
provider = FakeProvider()
|
||||
provider.queue_response(content="Hello there!", chunks=["Hello ", "there!"])
|
||||
|
||||
messages: List[Dict[str, Any]] = [{"role": "user", "content": "Hi assistant"}]
|
||||
emitted_events: List[Dict[str, Any]] = []
|
||||
|
||||
def emit(event: Dict[str, Any]) -> None:
|
||||
emitted_events.append(event)
|
||||
|
||||
result = chat_agent.run_chat(
|
||||
provider=provider,
|
||||
messages=messages,
|
||||
emit=emit,
|
||||
)
|
||||
|
||||
# 1. Verify system prompt was injected at position 0
|
||||
assert messages[0]["role"] == "system"
|
||||
assert "Cowork Local" in messages[0]["content"]
|
||||
|
||||
# 2. Verify returned assistant message
|
||||
assert result["role"] == "assistant"
|
||||
assert result["content"] == "Hello there!"
|
||||
|
||||
# 3. Verify assistant message was appended to messages list
|
||||
assert messages[-1] == result
|
||||
|
||||
# 4. Verify emitted events sequence
|
||||
text_deltas = [e["delta"] for e in emitted_events if e["type"] == "text"]
|
||||
assert "".join(text_deltas) == "Hello there!"
|
||||
assert any(e["type"] == "assistant_done" for e in emitted_events)
|
||||
|
||||
|
||||
def test_run_cowork_save_file_characterization(tmp_path: Path) -> None:
|
||||
"""Capture baseline behavior of run_cowork: tool execution loop and file production."""
|
||||
output_dir = tmp_path / "output"
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
provider = FakeProvider()
|
||||
# Step 1: Model requests save_file tool
|
||||
provider.queue_response(
|
||||
content="Saving your requested report.",
|
||||
tool_calls=[{
|
||||
"id": "call_save_1",
|
||||
"name": "save_file",
|
||||
"arguments": {
|
||||
"filename": "report.md",
|
||||
"content": "# Executive Summary\nAll systems nominal.",
|
||||
},
|
||||
}],
|
||||
)
|
||||
# Step 2: Model finishes after tool result
|
||||
provider.queue_response(
|
||||
content="I have created report.md in your output directory.",
|
||||
chunks=["I have created report.md in your output directory."],
|
||||
)
|
||||
|
||||
messages: List[Dict[str, Any]] = [{"role": "user", "content": "Export report to markdown file"}]
|
||||
emitted_events: List[Dict[str, Any]] = []
|
||||
|
||||
def emit(event: Dict[str, Any]) -> None:
|
||||
emitted_events.append(event)
|
||||
|
||||
final_messages = chat_agent.run_cowork(
|
||||
provider=provider,
|
||||
messages=messages,
|
||||
output_dir=output_dir,
|
||||
emit=emit,
|
||||
enforce_rules=False,
|
||||
)
|
||||
|
||||
# 1. Verify file was created in output directory with expected content
|
||||
created_file = output_dir / "report.md"
|
||||
assert created_file.exists()
|
||||
assert created_file.read_text(encoding="utf-8") == "# Executive Summary\nAll systems nominal."
|
||||
|
||||
# 2. Verify message history contains user -> assistant (tool_calls) -> tool -> assistant
|
||||
roles = [m["role"] for m in final_messages]
|
||||
assert "system" in roles
|
||||
assert "user" in roles
|
||||
assert "tool" in roles
|
||||
|
||||
# 3. Verify tool result message content
|
||||
tool_msg = next(m for m in final_messages if m["role"] == "tool")
|
||||
assert tool_msg["name"] == "save_file"
|
||||
assert "Saved report.md" in tool_msg["content"]
|
||||
|
||||
|
||||
def test_run_cowork_cancellation_characterization(tmp_path: Path) -> None:
|
||||
"""Capture cancellation behavior in run_cowork."""
|
||||
output_dir = tmp_path / "output_cancel"
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
provider = FakeProvider()
|
||||
provider.queue_response(content="Working...")
|
||||
|
||||
is_cancelled = True
|
||||
|
||||
def check_cancel() -> bool:
|
||||
return is_cancelled
|
||||
|
||||
emitted_events: List[Dict[str, Any]] = []
|
||||
messages: List[Dict[str, Any]] = [{"role": "user", "content": "Please start"}]
|
||||
|
||||
chat_agent.run_cowork(
|
||||
provider=provider,
|
||||
messages=messages,
|
||||
output_dir=output_dir,
|
||||
emit=lambda e: emitted_events.append(e),
|
||||
cancel=check_cancel,
|
||||
enforce_rules=False,
|
||||
)
|
||||
|
||||
# Provider should not have executed turns if cancelled right away
|
||||
assert provider.call_count == 0
|
||||
|
||||
|
||||
def test_cleanup_turn_output_characterization(tmp_path: Path) -> None:
|
||||
"""Capture behavior of temporary .scratch folder cleanup and artifact preservation."""
|
||||
output_dir = tmp_path / "output_cleanup"
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
scratch_dir = output_dir / ".scratch"
|
||||
scratch_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Create a generator script and a deliverable inside scratch
|
||||
generator_script = scratch_dir / "gen.py"
|
||||
generator_script.write_text("print('generating')", encoding="utf-8")
|
||||
deliverable = scratch_dir / "data.csv"
|
||||
deliverable.write_text("a,b,c\n1,2,3", encoding="utf-8")
|
||||
|
||||
before_snapshot = chat_agent._snapshot(output_dir)
|
||||
removed, moved = chat_agent._cleanup_cowork_intermediates(output_dir, before_snapshot, cancelled=False)
|
||||
|
||||
# .scratch directory should be removed
|
||||
assert not scratch_dir.exists()
|
||||
# deliverable should be moved to output root
|
||||
root_csv = output_dir / "data.csv"
|
||||
assert root_csv.exists()
|
||||
# script should not be in output root
|
||||
assert not (output_dir / "gen.py").exists()
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
"""Test doubles and offline fakes package for Cowork Local test pyramid."""
|
||||
from .fake_provider import FakeProvider
|
||||
from .fake_tool_executor import FakeToolExecutor
|
||||
|
||||
__all__ = ["FakeProvider", "FakeToolExecutor"]
|
||||
@@ -0,0 +1,113 @@
|
||||
"""Fake LLM Provider for offline unit, contract, and characterization testing.
|
||||
|
||||
Provides deterministic responses, stream simulation, tool-call dispatching,
|
||||
and fault injection without requiring any external network access or API keys.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
from providers.base import CancelFn, Provider, ProviderError, TextCallback, ToolSpec
|
||||
|
||||
|
||||
class FakeProvider(Provider):
|
||||
"""Deterministic test double mimicking real LLM Providers (OpenAI, Anthropic, Ollama)."""
|
||||
|
||||
name = "fake"
|
||||
supports_vision = True
|
||||
|
||||
def __init__(self, conf: Optional[Dict[str, Any]] = None) -> None:
|
||||
# Initialize base provider with default configuration if none provided
|
||||
super().__init__(conf or {"model": "fake-model-v1"})
|
||||
# History of all message batches sent across all chat calls
|
||||
self.call_history: List[List[Dict[str, Any]]] = []
|
||||
# Queue of programmed assistant responses to return sequentially
|
||||
self.response_queue: List[Dict[str, Any]] = []
|
||||
# Queue of exceptions to raise on corresponding calls
|
||||
self.error_queue: List[Exception] = []
|
||||
# Default text returned when response queue is empty
|
||||
self.default_text: str = "Fake model response."
|
||||
# Total number of chat invocations
|
||||
self.call_count: int = 0
|
||||
# Recorded tool specs passed into each turn
|
||||
self.last_tools: Optional[List[ToolSpec]] = None
|
||||
|
||||
def queue_response(
|
||||
self,
|
||||
content: str = "",
|
||||
tool_calls: Optional[List[Dict[str, Any]]] = None,
|
||||
reasoning: Optional[str] = None,
|
||||
chunks: Optional[List[str]] = None,
|
||||
) -> FakeProvider:
|
||||
"""Enqueue a pre-configured response structure for upcoming chat turns."""
|
||||
self.response_queue.append({
|
||||
"content": content,
|
||||
"tool_calls": tool_calls or [],
|
||||
"reasoning": reasoning,
|
||||
"chunks": chunks or ([content] if content else []),
|
||||
})
|
||||
return self
|
||||
|
||||
def queue_error(self, exc: Exception) -> FakeProvider:
|
||||
"""Enqueue an exception to simulate network/API errors on the next turn."""
|
||||
self.error_queue.append(exc)
|
||||
return self
|
||||
|
||||
def chat(
|
||||
self,
|
||||
messages: List[Dict[str, Any]],
|
||||
tools: Optional[List[ToolSpec]] = None,
|
||||
on_text: Optional[TextCallback] = None,
|
||||
cancel: Optional[CancelFn] = None,
|
||||
on_reasoning: Optional[TextCallback] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Simulate single LLM turn with full streaming and tool-call support."""
|
||||
self.call_count += 1
|
||||
self.call_history.append([dict(m) for m in messages])
|
||||
self.last_tools = tools
|
||||
|
||||
# 1. Check for injected errors
|
||||
if self.error_queue:
|
||||
raise self.error_queue.pop(0)
|
||||
|
||||
# 2. Check early cancellation before processing
|
||||
if cancel and cancel():
|
||||
raise ProviderError("Execution aborted by user cancel signal before response generation.")
|
||||
|
||||
# 3. Retrieve queued response or construct default response
|
||||
if self.response_queue:
|
||||
resp_spec = self.response_queue.pop(0)
|
||||
content = resp_spec.get("content", "")
|
||||
tool_calls = resp_spec.get("tool_calls", [])
|
||||
reasoning = resp_spec.get("reasoning")
|
||||
chunks = resp_spec.get("chunks", [content] if content else [])
|
||||
else:
|
||||
content = self.default_text
|
||||
tool_calls = []
|
||||
reasoning = None
|
||||
chunks = [content]
|
||||
|
||||
# 4. Stream reasoning chunks if provided
|
||||
if reasoning and on_reasoning:
|
||||
on_reasoning(reasoning)
|
||||
|
||||
# 5. Stream text chunks, checking cancellation between fragments
|
||||
for chunk in chunks:
|
||||
if cancel and cancel():
|
||||
raise ProviderError("Execution cancelled during text chunk streaming.")
|
||||
if on_text and chunk:
|
||||
on_text(chunk)
|
||||
|
||||
# 6. Return canonical assistant message payload
|
||||
assistant_msg: Dict[str, Any] = {
|
||||
"role": "assistant",
|
||||
"content": content,
|
||||
}
|
||||
if tool_calls:
|
||||
assistant_msg["tool_calls"] = tool_calls
|
||||
|
||||
return assistant_msg
|
||||
|
||||
def list_models(self) -> List[str]:
|
||||
"""Return available mock models for settings and validation tests."""
|
||||
return ["fake-model-v1", "fake-reasoner-pro", "fake-vision-plus"]
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Fake Tool Executor for isolated, offline agent tool-call verification.
|
||||
|
||||
Allows tests to verify tool invocation arguments, mock tool return values,
|
||||
and simulate failures/delays without performing unsafe host disk or OS operations.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
|
||||
class FakeToolExecutor:
|
||||
"""Mock execution engine for agent tool-call dispatching."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
# History of all executed tool invocations: List of {"name": str, "args": dict, "result": dict}
|
||||
self.call_log: List[Dict[str, Any]] = []
|
||||
# Custom handlers registered per tool name
|
||||
self.handlers: Dict[str, Callable[[Dict[str, Any]], Dict[str, Any]]] = {}
|
||||
# Pre-programmed fixed responses keyed by tool name
|
||||
self.mock_responses: Dict[str, Dict[str, Any]] = {}
|
||||
# Default response when no specific handler or response is found
|
||||
self.default_result: Dict[str, Any] = {"ok": True, "output": "Fake tool executed successfully."}
|
||||
|
||||
def register_handler(
|
||||
self,
|
||||
tool_name: str,
|
||||
handler: Callable[[Dict[str, Any]], Dict[str, Any]],
|
||||
) -> FakeToolExecutor:
|
||||
"""Register a dynamic handler function for a specific tool name."""
|
||||
self.handlers[tool_name] = handler
|
||||
return self
|
||||
|
||||
def set_mock_response(
|
||||
self,
|
||||
tool_name: str,
|
||||
result: Dict[str, Any],
|
||||
) -> FakeToolExecutor:
|
||||
"""Set a static return payload for a specific tool name."""
|
||||
self.mock_responses[tool_name] = result
|
||||
return self
|
||||
|
||||
def execute(self, tool_name: str, arguments: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Execute a tool call using registered mocks and record invocation details."""
|
||||
# 1. Resolve result from handler, preset response, or default fallback
|
||||
if tool_name in self.handlers:
|
||||
result = self.handlers[tool_name](arguments)
|
||||
elif tool_name in self.mock_responses:
|
||||
result = self.mock_responses[tool_name]
|
||||
else:
|
||||
result = dict(self.default_result)
|
||||
result["tool"] = tool_name
|
||||
result["received_args"] = arguments
|
||||
|
||||
# 2. Record execution trace for post-test assertions
|
||||
self.call_log.append({
|
||||
"name": tool_name,
|
||||
"args": dict(arguments),
|
||||
"result": dict(result),
|
||||
})
|
||||
|
||||
return result
|
||||
|
||||
def get_calls_for(self, tool_name: str) -> List[Dict[str, Any]]:
|
||||
"""Retrieve all recorded calls for a given tool name."""
|
||||
return [call for call in self.call_log if call["name"] == tool_name]
|
||||
|
||||
def reset(self) -> None:
|
||||
"""Clear recorded logs and registered mock responses."""
|
||||
self.call_log.clear()
|
||||
self.handlers.clear()
|
||||
self.mock_responses.clear()
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Unit tests for the Clean Architecture AST Import Guard (check_imports.py)."""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from scripts.check_imports import FORBIDDEN_MODULE_PREFIXES, scan_file
|
||||
|
||||
|
||||
def test_clean_python_file_passes(tmp_path: Path) -> None:
|
||||
"""Verify that pure Python code without GUI imports produces 0 violations."""
|
||||
clean_code = """
|
||||
import os
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from typing import List
|
||||
|
||||
@dataclass
|
||||
class UserRequest:
|
||||
id: str
|
||||
prompt: str
|
||||
"""
|
||||
clean_file = tmp_path / "clean_service.py"
|
||||
clean_file.write_text(clean_code, encoding="utf-8")
|
||||
|
||||
violations = scan_file(clean_file, FORBIDDEN_MODULE_PREFIXES)
|
||||
assert len(violations) == 0
|
||||
|
||||
|
||||
def test_forbidden_pyside_import_detected(tmp_path: Path) -> None:
|
||||
"""Verify that PySide6 import is caught with correct line number."""
|
||||
dirty_code = """
|
||||
from dataclasses import dataclass
|
||||
from PySide6.QtWidgets import QWidget
|
||||
|
||||
class BadService:
|
||||
pass
|
||||
"""
|
||||
dirty_file = tmp_path / "bad_service.py"
|
||||
dirty_file.write_text(dirty_code, encoding="utf-8")
|
||||
|
||||
violations = scan_file(dirty_file, FORBIDDEN_MODULE_PREFIXES)
|
||||
assert len(violations) == 1
|
||||
assert violations[0].line_number == 3
|
||||
assert "PySide6" in violations[0].imported_module
|
||||
|
||||
|
||||
def test_forbidden_ui_and_app_import_detected(tmp_path: Path) -> None:
|
||||
"""Verify that importing concrete UI or app modules from domain is caught."""
|
||||
dirty_code = """
|
||||
import ui.chat_panel
|
||||
from app import MainWindow
|
||||
"""
|
||||
dirty_file = tmp_path / "cross_layer_leak.py"
|
||||
dirty_file.write_text(dirty_code, encoding="utf-8")
|
||||
|
||||
violations = scan_file(dirty_file, FORBIDDEN_MODULE_PREFIXES)
|
||||
assert len(violations) == 2
|
||||
modules = [v.imported_module for v in violations]
|
||||
assert "ui.chat_panel" in modules
|
||||
assert "app" in modules
|
||||
@@ -0,0 +1,94 @@
|
||||
"""Unit tests for FakeProvider and FakeToolExecutor test doubles."""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from providers.base import ProviderError
|
||||
from tests.fakes.fake_provider import FakeProvider
|
||||
from tests.fakes.fake_tool_executor import FakeToolExecutor
|
||||
|
||||
|
||||
def test_fake_provider_text_streaming() -> None:
|
||||
"""Verify that FakeProvider streams text chunks to on_text callback."""
|
||||
provider = FakeProvider()
|
||||
provider.queue_response(content="Hello world", chunks=["Hello ", "world"])
|
||||
|
||||
streamed: list[str] = []
|
||||
response = provider.chat(
|
||||
messages=[{"role": "user", "content": "Hi"}],
|
||||
on_text=lambda piece: streamed.append(piece),
|
||||
)
|
||||
|
||||
assert response["role"] == "assistant"
|
||||
assert response["content"] == "Hello world"
|
||||
assert "".join(streamed) == "Hello world"
|
||||
assert provider.call_count == 1
|
||||
|
||||
|
||||
def test_fake_provider_tool_calls_and_reasoning() -> None:
|
||||
"""Verify reasoning streaming and tool_calls payload emission."""
|
||||
provider = FakeProvider()
|
||||
tool_call = {
|
||||
"id": "call_123",
|
||||
"name": "save_file",
|
||||
"arguments": {"filename": "out.txt", "content": "data"},
|
||||
}
|
||||
provider.queue_response(
|
||||
content="Creating file",
|
||||
tool_calls=[tool_call],
|
||||
reasoning="User wants output in a file",
|
||||
)
|
||||
|
||||
reasoning_chunks: list[str] = []
|
||||
response = provider.chat(
|
||||
messages=[{"role": "user", "content": "Save to out.txt"}],
|
||||
on_reasoning=lambda piece: reasoning_chunks.append(piece),
|
||||
)
|
||||
|
||||
assert response["content"] == "Creating file"
|
||||
assert response["tool_calls"] == [tool_call]
|
||||
assert reasoning_chunks == ["User wants output in a file"]
|
||||
|
||||
|
||||
def test_fake_provider_error_injection() -> None:
|
||||
"""Verify that queued exceptions are raised on demand."""
|
||||
provider = FakeProvider()
|
||||
provider.queue_error(ProviderError("Rate limit exceeded (429)"))
|
||||
|
||||
with pytest.raises(ProviderError, match="Rate limit exceeded"):
|
||||
provider.chat(messages=[{"role": "user", "content": "Hi"}])
|
||||
|
||||
|
||||
def test_fake_provider_cancellation() -> None:
|
||||
"""Verify that cancellation stops execution immediately."""
|
||||
provider = FakeProvider()
|
||||
provider.queue_response(content="Long reply", chunks=["Part 1", "Part 2"])
|
||||
|
||||
is_cancelled = False
|
||||
|
||||
def cancel_fn() -> bool:
|
||||
return is_cancelled
|
||||
|
||||
is_cancelled = True
|
||||
with pytest.raises(ProviderError, match="aborted by user cancel"):
|
||||
provider.chat(
|
||||
messages=[{"role": "user", "content": "Hi"}],
|
||||
cancel=cancel_fn,
|
||||
)
|
||||
|
||||
|
||||
def test_fake_tool_executor() -> None:
|
||||
"""Verify that FakeToolExecutor records calls and returns expected mock outputs."""
|
||||
executor = FakeToolExecutor()
|
||||
executor.set_mock_response("read_file", {"ok": True, "content": "file contents"})
|
||||
executor.register_handler("calc", lambda args: {"ok": True, "result": args.get("a", 0) + args.get("b", 0)})
|
||||
|
||||
res1 = executor.execute("read_file", {"path": "test.txt"})
|
||||
assert res1["ok"] is True
|
||||
assert res1["content"] == "file contents"
|
||||
|
||||
res2 = executor.execute("calc", {"a": 5, "b": 10})
|
||||
assert res2["result"] == 15
|
||||
|
||||
assert len(executor.call_log) == 2
|
||||
assert executor.get_calls_for("calc")[0]["args"] == {"a": 5, "b": 10}
|
||||
Reference in New Issue
Block a user