"""Retrieval regression suite for Jira Project Knowledge. This suite uses a synthetic Jira corpus to evaluate retrieval quality without requiring a live Jira instance or confidential customer data. It covers: - Exact term matching - Paraphrasing - Ambiguous queries - Negative/no-result cases - Cross-project isolation - Citation completeness """ from __future__ import annotations import json from dataclasses import dataclass from pathlib import Path from typing import Any import pytest from cowork_local.application.jira_knowledge.index_repository import JiraKnowledgeIndex from cowork_local.domain.jira_knowledge.canonical_issue import CanonicalJiraIssue, JiraProvenance from cowork_local.mcp_servers.project_context.providers.jira_knowledge import JiraKnowledgeProvider, _JiraKbTarget # --------------------------------------------------------------------------- # Synthetic Corpus # --------------------------------------------------------------------------- PROJECT_A = "proj-alpha" PROJECT_B = "proj-beta" JIRA_KEY_A = "ALPHA" JIRA_KEY_B = "BETA" CORPUS = [ # Project A: Requirements CanonicalJiraIssue( knowledge_id=f"{JIRA_KEY_A}/REQ-1", project_id=PROJECT_A, title="User Authentication Requirement", content="The system must support user login via email and password. Account lockout occurs after 5 failed attempts.", provenance=JiraProvenance(system="jira", issue_key="REQ-1", project_key=JIRA_KEY_A, source_url="http://jira/REQ-1", source_updated="2026-01-01T00:00:00Z", issue_type="Requirement", status="Done"), ingested_at="2026-01-01T00:00:00Z" ), CanonicalJiraIssue( knowledge_id=f"{JIRA_KEY_A}/REQ-2", project_id=PROJECT_A, title="Password Reset Policy", content="Password reset links expire after 30 minutes. Users must verify their email address.", provenance=JiraProvenance(system="jira", issue_key="REQ-2", project_key=JIRA_KEY_A, source_url="http://jira/REQ-2", source_updated="2026-01-02T00:00:00Z", issue_type="Requirement", status="Done"), ingested_at="2026-01-02T00:00:00Z" ), # Project A: Bugs CanonicalJiraIssue( knowledge_id=f"{JIRA_KEY_A}/BUG-101", project_id=PROJECT_A, title="Database Timeout on Login", content="Users experience a 500 error when logging in during peak hours due to database connection pool exhaustion.", provenance=JiraProvenance(system="jira", issue_key="BUG-101", project_key=JIRA_KEY_A, source_url="http://jira/BUG-101", source_updated="2026-02-01T00:00:00Z", issue_type="Bug", status="Open"), ingested_at="2026-02-01T00:00:00Z" ), # Project B: Secret/Isolation Test CanonicalJiraIssue( knowledge_id=f"{JIRA_KEY_B}/SECRET-1", project_id=PROJECT_B, title="Project Beta Secret Key", content="The secret key for Project Beta is SUPER_SECRET_BETA_KEY_12345. Do not share.", provenance=JiraProvenance(system="jira", issue_key="SECRET-1", project_key=JIRA_KEY_B, source_url="http://jira/SECRET-1", source_updated="2026-01-01T00:00:00Z", issue_type="Task", status="Done"), ingested_at="2026-01-01T00:00:00Z" ), # Project B: Similar terminology to Project A (for ambiguity test) CanonicalJiraIssue( knowledge_id=f"{JIRA_KEY_B}/REQ-1", project_id=PROJECT_B, title="User Authentication Requirement (Beta)", content="The beta system supports SSO login. Account lockout is disabled for testing.", provenance=JiraProvenance(system="jira", issue_key="REQ-1", project_key=JIRA_KEY_B, source_url="http://jira/REQ-1", source_updated="2026-01-01T00:00:00Z", issue_type="Requirement", status="Done"), ingested_at="2026-01-01T00:00:00Z" ), ] @pytest.fixture def populated_index(tmp_path: Path) -> JiraKnowledgeIndex: """Create an index populated with the synthetic corpus.""" index = JiraKnowledgeIndex(index_root=tmp_path) for issue in CORPUS: index.upsert(issue) return index @pytest.fixture def provider_a(populated_index: JiraKnowledgeIndex) -> JiraKnowledgeProvider: """Provider scoped to Project A.""" target = _JiraKbTarget(cowork_project_id=PROJECT_A, jira_project_key=JIRA_KEY_A) return JiraKnowledgeProvider(target, index=populated_index) @pytest.fixture def provider_b(populated_index: JiraKnowledgeIndex) -> JiraKnowledgeProvider: """Provider scoped to Project B.""" target = _JiraKbTarget(cowork_project_id=PROJECT_B, jira_project_key=JIRA_KEY_B) return JiraKnowledgeProvider(target, index=populated_index) # --------------------------------------------------------------------------- # Retrieval Quality Tests # --------------------------------------------------------------------------- def test_exact_term_match(provider_a: JiraKnowledgeProvider): """Query with exact terms from REQ-1 should return REQ-1.""" result = provider_a.search_knowledge(project_id=PROJECT_A, query="account lockout 5 failed attempts") assert result["returned"] > 0 assert any("REQ-1" in item["document_id"] for item in result["items"]) def test_paraphrase_match(provider_a: JiraKnowledgeProvider): """Query paraphrasing REQ-2 should return REQ-2.""" result = provider_a.search_knowledge(project_id=PROJECT_A, query="how long does password reset link last") assert result["returned"] > 0 assert any("REQ-2" in item["document_id"] for item in result["items"]) def test_ambiguous_query_prefers_local_context(provider_a: JiraKnowledgeProvider): """Query 'authentication' exists in both projects, but provider_a should only return Project A results.""" result = provider_a.search_knowledge(project_id=PROJECT_A, query="user authentication login") assert result["returned"] > 0 for item in result["items"]: assert PROJECT_A in item["document_id"] or JIRA_KEY_A in item["document_id"] assert PROJECT_B not in item["document_id"] def test_no_result_query(provider_a: JiraKnowledgeProvider): """Query with no matching terms should return empty results.""" result = provider_a.search_knowledge(project_id=PROJECT_A, query="quantum computing blockchain") assert result["returned"] == 0 assert result["items"] == () def test_cross_project_isolation(provider_a: JiraKnowledgeProvider): """Project A provider must never return Project B's secret.""" result = provider_a.search_knowledge(project_id=PROJECT_A, query="SUPER_SECRET_BETA_KEY_12345") assert result["returned"] == 0 # Double check: ensure the secret string is not in any excerpt for item in result["items"]: assert "SUPER_SECRET_BETA_KEY_12345" not in item["excerpt"] def test_citation_completeness(provider_a: JiraKnowledgeProvider): """Every result must have a valid Jira source URL.""" result = provider_a.search_knowledge(project_id=PROJECT_A, query="database timeout") assert result["returned"] > 0 for item in result["items"]: assert "source" in item assert "url" in item["source"] assert item["source"]["url"].startswith("http") assert "system" in item["source"] assert item["source"]["system"] == "jira" def test_bug_retrieval(provider_a: JiraKnowledgeProvider): """Query about bugs should return BUG-101.""" result = provider_a.search_knowledge(project_id=PROJECT_A, query="500 error login peak hours") assert result["returned"] > 0 assert any("BUG-101" in item["document_id"] for item in result["items"]) # --------------------------------------------------------------------------- # Metrics Collection (Baseline) # --------------------------------------------------------------------------- def test_baseline_metrics(provider_a: JiraKnowledgeProvider, provider_b: JiraKnowledgeProvider): """Collect Hit@1, Hit@3, Hit@5 for a set of queries.""" queries = [ ("account lockout", ["REQ-1"]), ("password reset expire", ["REQ-2"]), ("database timeout", ["BUG-101"]), ("SSO login", []), # Should be empty for Project A ] hits_at_1 = 0 hits_at_3 = 0 hits_at_5 = 0 total = len(queries) for query, expected_ids in queries: result = provider_a.search_knowledge(project_id=PROJECT_A, query=query, top_k=5) returned_ids = [item["document_id"] for item in result["items"]] if not expected_ids: if len(returned_ids) == 0: hits_at_1 += 1 hits_at_3 += 1 hits_at_5 += 1 continue found_at_1 = any(eid in rid for rid in returned_ids[:1] for eid in expected_ids) found_at_3 = any(eid in rid for rid in returned_ids[:3] for eid in expected_ids) found_at_5 = any(eid in rid for rid in returned_ids[:5] for eid in expected_ids) if found_at_1: hits_at_1 += 1 if found_at_3: hits_at_3 += 1 if found_at_5: hits_at_5 += 1 # Record baseline (in a real CI, this would be asserted against a stored baseline) print(f"\n--- Retrieval Baseline ---") print(f"Hit@1: {hits_at_1}/{total} ({hits_at_1/total:.2f})") print(f"Hit@3: {hits_at_3}/{total} ({hits_at_3/total:.2f})") print(f"Hit@5: {hits_at_5}/{total} ({hits_at_5/total:.2f})") # For this synthetic corpus, we expect perfect scores assert hits_at_1 == total assert hits_at_3 == total assert hits_at_5 == total