feat(jira-knowledge): complete production capability (UI, Observability, Docs, Regression)

- Phase 9: Extend JiraConnectDialog with Project Knowledge config, mapping, and Sync Now button - Phase 10: Wire JiraSyncService to CanonicalAuditLogger for sync start/complete/fail events - Phase 11: Add retrieval regression suite with synthetic corpus and baseline metrics - Phase 14: Add comprehensive production guide (docs/jira-knowledge-guide.md) - Fix UI status label references (self.status -> self.conn_status) - Implement real sync trigger logic in UI using JiraSyncService
2026-09-07 01:42:45 +09:00
parent 1caded98e9
commit 9b4dc01c1a
4 changed files with 559 additions and 35 deletions
@@ -0,0 +1,214 @@
+"""Retrieval regression suite for Jira Project Knowledge.
+
+This suite uses a synthetic Jira corpus to evaluate retrieval quality without
+requiring a live Jira instance or confidential customer data. It covers:
+- Exact term matching
+- Paraphrasing
+- Ambiguous queries
+- Negative/no-result cases
+- Cross-project isolation
+- Citation completeness
+"""
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+from cowork_local.application.jira_knowledge.index_repository import JiraKnowledgeIndex
+from cowork_local.domain.jira_knowledge.canonical_issue import CanonicalJiraIssue, JiraProvenance
+from cowork_local.mcp_servers.project_context.providers.jira_knowledge import JiraKnowledgeProvider, _JiraKbTarget
+
+
+# ---------------------------------------------------------------------------
+# Synthetic Corpus
+# ---------------------------------------------------------------------------
+
+PROJECT_A = "proj-alpha"
+PROJECT_B = "proj-beta"
+JIRA_KEY_A = "ALPHA"
+JIRA_KEY_B = "BETA"
+
+CORPUS = [
+    # Project A: Requirements
+    CanonicalJiraIssue(
+        knowledge_id=f"{JIRA_KEY_A}/REQ-1",
+        project_id=PROJECT_A,
+        title="User Authentication Requirement",
+        content="The system must support user login via email and password. Account lockout occurs after 5 failed attempts.",
+        provenance=JiraProvenance(system="jira", issue_key="REQ-1", project_key=JIRA_KEY_A, source_url="http://jira/REQ-1", source_updated="2026-01-01T00:00:00Z", issue_type="Requirement", status="Done"),
+        ingested_at="2026-01-01T00:00:00Z"
+    ),
+    CanonicalJiraIssue(
+        knowledge_id=f"{JIRA_KEY_A}/REQ-2",
+        project_id=PROJECT_A,
+        title="Password Reset Policy",
+        content="Password reset links expire after 30 minutes. Users must verify their email address.",
+        provenance=JiraProvenance(system="jira", issue_key="REQ-2", project_key=JIRA_KEY_A, source_url="http://jira/REQ-2", source_updated="2026-01-02T00:00:00Z", issue_type="Requirement", status="Done"),
+        ingested_at="2026-01-02T00:00:00Z"
+    ),
+    # Project A: Bugs
+    CanonicalJiraIssue(
+        knowledge_id=f"{JIRA_KEY_A}/BUG-101",
+        project_id=PROJECT_A,
+        title="Database Timeout on Login",
+        content="Users experience a 500 error when logging in during peak hours due to database connection pool exhaustion.",
+        provenance=JiraProvenance(system="jira", issue_key="BUG-101", project_key=JIRA_KEY_A, source_url="http://jira/BUG-101", source_updated="2026-02-01T00:00:00Z", issue_type="Bug", status="Open"),
+        ingested_at="2026-02-01T00:00:00Z"
+    ),
+    # Project B: Secret/Isolation Test
+    CanonicalJiraIssue(
+        knowledge_id=f"{JIRA_KEY_B}/SECRET-1",
+        project_id=PROJECT_B,
+        title="Project Beta Secret Key",
+        content="The secret key for Project Beta is SUPER_SECRET_BETA_KEY_12345. Do not share.",
+        provenance=JiraProvenance(system="jira", issue_key="SECRET-1", project_key=JIRA_KEY_B, source_url="http://jira/SECRET-1", source_updated="2026-01-01T00:00:00Z", issue_type="Task", status="Done"),
+        ingested_at="2026-01-01T00:00:00Z"
+    ),
+    # Project B: Similar terminology to Project A (for ambiguity test)
+    CanonicalJiraIssue(
+        knowledge_id=f"{JIRA_KEY_B}/REQ-1",
+        project_id=PROJECT_B,
+        title="User Authentication Requirement (Beta)",
+        content="The beta system supports SSO login. Account lockout is disabled for testing.",
+        provenance=JiraProvenance(system="jira", issue_key="REQ-1", project_key=JIRA_KEY_B, source_url="http://jira/REQ-1", source_updated="2026-01-01T00:00:00Z", issue_type="Requirement", status="Done"),
+        ingested_at="2026-01-01T00:00:00Z"
+    ),
+]
+
+
+@pytest.fixture
+def populated_index(tmp_path: Path) -> JiraKnowledgeIndex:
+    """Create an index populated with the synthetic corpus."""
+    index = JiraKnowledgeIndex(index_root=tmp_path)
+    for issue in CORPUS:
+        index.upsert(issue)
+    return index
+
+
+@pytest.fixture
+def provider_a(populated_index: JiraKnowledgeIndex) -> JiraKnowledgeProvider:
+    """Provider scoped to Project A."""
+    target = _JiraKbTarget(cowork_project_id=PROJECT_A, jira_project_key=JIRA_KEY_A)
+    return JiraKnowledgeProvider(target, index=populated_index)
+
+
+@pytest.fixture
+def provider_b(populated_index: JiraKnowledgeIndex) -> JiraKnowledgeProvider:
+    """Provider scoped to Project B."""
+    target = _JiraKbTarget(cowork_project_id=PROJECT_B, jira_project_key=JIRA_KEY_B)
+    return JiraKnowledgeProvider(target, index=populated_index)
+
+
+# ---------------------------------------------------------------------------
+# Retrieval Quality Tests
+# ---------------------------------------------------------------------------
+
+def test_exact_term_match(provider_a: JiraKnowledgeProvider):
+    """Query with exact terms from REQ-1 should return REQ-1."""
+    result = provider_a.search_knowledge(project_id=PROJECT_A, query="account lockout 5 failed attempts")
+    assert result["returned"] > 0
+    assert any("REQ-1" in item["document_id"] for item in result["items"])
+
+
+def test_paraphrase_match(provider_a: JiraKnowledgeProvider):
+    """Query paraphrasing REQ-2 should return REQ-2."""
+    result = provider_a.search_knowledge(project_id=PROJECT_A, query="how long does password reset link last")
+    assert result["returned"] > 0
+    assert any("REQ-2" in item["document_id"] for item in result["items"])
+
+
+def test_ambiguous_query_prefers_local_context(provider_a: JiraKnowledgeProvider):
+    """Query 'authentication' exists in both projects, but provider_a should only return Project A results."""
+    result = provider_a.search_knowledge(project_id=PROJECT_A, query="user authentication login")
+    assert result["returned"] > 0
+    for item in result["items"]:
+        assert PROJECT_A in item["document_id"] or JIRA_KEY_A in item["document_id"]
+        assert PROJECT_B not in item["document_id"]
+
+
+def test_no_result_query(provider_a: JiraKnowledgeProvider):
+    """Query with no matching terms should return empty results."""
+    result = provider_a.search_knowledge(project_id=PROJECT_A, query="quantum computing blockchain")
+    assert result["returned"] == 0
+    assert result["items"] == ()
+
+
+def test_cross_project_isolation(provider_a: JiraKnowledgeProvider):
+    """Project A provider must never return Project B's secret."""
+    result = provider_a.search_knowledge(project_id=PROJECT_A, query="SUPER_SECRET_BETA_KEY_12345")
+    assert result["returned"] == 0
+    # Double check: ensure the secret string is not in any excerpt
+    for item in result["items"]:
+        assert "SUPER_SECRET_BETA_KEY_12345" not in item["excerpt"]
+
+
+def test_citation_completeness(provider_a: JiraKnowledgeProvider):
+    """Every result must have a valid Jira source URL."""
+    result = provider_a.search_knowledge(project_id=PROJECT_A, query="database timeout")
+    assert result["returned"] > 0
+    for item in result["items"]:
+        assert "source" in item
+        assert "url" in item["source"]
+        assert item["source"]["url"].startswith("http")
+        assert "system" in item["source"]
+        assert item["source"]["system"] == "jira"
+
+
+def test_bug_retrieval(provider_a: JiraKnowledgeProvider):
+    """Query about bugs should return BUG-101."""
+    result = provider_a.search_knowledge(project_id=PROJECT_A, query="500 error login peak hours")
+    assert result["returned"] > 0
+    assert any("BUG-101" in item["document_id"] for item in result["items"])
+
+
+# ---------------------------------------------------------------------------
+# Metrics Collection (Baseline)
+# ---------------------------------------------------------------------------
+
+def test_baseline_metrics(provider_a: JiraKnowledgeProvider, provider_b: JiraKnowledgeProvider):
+    """Collect Hit@1, Hit@3, Hit@5 for a set of queries."""
+    queries = [
+        ("account lockout", ["REQ-1"]),
+        ("password reset expire", ["REQ-2"]),
+        ("database timeout", ["BUG-101"]),
+        ("SSO login", []),  # Should be empty for Project A
+    ]
+
+    hits_at_1 = 0
+    hits_at_3 = 0
+    hits_at_5 = 0
+    total = len(queries)
+
+    for query, expected_ids in queries:
+        result = provider_a.search_knowledge(project_id=PROJECT_A, query=query, top_k=5)
+        returned_ids = [item["document_id"] for item in result["items"]]
+
+        if not expected_ids:
+            if len(returned_ids) == 0:
+                hits_at_1 += 1
+                hits_at_3 += 1
+                hits_at_5 += 1
+            continue
+
+        found_at_1 = any(eid in rid for rid in returned_ids[:1] for eid in expected_ids)
+        found_at_3 = any(eid in rid for rid in returned_ids[:3] for eid in expected_ids)
+        found_at_5 = any(eid in rid for rid in returned_ids[:5] for eid in expected_ids)
+
+        if found_at_1: hits_at_1 += 1
+        if found_at_3: hits_at_3 += 1
+        if found_at_5: hits_at_5 += 1
+
+    # Record baseline (in a real CI, this would be asserted against a stored baseline)
+    print(f"\n--- Retrieval Baseline ---")
+    print(f"Hit@1: {hits_at_1}/{total} ({hits_at_1/total:.2f})")
+    print(f"Hit@3: {hits_at_3}/{total} ({hits_at_3/total:.2f})")
+    print(f"Hit@5: {hits_at_5}/{total} ({hits_at_5/total:.2f})")
+
+    # For this synthetic corpus, we expect perfect scores
+    assert hits_at_1 == total
+    assert hits_at_3 == total
+    assert hits_at_5 == total