feat: ship production local assurance viewer in core

2026-07-28 23:09:22 +07:00
parent 3cc897c5bb
commit 9c9efb1043
21 changed files with 2544 additions and 136 deletions
@@ -0,0 +1,182 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Proves that the release artifact—not the source checkout—contains a complete,
+# offline, read-only Local Assurance Viewer.
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
+WORK="$(mktemp -d)"
+DIST="$WORK/dist"
+PROJECT="$WORK/clean-project"
+VIEWER_STARTED=0
+
+cleanup() {
+  if [[ "$VIEWER_STARTED" == "1" && -n "${CASAN_BIN:-}" ]]; then
+    CASAN_APP_ROOT="$PROJECT" "$CASAN_BIN" dashboard stop >/dev/null 2>&1 || true
+  fi
+  rm -rf "$WORK"
+}
+trap cleanup EXIT
+
+CASAN_DIST_DIR="$DIST" bash "$ROOT/scripts/package-release.sh" core >/dev/null
+ARTIFACT="$DIST/casan-core-v$(cat "$ROOT/VERSION").tar.gz"
+tar -xzf "$ARTIFACT" -C "$WORK"
+BUNDLE="$WORK/casan-core-v$(cat "$ROOT/VERSION")"
+CASAN_BIN="$BUNDLE/bin/casan"
+
+test -f "$BUNDLE/packages/casan-harness/assets/local-viewer/index.html"
+test -f "$BUNDLE/packages/casan-harness/scripts/python/local_report.py"
+test -f "$BUNDLE/packages/casan-harness/scripts/python/local_viewer.py"
+test ! -e "$BUNDLE/packages/casan-control-panel"
+test ! -e "$BUNDLE/package.json"
+
+mkdir -p \
+  "$PROJECT/.casan" \
+  "$PROJECT/.specify/state" \
+  "$PROJECT/.specify/logs/trace" \
+  "$PROJECT/.specify/logs/trace-events" \
+  "$PROJECT/.specify/logs/cost"
+
+python3 - "$PROJECT" <<'PY'
+import json
+from pathlib import Path
+import sys
+
+root = Path(sys.argv[1])
+(root / ".casan/config.json").write_text(json.dumps({
+    "schema_version": "21.3",
+    "project_id": "artifact-project",
+    "project_name": "Artifact Project",
+    "edition": "core",
+    "enforcement_mode": "enforce",
+}), encoding="utf-8")
+trace_id = "artifact-run-1"
+trace = {
+    "schema_version": "20.1",
+    "trace_id": trace_id,
+    "project_id": "artifact-project",
+    "certified": True,
+    "certification_strength": "project_hook",
+    "finalized_at": "2026-07-28T10:00:07Z",
+    "tool_calls": 1,
+    "failures": 0,
+    "evidence": [
+        {"h": f"H{i}", "at": f"2026-07-28T10:00:0{i}Z", "decision": "pass", "kind": "gate", "detail": f"H{i} passed"}
+        for i in range(1, 8)
+    ],
+}
+(root / f".specify/logs/trace/agentic-{trace_id}.json").write_text(
+    json.dumps(trace), encoding="utf-8")
+metric = {
+    "timestamp": "2026-07-28T10:00:06Z",
+    "trace_id": trace_id,
+    "project_id": "artifact-project",
+    "harness": "H6-agentic",
+    "step": "agentic-turn",
+    "status": "success",
+    "latency_ms": 812,
+    "retry_count": 0,
+    "input_tokens": None,
+    "output_tokens": None,
+    "total_tokens": None,
+    "cost_estimate": None,
+    "telemetry_quality": "insufficient",
+    "alerts": [],
+}
+(root / ".specify/logs/cost/metrics.jsonl").write_text(
+    json.dumps(metric) + "\n", encoding="utf-8")
+(root / ".specify/state/latest-run.json").write_text(json.dumps({
+    "schema_version": 1,
+    "trace_id": trace_id,
+    "project_id": "artifact-project",
+    "certified": True,
+    "telemetry_quality": "insufficient",
+    "duration_ms": 812,
+    "tool_calls": 1,
+    "failures": 0,
+    "finalized_at": "2026-07-28T10:00:07Z",
+}), encoding="utf-8")
+PY
+
+(
+  cd "$PROJECT"
+  CASAN_APP_ROOT="$PROJECT" "$CASAN_BIN" report latest | grep -q "view_command=casan view artifact-run-1"
+  CASAN_APP_ROOT="$PROJECT" "$CASAN_BIN" report export artifact-run-1 \
+    --format html --output "$WORK/run.html" >/dev/null
+  CASAN_APP_ROOT="$PROJECT" "$CASAN_BIN" report export --h6 \
+    --format json --output "$WORK/h6.json" >/dev/null
+  CASAN_APP_ROOT="$PROJECT" "$CASAN_BIN" report export --h6 \
+    --format html --output "$WORK/h6.html" >/dev/null
+)
+grep -q "Independent assurance dossier" "$WORK/run.html"
+grep -q "H6 · AgentOps Report" "$WORK/h6.html"
+python3 - "$WORK/h6.json" <<'PY'
+import json, sys
+d = json.load(open(sys.argv[1], encoding="utf-8"))
+assert d["harness"] == "H6"
+assert d["summary"]["runs"] == 1
+assert d["summary"]["tokens"]["total"] is None
+assert d["summary"]["cost_usd"]["estimated"] is None
+PY
+
+URL="$(CASAN_APP_ROOT="$PROJECT" "$CASAN_BIN" view artifact-run-1 --no-open)"
+VIEWER_STARTED=1
+python3 - "$URL" <<'PY'
+import json
+import sys
+from urllib.error import HTTPError
+from urllib.parse import parse_qs, urlparse
+from urllib.request import Request, urlopen
+
+url = urlparse(sys.argv[1])
+token = parse_qs(url.query)["token"][0]
+base = f"{url.scheme}://{url.netloc}"
+
+def get(path):
+    request = Request(base + path, headers={"X-CASAN-Viewer-Token": token})
+    return json.loads(urlopen(request, timeout=3).read())
+
+project = get("/api/v1/project")
+assert project["project_id"] == "artifact-project"
+assert project["viewer"] == {
+    "mode": "local_core",
+    "scope": "single_project",
+    "read_only": True,
+    "network": "loopback_only",
+}
+run = get("/api/v1/reports/run/artifact-run-1")
+assert run["verdict"] == "certified"
+assert run["summary"]["gates_observed"] == 7
+h6 = get("/api/v1/reports/h6?run=artifact-run-1")
+assert h6["summary"]["runs"] == 1
+
+try:
+    urlopen(Request(base + "/api/v1/project"), timeout=3)
+    raise AssertionError("API accepted a request without the session token")
+except HTTPError as error:
+    assert error.code == 401
+
+try:
+    urlopen(Request(
+        base + "/api/v1/project",
+        method="POST",
+        headers={"X-CASAN-Viewer-Token": token},
+    ), timeout=3)
+    raise AssertionError("read-only viewer accepted POST")
+except HTTPError as error:
+    assert error.code == 405
+
+try:
+    urlopen(Request(
+        base + "/api/v1/project",
+        headers={"X-CASAN-Viewer-Token": token, "Host": "attacker.example"},
+    ), timeout=3)
+    raise AssertionError("viewer accepted a non-loopback Host header")
+except HTTPError as error:
+    assert error.code == 403
+PY
+
+CASAN_APP_ROOT="$PROJECT" "$CASAN_BIN" dashboard status | grep -q '"mode": "local_core"'
+CASAN_APP_ROOT="$PROJECT" "$CASAN_BIN" dashboard stop | grep -q "CASAN_VIEW_STOPPED"
+VIEWER_STARTED=0
+echo "PASS: Core release artifact provides offline visual reports without Platform or Node"
@@ -0,0 +1,149 @@
+#!/usr/bin/env python3
+"""Acceptance tests for the stdlib-only CASAN Core reporting boundary."""
+
+from __future__ import annotations
+
+import importlib.util
+import json
+from pathlib import Path
+import tempfile
+import unittest
+
+
+MODULE = Path(__file__).resolve().parents[1] / "scripts" / "python" / "local_report.py"
+SPEC = importlib.util.spec_from_file_location("casan_local_report", MODULE)
+REPORT = importlib.util.module_from_spec(SPEC)
+assert SPEC.loader
+SPEC.loader.exec_module(REPORT)
+
+
+class LocalReportTests(unittest.TestCase):
+    def setUp(self):
+        self.temporary = tempfile.TemporaryDirectory()
+        self.root = Path(self.temporary.name)
+        (self.root / ".casan").mkdir()
+        (self.root / ".casan" / "config.json").write_text(json.dumps({
+            "project_id": "project-one",
+            "project_name": "Project One",
+            "edition": "core",
+            "enforcement_mode": "enforce",
+            "maturity": {"level": 4, "status": "assessed"},
+        }), encoding="utf-8")
+        for path in (
+            ".specify/state",
+            ".specify/logs/trace",
+            ".specify/logs/trace-events",
+            ".specify/logs/cost",
+            ".specify/logs/level5",
+            ".specify/agentops",
+        ):
+            (self.root / path).mkdir(parents=True, exist_ok=True)
+
+    def tearDown(self):
+        self.temporary.cleanup()
+
+    def write_jsonl(self, relative, rows):
+        path = self.root / relative
+        path.write_text(
+            "".join(json.dumps(row, separators=(",", ":")) + "\n" for row in rows),
+            encoding="utf-8",
+        )
+
+    def write_run(self, trace_id="trace-1", quality="partial"):
+        trace = {
+            "trace_id": trace_id,
+            "project_id": "project-one",
+            "certified": True,
+            "certification_strength": "project_hook",
+            "finalized_at": "2026-07-28T10:00:07Z",
+            "tool_calls": 2,
+            "failures": 0,
+            "evidence": [
+                {"h": f"H{index}", "at": f"2026-07-28T10:00:0{index}Z", "decision": "pass", "kind": "gate", "detail": f"H{index} passed"}
+                for index in range(1, 8)
+            ],
+        }
+        (self.root / f".specify/logs/trace/agentic-{trace_id}.json").write_text(
+            json.dumps(trace), encoding="utf-8")
+        metric = {
+            "timestamp": "2026-07-28T10:00:06Z",
+            "trace_id": trace_id,
+            "project_id": "project-one",
+            "step": "agentic-turn",
+            "status": "success",
+            "latency_ms": 1200,
+            "retry_count": 0,
+            "input_tokens": None,
+            "output_tokens": None,
+            "total_tokens": None,
+            "cost_estimate": None,
+            "telemetry_quality": quality,
+            "alerts": [],
+        }
+        self.write_jsonl(".specify/logs/cost/metrics.jsonl", [metric])
+        return metric
+
+    def test_core_empty_state_is_truthful(self):
+        registry = REPORT.runs(self.root)
+        h6 = REPORT.h6_report(self.root)
+        self.assertEqual(registry["count"], 0)
+        self.assertEqual(h6["verdict"], "no_data")
+        self.assertEqual(h6["summary"]["runs"], 0)
+        self.assertIsNone(h6["summary"]["tokens"]["total"])
+        self.assertTrue(any(item["code"] == "METRICS_MISSING" for item in h6["findings"]))
+
+    def test_run_reconstructs_all_gates_and_redacts_sensitive_fields(self):
+        self.write_run()
+        event = {
+            "timestamp": "2026-07-28T10:00:01Z",
+            "trace_id": "trace-1",
+            "gate_id": "H1-context",
+            "status": "pass",
+            "reason": "admitted",
+            "evidence": {"kind": "context", "prompt": "private customer prompt", "access_token": "secret"},
+        }
+        self.write_jsonl(".specify/logs/trace-events/trace-1.jsonl", [event])
+        report = REPORT.run_report(self.root, "trace-1")
+        self.assertEqual(report["verdict"], "certified")
+        self.assertEqual(report["summary"]["gates_observed"], 1)
+        self.assertEqual(report["gates"][0]["evidence"]["prompt"], "[redacted]")
+        self.assertEqual(report["gates"][0]["evidence"]["access_token"], "[redacted]")
+        self.assertIsNone(report["h6"]["total_tokens"])
+
+    def test_legacy_trace_reconstructs_h1_to_h7(self):
+        self.write_run()
+        graph = REPORT.trace_graph(self.root, "trace-1")
+        self.assertTrue(graph["terminal"])
+        self.assertEqual(graph["progress"], 7)
+        self.assertEqual([node["status"] for node in graph["nodes"]], [
+            "pass", "pass", "pass", "pass", "pass", "warning", "pass",
+        ])
+
+    def test_h6_preserves_unknown_token_and_cost_as_null(self):
+        self.write_run()
+        report = REPORT.h6_report(self.root, run="trace-1")
+        self.assertEqual(report["summary"]["coverage"]["token_pct"], 0)
+        self.assertEqual(report["summary"]["coverage"]["cost_pct"], 0)
+        self.assertIsNone(report["summary"]["tokens"]["total"])
+        self.assertIsNone(report["summary"]["cost_usd"]["estimated"])
+        self.assertEqual(report["data_quality"]["status"], "insufficient")
+        self.assertIn("TELEMETRY_COVERAGE_GAP", report["verdict_reasons"])
+
+    def test_self_contained_html_escapes_evidence(self):
+        self.write_run()
+        report = REPORT.run_report(self.root, "trace-1")
+        report["gates"][0]["reason"] = "<script>alert(1)</script>"
+        html = REPORT.report_html(report, "run")
+        self.assertIn("<!doctype html>", html.lower())
+        self.assertNotIn("<script>alert(1)</script>", html)
+        self.assertIn("&lt;script&gt;alert(1)&lt;/script&gt;", html)
+        self.assertIn("Machine-auditable contract", html)
+
+    def test_unsafe_trace_id_never_reads_outside_project(self):
+        report = REPORT.run_report(self.root, "../../etc/passwd")
+        self.assertEqual(report["verdict"], "not_found")
+        self.assertFalse(report["source"]["trace_found"])
+
+
+if __name__ == "__main__":
+    unittest.main()