feat: ship production local assurance viewer in core
This commit is contained in:
thanhnv
@@ -0,0 +1,701 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Zero-dependency, read-only reporting engine for CASAN Core.
|
||||
|
||||
The evidence files under .specify remain the source of truth. This module only
|
||||
builds bounded, sanitized projections for the Local Assurance Viewer and
|
||||
on-demand JSON/HTML exports.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import Counter, defaultdict
|
||||
from datetime import datetime, timezone
|
||||
from html import escape
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
|
||||
SAFE_ID = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9._:-]{0,127}$")
|
||||
STALE_AFTER_S = 900
|
||||
MAX_SOURCE_BYTES = 20 * 1024 * 1024
|
||||
MAX_RECORDS = 2000
|
||||
GATES = (
|
||||
("H1-context", "H1 · Context", "Prompt contract, mode and risk classification"),
|
||||
("H2-tool", "H2 · Tool", "Allowlisted source and tool preparation"),
|
||||
("H3-eval", "H3 · Eval", "Grounded synthesis and quality evaluation"),
|
||||
("H4-security", "H4 · Security", "Input and output security boundary"),
|
||||
("H5-governance", "H5 · Governance", "Decision policy and append-only audit"),
|
||||
("H6-agentops", "H6 · AgentOps", "Runtime, token, cost and failure telemetry"),
|
||||
("H7-orchestration", "H7 · Orchestration", "Final governed outcome and certification"),
|
||||
)
|
||||
GATE_MAP = {f"H{index}": gate[0] for index, gate in enumerate(GATES, 1)}
|
||||
SENSITIVE_KEYS = {
|
||||
"authorization", "api_key", "apikey", "access_token", "refresh_token",
|
||||
"password", "secret", "prompt", "raw_prompt", "tool_input", "tool_output",
|
||||
"request_body", "response_body",
|
||||
}
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def state_root(root: str | Path) -> Path:
|
||||
override = os.environ.get("CASAN_STATE_ROOT")
|
||||
return Path(override).expanduser().resolve() if override else Path(root).resolve() / ".specify"
|
||||
|
||||
|
||||
def _read_text(path: Path, maximum: int = MAX_SOURCE_BYTES) -> str:
|
||||
try:
|
||||
if not path.is_file() or path.stat().st_size > maximum:
|
||||
return ""
|
||||
return path.read_text(encoding="utf-8", errors="replace")
|
||||
except OSError:
|
||||
return ""
|
||||
|
||||
|
||||
def _read_tail_text(path: Path, maximum: int = MAX_SOURCE_BYTES) -> str:
|
||||
try:
|
||||
if not path.is_file():
|
||||
return ""
|
||||
size = path.stat().st_size
|
||||
with path.open("rb") as handle:
|
||||
if size > maximum:
|
||||
handle.seek(-maximum, os.SEEK_END)
|
||||
handle.readline() # discard a possibly partial first record
|
||||
return handle.read(maximum).decode("utf-8", errors="replace")
|
||||
except OSError:
|
||||
return ""
|
||||
|
||||
|
||||
def read_json(path: Path) -> dict[str, Any]:
|
||||
try:
|
||||
value = json.loads(_read_text(path))
|
||||
return value if isinstance(value, dict) else {}
|
||||
except (TypeError, ValueError):
|
||||
return {}
|
||||
|
||||
|
||||
def read_jsonl(path: Path, maximum_records: int = MAX_RECORDS) -> list[dict[str, Any]]:
|
||||
rows: list[dict[str, Any]] = []
|
||||
for line in _read_tail_text(path).splitlines()[-maximum_records:]:
|
||||
try:
|
||||
value = json.loads(line)
|
||||
except ValueError:
|
||||
continue
|
||||
if isinstance(value, dict):
|
||||
rows.append(value)
|
||||
return rows
|
||||
|
||||
|
||||
def _safe_value(value: Any, depth: int = 0) -> Any:
|
||||
if depth > 5:
|
||||
return "[depth limited]"
|
||||
if isinstance(value, dict):
|
||||
result: dict[str, Any] = {}
|
||||
for key, item in list(value.items())[:80]:
|
||||
normalized = str(key).lower().replace("-", "_")
|
||||
if normalized in SENSITIVE_KEYS:
|
||||
result[str(key)] = "[redacted]"
|
||||
else:
|
||||
result[str(key)] = _safe_value(item, depth + 1)
|
||||
return result
|
||||
if isinstance(value, list):
|
||||
return [_safe_value(item, depth + 1) for item in value[:80]]
|
||||
if isinstance(value, str):
|
||||
return value if len(value) <= 4096 else value[:4096] + "…"
|
||||
if value is None or isinstance(value, (bool, int, float)):
|
||||
return value
|
||||
return str(value)[:4096]
|
||||
|
||||
|
||||
def sanitize(row: dict[str, Any]) -> dict[str, Any]:
|
||||
value = _safe_value(row)
|
||||
return value if isinstance(value, dict) else {}
|
||||
|
||||
|
||||
def _number(value: Any) -> float | None:
|
||||
if isinstance(value, bool):
|
||||
return None
|
||||
if isinstance(value, (int, float)) and math.isfinite(float(value)):
|
||||
return float(value)
|
||||
return None
|
||||
|
||||
|
||||
def _threshold(name: str, fallback: float) -> float:
|
||||
try:
|
||||
value = float(os.environ.get(name, str(fallback)))
|
||||
return value if math.isfinite(value) and value >= 0 else fallback
|
||||
except ValueError:
|
||||
return fallback
|
||||
|
||||
|
||||
def _text(value: Any, fallback: str = "") -> str:
|
||||
return value.strip() if isinstance(value, str) and value.strip() else fallback
|
||||
|
||||
|
||||
def _row_run(row: dict[str, Any]) -> str:
|
||||
return _text(row.get("run_id") or row.get("trace_id"), "unattributed")
|
||||
|
||||
|
||||
def _row_project(row: dict[str, Any]) -> str:
|
||||
return _text(row.get("project") or row.get("project_id"), "default")
|
||||
|
||||
|
||||
def _parse_time(value: Any) -> float | None:
|
||||
if not isinstance(value, str) or not value:
|
||||
return None
|
||||
try:
|
||||
return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp()
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _status(value: Any) -> str:
|
||||
status = _text(value).lower()
|
||||
if status in {"success", "pass", "passed", "allow", "allowed", "answered", "opened", "certified"}:
|
||||
return "pass"
|
||||
if status in {"warn", "warning", "degraded", "partial", "insufficient"}:
|
||||
return "warning"
|
||||
if status in {"block", "blocked", "deny", "denied", "flag", "non_certified"}:
|
||||
return "blocked"
|
||||
if status in {"fail", "failed", "error"}:
|
||||
return "error"
|
||||
if status in {"running", "skipped"}:
|
||||
return status
|
||||
return "queued"
|
||||
|
||||
|
||||
def paths(root: str | Path) -> dict[str, Path]:
|
||||
state = state_root(root)
|
||||
return {
|
||||
"config": Path(root).resolve() / ".casan" / "config.json",
|
||||
"receipt": state / "state" / "latest-run.json",
|
||||
"metrics": state / "logs" / "cost" / "metrics.jsonl",
|
||||
"provider": state / "logs" / "level5" / "provider-usage.jsonl",
|
||||
"alerts": state / "agentops" / "alerts.log",
|
||||
"trace_dir": state / "logs" / "trace",
|
||||
"event_dir": state / "logs" / "trace-events",
|
||||
}
|
||||
|
||||
|
||||
def project_profile(root: str | Path) -> dict[str, Any]:
|
||||
config = read_json(paths(root)["config"])
|
||||
maturity = config.get("maturity") if isinstance(config.get("maturity"), dict) else {}
|
||||
return {
|
||||
"project_id": _text(config.get("project_id"), Path(root).resolve().name),
|
||||
"project_name": _text(config.get("project_name"), Path(root).resolve().name),
|
||||
"edition": _text(config.get("edition") or config.get("target_level_name"), "core"),
|
||||
"edition_status": _text(config.get("edition_status"), "configured"),
|
||||
"enforcement_mode": _text(config.get("enforcement_mode"), "unknown"),
|
||||
"integration_mode": _text(config.get("integration_mode"), "unknown"),
|
||||
"clients": [str(value) for value in config.get("clients", []) if isinstance(value, str)],
|
||||
"maturity": {
|
||||
"level": maturity.get("level") if _number(maturity.get("level")) is not None else None,
|
||||
"status": _text(maturity.get("status"), "not_assessed"),
|
||||
"evidence": _text(maturity.get("evidence")) or None,
|
||||
},
|
||||
"viewer": {
|
||||
"mode": "local_core",
|
||||
"scope": "single_project",
|
||||
"read_only": True,
|
||||
"network": "loopback_only",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _trace(root: str | Path, trace_id: str) -> dict[str, Any]:
|
||||
if not SAFE_ID.fullmatch(trace_id):
|
||||
return {}
|
||||
return read_json(paths(root)["trace_dir"] / f"agentic-{trace_id}.json")
|
||||
|
||||
|
||||
def _metric_rows(root: str | Path) -> list[dict[str, Any]]:
|
||||
return read_jsonl(paths(root)["metrics"])
|
||||
|
||||
|
||||
def trace_graph(root: str | Path, trace_id: str) -> dict[str, Any]:
|
||||
if not SAFE_ID.fullmatch(trace_id):
|
||||
return _empty_graph(trace_id)
|
||||
source = paths(root)
|
||||
events: list[dict[str, Any]] = []
|
||||
for row in read_jsonl(source["event_dir"] / f"{trace_id}.jsonl", 500):
|
||||
if row.get("trace_id") != trace_id or row.get("gate_id") not in {gate[0] for gate in GATES}:
|
||||
continue
|
||||
events.append({
|
||||
"timestamp": _text(row.get("timestamp")),
|
||||
"trace_id": trace_id,
|
||||
"gate_id": _text(row.get("gate_id")),
|
||||
"status": _status(row.get("status")),
|
||||
"reason": _text(row.get("reason"), "Evidence observed"),
|
||||
"evidence": sanitize(row.get("evidence") if isinstance(row.get("evidence"), dict) else {}),
|
||||
})
|
||||
if not events:
|
||||
trace = _trace(root, trace_id)
|
||||
harness = _text(trace.get("harness"))
|
||||
if harness in {gate[0] for gate in GATES}:
|
||||
events.append({
|
||||
"timestamp": _text(trace.get("timestamp")),
|
||||
"trace_id": trace_id,
|
||||
"gate_id": harness,
|
||||
"status": _status(trace.get("status") or trace.get("action")),
|
||||
"reason": f"Legacy {harness} trace",
|
||||
"evidence": sanitize({
|
||||
"mode": trace.get("mode"),
|
||||
"action": trace.get("action"),
|
||||
"risk_level": trace.get("risk_level"),
|
||||
}),
|
||||
})
|
||||
evidence_rows = trace.get("evidence") if isinstance(trace.get("evidence"), list) else []
|
||||
for evidence in evidence_rows[:200]:
|
||||
if not isinstance(evidence, dict):
|
||||
continue
|
||||
for marker in _text(evidence.get("h")).split("/"):
|
||||
gate_id = GATE_MAP.get(marker)
|
||||
if gate_id:
|
||||
events.append({
|
||||
"timestamp": _text(evidence.get("at") or trace.get("finalized_at")),
|
||||
"trace_id": trace_id,
|
||||
"gate_id": gate_id,
|
||||
"status": _status(evidence.get("decision")),
|
||||
"reason": f"{_text(evidence.get('kind'), 'legacy-evidence')}: {_text(evidence.get('detail'))}",
|
||||
"evidence": sanitize({
|
||||
"kind": evidence.get("kind"),
|
||||
"decision": evidence.get("decision"),
|
||||
"detail": evidence.get("detail"),
|
||||
"certification_strength": trace.get("certification_strength"),
|
||||
}),
|
||||
})
|
||||
for metric in _metric_rows(root):
|
||||
if metric.get("trace_id") != trace_id:
|
||||
continue
|
||||
quality = _text(metric.get("telemetry_quality"))
|
||||
events.append({
|
||||
"timestamp": _text(metric.get("timestamp")),
|
||||
"trace_id": trace_id,
|
||||
"gate_id": "H6-agentops",
|
||||
"status": "warning" if quality in {"partial", "insufficient"} else _status(metric.get("status")),
|
||||
"reason": _text(metric.get("step"), "Runtime metric"),
|
||||
"evidence": sanitize({
|
||||
"latency_ms": metric.get("latency_ms"),
|
||||
"total_tokens": metric.get("total_tokens"),
|
||||
"cost_estimate": metric.get("cost_estimate"),
|
||||
"telemetry_quality": metric.get("telemetry_quality"),
|
||||
}),
|
||||
})
|
||||
events.sort(key=lambda row: _text(row.get("timestamp")))
|
||||
nodes = []
|
||||
for gate_id, title, description in GATES:
|
||||
gate_events = [row for row in events if row["gate_id"] == gate_id]
|
||||
latest = gate_events[-1] if gate_events else {}
|
||||
nodes.append({
|
||||
"id": gate_id,
|
||||
"title": title,
|
||||
"description": description,
|
||||
"status": latest.get("status", "queued"),
|
||||
"reason": latest.get("reason", "Waiting for evidence"),
|
||||
"updated_at": latest.get("timestamp") or None,
|
||||
"evidence": latest.get("evidence", {}),
|
||||
"events": gate_events,
|
||||
})
|
||||
outcome = nodes[-1]
|
||||
return {
|
||||
"found": bool(events),
|
||||
"trace_id": trace_id,
|
||||
"updated_at": events[-1]["timestamp"] if events else None,
|
||||
"terminal": outcome["status"] in {"pass", "blocked", "error"},
|
||||
"progress": sum(1 for node in nodes if node["status"] != "queued"),
|
||||
"nodes": nodes,
|
||||
"events": events,
|
||||
}
|
||||
|
||||
|
||||
def _empty_graph(trace_id: str) -> dict[str, Any]:
|
||||
nodes = [{
|
||||
"id": gate_id, "title": title, "description": description,
|
||||
"status": "queued", "reason": "Waiting for evidence",
|
||||
"updated_at": None, "evidence": {}, "events": [],
|
||||
} for gate_id, title, description in GATES]
|
||||
return {
|
||||
"found": False, "trace_id": trace_id, "updated_at": None,
|
||||
"terminal": False, "progress": 0, "nodes": nodes, "events": [],
|
||||
}
|
||||
|
||||
|
||||
def run_report(root: str | Path, trace_id: str) -> dict[str, Any]:
|
||||
graph = trace_graph(root, trace_id)
|
||||
trace = _trace(root, trace_id)
|
||||
metric = next(
|
||||
(row for row in reversed(_metric_rows(root)) if row.get("trace_id") == trace_id),
|
||||
None,
|
||||
)
|
||||
profile = project_profile(root)
|
||||
metric_row = metric or {}
|
||||
duration = _number(metric_row.get("duration_ms"))
|
||||
if duration is None:
|
||||
duration = _number(metric_row.get("latency_ms"))
|
||||
found = bool(trace or graph["found"] or metric)
|
||||
finalized = _text(trace.get("finalized_at") or metric_row.get("finished_at")) or None
|
||||
if not found:
|
||||
verdict = "not_found"
|
||||
elif not graph["terminal"] and not finalized:
|
||||
verdict = "in_progress"
|
||||
elif trace.get("certified") is True:
|
||||
verdict = "certified"
|
||||
else:
|
||||
verdict = "non_certified"
|
||||
cost = metric_row.get("cost") if isinstance(metric_row.get("cost"), dict) else {}
|
||||
reasons = trace.get("certification_reasons") if isinstance(trace.get("certification_reasons"), list) else []
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"report_id": f"RUN-{trace_id}",
|
||||
"generated_at": utc_now(),
|
||||
"trace_id": trace_id,
|
||||
"project": {
|
||||
"id": profile["project_id"],
|
||||
"name": profile["project_name"],
|
||||
"edition": profile["edition"],
|
||||
"maturity": profile["maturity"],
|
||||
},
|
||||
"verdict": verdict,
|
||||
"certification": {
|
||||
"strength": _text(trace.get("certification_strength") or metric_row.get("certification_strength")) or None,
|
||||
"reasons": [str(value) for value in reasons if isinstance(value, str)],
|
||||
"finalized_at": finalized,
|
||||
},
|
||||
"summary": {
|
||||
"gates_observed": graph["progress"],
|
||||
"gates_total": 7,
|
||||
"tool_calls": int(_number(trace.get("tool_calls") or metric_row.get("tool_calls")) or 0),
|
||||
"failures": int(_number(trace.get("failures") or metric_row.get("failures")) or 0),
|
||||
"duration_ms": duration,
|
||||
"telemetry_quality": _text(metric_row.get("telemetry_quality"), "unknown"),
|
||||
"token_usage_available": any(_number(metric_row.get(key)) is not None for key in ("total_tokens", "input_tokens", "output_tokens")),
|
||||
"cost_available": _number(metric_row.get("cost_estimate")) is not None or _number(cost.get("amount")) is not None,
|
||||
},
|
||||
"gates": graph["nodes"],
|
||||
"h6": sanitize(metric_row) if metric else None,
|
||||
"source": {
|
||||
"trace_found": bool(trace),
|
||||
"graph_found": graph["found"],
|
||||
"metric_found": metric is not None,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def runs(root: str | Path, limit: int = 100) -> dict[str, Any]:
|
||||
limit = max(1, min(int(limit), 200))
|
||||
rows = _metric_rows(root)
|
||||
receipts = read_json(paths(root)["receipt"])
|
||||
safe_rows = [sanitize(row) for row in reversed(rows[-limit:])]
|
||||
if not safe_rows and receipts.get("trace_id"):
|
||||
safe_rows = [sanitize({
|
||||
"trace_id": receipts.get("trace_id"),
|
||||
"project_id": receipts.get("project_id"),
|
||||
"status": "success" if receipts.get("certified") else "failed",
|
||||
"timestamp": receipts.get("finalized_at"),
|
||||
"latency_ms": receipts.get("duration_ms"),
|
||||
"tool_calls": receipts.get("tool_calls"),
|
||||
"failures": receipts.get("failures"),
|
||||
"telemetry_quality": receipts.get("telemetry_quality"),
|
||||
})]
|
||||
return {"count": len(rows) if rows else len(safe_rows), "runs": safe_rows, "latest": sanitize(receipts)}
|
||||
|
||||
|
||||
def _freshness(path: Path, source: str, required: bool) -> dict[str, Any]:
|
||||
try:
|
||||
stat = path.stat()
|
||||
age = max(0, int(datetime.now(timezone.utc).timestamp() - stat.st_mtime))
|
||||
present = path.is_file()
|
||||
except OSError:
|
||||
age, present = None, False
|
||||
return {
|
||||
"source": source,
|
||||
"path": str(path),
|
||||
"present": present,
|
||||
"updated_at": datetime.fromtimestamp(stat.st_mtime, timezone.utc).isoformat().replace("+00:00", "Z") if present else None,
|
||||
"age_s": age,
|
||||
"stale": bool(present and age is not None and age > STALE_AFTER_S),
|
||||
"required": required,
|
||||
}
|
||||
|
||||
|
||||
def _display_path(root: str | Path, path: Path) -> str:
|
||||
try:
|
||||
return str(path.resolve().relative_to(Path(root).resolve()))
|
||||
except (OSError, ValueError):
|
||||
return f"[external]/{path.name}"
|
||||
|
||||
|
||||
def _percentile(values: list[float], pct: int) -> float:
|
||||
if not values:
|
||||
return 0
|
||||
ordered = sorted(values)
|
||||
return ordered[max(0, math.ceil((pct / 100) * len(ordered)) - 1)]
|
||||
|
||||
|
||||
def _grouped(rows: list[dict[str, Any]], key) -> list[dict[str, Any]]:
|
||||
groups: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||
for row in rows:
|
||||
groups[key(row)].append(row)
|
||||
result = []
|
||||
for name, records in groups.items():
|
||||
latencies = [_number(row.get("latency_ms")) for row in records]
|
||||
latencies = [value for value in latencies if value is not None and value > 0]
|
||||
tokens = [_number(row.get("total_tokens")) for row in records]
|
||||
cost_values = [
|
||||
_number(row.get("cost_usd")) if _number(row.get("cost_usd")) is not None
|
||||
else _number(row.get("cost_estimate"))
|
||||
for row in records
|
||||
]
|
||||
result.append({
|
||||
"key": name,
|
||||
"runs": len(records),
|
||||
"failures": sum(1 for row in records if _text(row.get("status"), "unknown") == "failed"),
|
||||
"latency_avg_ms": round(sum(latencies) / len(latencies)) if latencies else 0,
|
||||
"tokens": round(sum(value for value in tokens if value is not None)) if any(value is not None for value in tokens) else None,
|
||||
"cost_usd": round(sum(value for value in cost_values if value is not None), 6) if any(value is not None for value in cost_values) else None,
|
||||
})
|
||||
return sorted(result, key=lambda row: (-row["runs"], row["key"]))
|
||||
|
||||
|
||||
def _matches(row: dict[str, Any], project: str | None, run: str | None,
|
||||
start: float | None, end: float | None) -> bool:
|
||||
if project and _row_project(row) != project:
|
||||
return False
|
||||
if run and _row_run(row) != run:
|
||||
return False
|
||||
timestamp = _parse_time(row.get("timestamp"))
|
||||
if start is not None and (timestamp is None or timestamp < start):
|
||||
return False
|
||||
if end is not None and (timestamp is None or timestamp > end):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _date_boundary(value: str | None, end: bool = False) -> tuple[str | None, float | None]:
|
||||
if not value:
|
||||
return None, None
|
||||
normalized = value
|
||||
if re.fullmatch(r"\d{4}-\d{2}-\d{2}", value):
|
||||
normalized += "T23:59:59.999Z" if end else "T00:00:00.000Z"
|
||||
parsed = _parse_time(normalized)
|
||||
if parsed is None:
|
||||
raise ValueError("invalid report time boundary")
|
||||
return datetime.fromtimestamp(parsed, timezone.utc).isoformat().replace("+00:00", "Z"), parsed
|
||||
|
||||
|
||||
def h6_report(root: str | Path, project: str | None = None, run: str | None = None,
|
||||
from_date: str | None = None, to_date: str | None = None,
|
||||
limit: int = 50) -> dict[str, Any]:
|
||||
for value in (project, run):
|
||||
if value and not SAFE_ID.fullmatch(value):
|
||||
raise ValueError("invalid project or run filter")
|
||||
normalized_from, start = _date_boundary(from_date)
|
||||
normalized_to, end = _date_boundary(to_date, True)
|
||||
if start is not None and end is not None and start > end:
|
||||
raise ValueError("invalid report time range")
|
||||
limit = max(1, min(int(limit), 200))
|
||||
source = paths(root)
|
||||
all_metrics = _metric_rows(root)
|
||||
all_provider = read_jsonl(source["provider"])
|
||||
all_alerts = read_jsonl(source["alerts"])
|
||||
metrics = [row for row in all_metrics if _matches(row, project, run, start, end)]
|
||||
provider = [row for row in all_provider if _matches(row, project, run, start, end)]
|
||||
alerts = [row for row in all_alerts if _matches(row, project, run, start, end)]
|
||||
latencies = [value for value in (_number(row.get("latency_ms")) for row in metrics) if value is not None and value > 0]
|
||||
failed = sum(1 for row in metrics if _text(row.get("status"), "unknown") == "failed")
|
||||
degraded = sum(1 for row in metrics if _text(row.get("status"), "unknown") == "degraded")
|
||||
success = sum(1 for row in metrics if _text(row.get("status"), "unknown") in {"success", "pass", "passed"})
|
||||
provider_token_runs = {_row_run(row) for row in provider if _number(row.get("total_tokens")) is not None}
|
||||
provider_cost_runs = {_row_run(row) for row in provider if _number(row.get("cost_usd")) is not None}
|
||||
token_records = sum(1 for row in metrics if any(_number(row.get(key)) is not None for key in ("total_tokens", "input_tokens", "output_tokens")) or _row_run(row) in provider_token_runs)
|
||||
cost_records = sum(1 for row in metrics if _number(row.get("cost_estimate")) is not None or _row_run(row) in provider_cost_runs)
|
||||
token_pct = round((token_records / len(metrics)) * 100, 1) if metrics else 0
|
||||
cost_pct = round((cost_records / len(metrics)) * 100, 1) if metrics else 0
|
||||
quality = Counter(_text(row.get("telemetry_quality"), "unknown") for row in metrics)
|
||||
evidence = [
|
||||
{**_freshness(source["metrics"], "metrics", True), "records": len(all_metrics)},
|
||||
{**_freshness(source["provider"], "provider", False), "records": len(all_provider)},
|
||||
{**_freshness(source["alerts"], "alerts", False), "records": len(all_alerts)},
|
||||
]
|
||||
for item in evidence:
|
||||
item["path"] = _display_path(root, Path(item["path"]))
|
||||
primary = evidence[0]
|
||||
failure_rate = round((failed / len(metrics)) * 100, 1) if metrics else 0
|
||||
p95 = _percentile(latencies, 95)
|
||||
failure_threshold = _threshold("CASAN_H6_FAILURE_RATE_THRESHOLD_PCT", 5)
|
||||
p95_threshold = _threshold("CASAN_H6_P95_LATENCY_THRESHOLD_MS", 5000)
|
||||
findings: list[dict[str, Any]] = []
|
||||
warnings: list[str] = []
|
||||
if not primary["present"]:
|
||||
findings.append({"severity": "critical", "code": "METRICS_MISSING", "message": "The required runtime metrics source is missing."})
|
||||
elif primary["stale"]:
|
||||
findings.append({"severity": "warning", "code": "METRICS_STALE", "message": "The primary runtime metrics source is stale.", "metric": "age_s", "value": primary["age_s"], "threshold": STALE_AFTER_S})
|
||||
if failure_rate > failure_threshold:
|
||||
findings.append({"severity": "critical", "code": "FAILURE_RATE_BREACH", "message": "Failure rate exceeds the configured H6 threshold.", "metric": "failure_rate_pct", "value": failure_rate, "threshold": failure_threshold})
|
||||
elif failed:
|
||||
findings.append({"severity": "warning", "code": "FAILURES_PRESENT", "message": f"{failed} failed run(s) are present in the selected scope.", "metric": "failed", "value": failed})
|
||||
if p95 > p95_threshold:
|
||||
findings.append({"severity": "warning", "code": "P95_LATENCY_BREACH", "message": "P95 latency exceeds the configured H6 threshold.", "metric": "p95_latency_ms", "value": p95, "threshold": p95_threshold})
|
||||
alert_counter: Counter[str] = Counter()
|
||||
fingerprints: set[tuple[str, str]] = set()
|
||||
for row in metrics + alerts:
|
||||
alert_types: list[str] = []
|
||||
if isinstance(row.get("alerts"), list):
|
||||
alert_types.extend(str(value) for value in row["alerts"] if isinstance(value, str))
|
||||
body = row.get("body") if isinstance(row.get("body"), dict) else {}
|
||||
if isinstance(body.get("alert.type"), str):
|
||||
alert_types.append(body["alert.type"])
|
||||
for alert in alert_types:
|
||||
row_run = _row_run(row)
|
||||
fingerprint = (
|
||||
_text(row.get("timestamp"), "unknown-time")
|
||||
if row_run == "unattributed" else row_run,
|
||||
alert,
|
||||
)
|
||||
if fingerprint not in fingerprints:
|
||||
alert_counter[alert] += 1
|
||||
fingerprints.add(fingerprint)
|
||||
alert_count = sum(alert_counter.values())
|
||||
if alert_count:
|
||||
findings.append({"severity": "warning", "code": "ALERTS_PRESENT", "message": f"{alert_count} alert signal(s) require review.", "metric": "alerts", "value": alert_count})
|
||||
if degraded:
|
||||
findings.append({"severity": "warning", "code": "DEGRADED_RUNS_PRESENT", "message": f"{degraded} degraded run(s) are present in the selected scope.", "metric": "degraded", "value": degraded})
|
||||
if metrics and (token_pct < 100 or cost_pct < 100):
|
||||
findings.append({"severity": "warning", "code": "TELEMETRY_COVERAGE_GAP", "message": "Some runs do not have reliable provider token or cost attribution.", "metric": "token_cost_coverage_pct", "value": f"{token_pct}/{cost_pct}", "threshold": "100/100"})
|
||||
if not provider:
|
||||
warnings.append("No provider usage records matched the selected scope; token and actual-cost breakdown may be incomplete.")
|
||||
if metrics and token_pct < 100:
|
||||
warnings.append(f"{token_pct}% of runtime records have reliable token attribution; unavailable values remain null, never zero.")
|
||||
if metrics and cost_pct < 100:
|
||||
warnings.append(f"{cost_pct}% of runtime records have reliable cost attribution; unavailable values remain null, never zero.")
|
||||
for item in evidence[1:]:
|
||||
if not item["present"]:
|
||||
warnings.append(f"Optional {item['source']} telemetry source is missing; its breakdown is unavailable.")
|
||||
elif item["stale"]:
|
||||
warnings.append(f"Optional {item['source']} telemetry source is stale; its breakdown may not reflect recent activity.")
|
||||
critical = any(item["severity"] == "critical" for item in findings)
|
||||
warning = any(item["severity"] == "warning" for item in findings)
|
||||
verdict = "no_data" if not metrics and not provider else "fail" if critical else "attention" if warning else "pass"
|
||||
status_counts = Counter(_text(row.get("status"), "unknown") for row in metrics)
|
||||
generated_at = utc_now()
|
||||
total = lambda rows, key: sum(value for value in (_number(row.get(key)) for row in rows) if value is not None)
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"report_id": "H6-" + re.sub(r"[-:.TZ+]", "", generated_at)[:14],
|
||||
"harness": "H6",
|
||||
"title": "H6 · AgentOps Report",
|
||||
"description": "Evidence-backed runtime, token, cost, failure, retry and alert telemetry.",
|
||||
"generated_at": generated_at,
|
||||
"scope": {"project": project, "from": normalized_from, "to": normalized_to, "run": run},
|
||||
"verdict": verdict,
|
||||
"verdict_reasons": [item["code"] for item in findings],
|
||||
"freshness": {"status": "missing" if not primary["present"] else "stale" if primary["stale"] else "live", "stale_after_s": STALE_AFTER_S, "primary_age_s": primary["age_s"], "sources": evidence},
|
||||
"summary": {
|
||||
"runs": len(metrics), "success": success, "failed": failed, "degraded": degraded,
|
||||
"failure_rate_pct": failure_rate, "retries": int(total(metrics, "retry_count")),
|
||||
"latency_ms": {
|
||||
"average": round(sum(latencies) / len(latencies)) if latencies else 0,
|
||||
"p50": _percentile(latencies, 50), "p95": p95,
|
||||
"p99": _percentile(latencies, 99), "max": max(latencies) if latencies else 0,
|
||||
},
|
||||
"tokens": {
|
||||
"input": total(metrics, "input_tokens") if any(_number(row.get("input_tokens")) is not None for row in metrics) else None,
|
||||
"output": total(metrics, "output_tokens") if any(_number(row.get("output_tokens")) is not None for row in metrics) else None,
|
||||
"total": total(metrics, "total_tokens") if any(_number(row.get("total_tokens")) is not None for row in metrics) else None,
|
||||
"provider_total": total(provider, "total_tokens") if any(_number(row.get("total_tokens")) is not None for row in provider) else None,
|
||||
},
|
||||
"cost_usd": {
|
||||
"provider_actual": round(total(provider, "cost_usd"), 6) if any(_number(row.get("cost_usd")) is not None for row in provider) else None,
|
||||
"estimated": round(total(metrics, "cost_estimate"), 6) if any(_number(row.get("cost_estimate")) is not None for row in metrics) else None,
|
||||
},
|
||||
"provider_calls": len(provider), "alerts": alert_count,
|
||||
"coverage": {
|
||||
"runtime_records": len(metrics), "token_records": token_records, "cost_records": cost_records,
|
||||
"token_pct": token_pct, "cost_pct": cost_pct,
|
||||
"quality": {name: quality.get(name, 0) for name in ("complete", "partial", "insufficient", "unknown")},
|
||||
},
|
||||
},
|
||||
"thresholds": {"failure_rate_pct": failure_threshold, "p95_latency_ms": p95_threshold, "freshness_age_s": STALE_AFTER_S},
|
||||
"findings": findings,
|
||||
"evidence_sources": evidence,
|
||||
"data_quality": {"status": "insufficient" if not primary["present"] or (metrics and not token_records and not cost_records) else "partial" if warnings else "complete", "warnings": warnings},
|
||||
"available_filters": {
|
||||
"projects": sorted({_row_project(row) for row in all_metrics + all_provider + all_alerts}),
|
||||
"runs": sorted({_row_run(row) for row in all_metrics + all_provider + all_alerts} - {"unattributed"}),
|
||||
},
|
||||
"details": {
|
||||
"by_status": [{"status": key, "count": value} for key, value in status_counts.most_common()],
|
||||
"by_step": _grouped(metrics, lambda row: _text(row.get("step"), "unknown-step")),
|
||||
"by_provider": _grouped(provider, lambda row: f"{_text(row.get('provider'), 'unknown-provider')} · {_text(row.get('model'), 'unknown-model')}"),
|
||||
"by_cost_source": [{"source": row["key"], "records": row["runs"], "cost_usd": row["cost_usd"] or 0} for row in _grouped(metrics, lambda row: _text(row.get("cost_source"), "unknown"))],
|
||||
"by_alert": [{"alert": key, "count": value} for key, value in alert_counter.most_common()],
|
||||
"recent_runs": [sanitize(row) for row in reversed(metrics[-limit:])],
|
||||
"recent_alerts": [sanitize(row) for row in reversed(alerts[-limit:])],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def report_html(report: dict[str, Any], kind: str) -> str:
|
||||
"""Render a self-contained, print-ready evidence dossier."""
|
||||
title = report.get("title") or (
|
||||
f"Run assurance · {report.get('trace_id', 'unknown')}"
|
||||
)
|
||||
verdict = _text(report.get("verdict"), "unknown").replace("_", " ")
|
||||
if kind == "h6":
|
||||
summary = report.get("summary", {})
|
||||
metrics = [
|
||||
("Governed runs", summary.get("runs")),
|
||||
("Failure rate", f"{summary.get('failure_rate_pct', 0)}%"),
|
||||
("P95 latency", f"{summary.get('latency_ms', {}).get('p95', 0)} ms"),
|
||||
("Data quality", report.get("data_quality", {}).get("status")),
|
||||
]
|
||||
findings = report.get("findings", [])
|
||||
else:
|
||||
summary = report.get("summary", {})
|
||||
metrics = [
|
||||
("Evidence gates", f"{summary.get('gates_observed', 0)}/7"),
|
||||
("Tool calls", summary.get("tool_calls")),
|
||||
("Failures", summary.get("failures")),
|
||||
("H6 quality", summary.get("telemetry_quality")),
|
||||
]
|
||||
findings = [
|
||||
{"code": gate.get("title"), "severity": gate.get("status"), "message": gate.get("reason")}
|
||||
for gate in report.get("gates", [])
|
||||
]
|
||||
cards = "".join(
|
||||
f'<div class="metric"><span>{escape(str(label))}</span><strong>{escape(str(value if value is not None else "Unavailable"))}</strong></div>'
|
||||
for label, value in metrics
|
||||
)
|
||||
rows = "".join(
|
||||
'<article class="finding"><div><code>%s</code><em>%s</em></div><p>%s</p></article>' % (
|
||||
escape(str(item.get("code", "EVIDENCE"))),
|
||||
escape(str(item.get("severity", "info"))),
|
||||
escape(str(item.get("message", "No explanation emitted."))),
|
||||
) for item in findings
|
||||
) or '<div class="clear">No threshold breach was detected.</div>'
|
||||
contract = escape(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
return f"""<!doctype html>
|
||||
<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<title>{escape(str(title))}</title>
|
||||
<style>
|
||||
:root{{--ink:#101828;--muted:#667085;--line:#e4e7ec;--paper:#f7f8fa;--navy:#101828;--cyan:#06b6d4;--green:#067647;--amber:#b54708}}
|
||||
*{{box-sizing:border-box}}body{{margin:0;background:var(--paper);color:var(--ink);font-family:Inter,ui-sans-serif,system-ui,-apple-system,sans-serif}}
|
||||
main{{max-width:1060px;margin:auto;padding:44px 24px 72px}}header{{border-radius:28px;background:var(--navy);color:white;padding:38px;box-shadow:0 24px 70px #10182820}}
|
||||
.eyebrow,dt,.metric span{{font-size:11px;font-weight:800;letter-spacing:.14em;text-transform:uppercase}}.eyebrow{{color:#67e8f9}}h1{{max-width:760px;margin:12px 0 14px;font-size:42px;line-height:1.05;letter-spacing:-.04em}}
|
||||
.meta{{display:flex;flex-wrap:wrap;gap:12px;color:#98a2b3;font:12px ui-monospace,monospace}}.badge{{display:inline-block;margin-top:20px;border:1px solid #344054;border-radius:999px;padding:8px 12px;font-size:11px;font-weight:800;text-transform:uppercase}}
|
||||
.grid{{display:grid;grid-template-columns:repeat(4,1fr);gap:14px;margin:18px 0}}.metric,.panel{{border:1px solid var(--line);border-radius:18px;background:white;padding:20px;box-shadow:0 10px 30px #10182808}}
|
||||
.metric span{{display:block;color:var(--muted)}}.metric strong{{display:block;margin-top:16px;font-size:24px;letter-spacing:-.03em}}.panel{{margin-top:18px;padding:26px}}h2{{margin:0 0 6px;font-size:20px}}.sub{{margin:0 0 20px;color:var(--muted);font-size:14px}}
|
||||
.finding{{border-left:4px solid var(--cyan);border-radius:10px;background:#f8fafc;padding:14px 16px;margin-top:10px}}.finding div{{display:flex;justify-content:space-between;gap:16px}}code{{font-weight:800}}em{{color:var(--muted);font-size:11px;text-transform:uppercase}}.finding p{{margin:8px 0 0;color:#344054;font-size:14px;line-height:1.55}}.clear{{border-radius:12px;background:#ecfdf3;color:var(--green);padding:16px}}
|
||||
details{{margin-top:18px}}summary{{cursor:pointer;font-weight:700}}pre{{overflow:auto;max-height:720px;border-radius:14px;background:#101828;color:#d0d5dd;padding:20px;font:11px/1.6 ui-monospace,monospace}}
|
||||
footer{{margin-top:22px;color:var(--muted);font-size:11px}}@media(max-width:760px){{h1{{font-size:32px}}.grid{{grid-template-columns:1fr 1fr}}header{{padding:28px}}}}@media print{{body{{background:white}}main{{max-width:none;padding:0}}header,.panel,.metric{{box-shadow:none}}details{{break-before:page}}}}
|
||||
</style></head><body><main><header><div class="eyebrow">CASAN · Independent assurance dossier</div><h1>{escape(str(title))}</h1>
|
||||
<div class="meta"><span>{escape(str(report.get("report_id", "")))}</span><span>Generated {escape(str(report.get("generated_at", "")))}</span></div><div class="badge">Verdict · {escape(verdict)}</div></header>
|
||||
<section class="grid">{cards}</section><section class="panel"><h2>Decision evidence</h2><p class="sub">Threshold findings and control outcomes from canonical local evidence.</p>{rows}</section>
|
||||
<section class="panel"><h2>Machine-auditable contract</h2><p class="sub">The complete sanitized report payload used to render this dossier.</p><details><summary>Open JSON evidence</summary><pre>{contract}</pre></details></section>
|
||||
<footer>Generated on demand by CASAN Core Local Assurance Viewer · read-only · offline-capable</footer></main></body></html>"""
|
||||
Reference in New Issue
Block a user