97 lines
4.5 KiB
Python
Executable File
97 lines
4.5 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import pathlib
|
|
from datetime import datetime, timezone
|
|
|
|
ROOT = pathlib.Path(__file__).resolve().parents[2]
|
|
METRICS = ROOT / ".specify" / "logs" / "cost" / "metrics.jsonl"
|
|
FALLBACK = ROOT / ".specify" / "logs" / "level5" / "fallback.jsonl"
|
|
TOOL = ROOT / ".specify" / "logs" / "level5" / "tool-registry.jsonl"
|
|
PROVIDER = ROOT / ".specify" / "logs" / "level5" / "provider-usage.jsonl"
|
|
PROJECT_REGISTRY = ROOT / ".specify" / "level5" / "project-registry.json"
|
|
DASHBOARD = ROOT / "docs" / "output" / "casan" / "central-agentops-dashboard.html"
|
|
LEGACY_DASHBOARD = ROOT / "docs" / "output" / "casan" / "agentops-dashboard.html"
|
|
|
|
def read_jsonl(path: pathlib.Path) -> list[dict]:
|
|
if not path.exists():
|
|
return []
|
|
rows = []
|
|
for line in path.read_text(encoding="utf-8").splitlines():
|
|
if line.strip():
|
|
rows.append(json.loads(line))
|
|
return rows
|
|
|
|
metrics = read_jsonl(METRICS)
|
|
fallback = read_jsonl(FALLBACK)
|
|
tools = read_jsonl(TOOL)
|
|
provider_usage = read_jsonl(PROVIDER)
|
|
project_registry = json.loads(PROJECT_REGISTRY.read_text(encoding="utf-8")) if PROJECT_REGISTRY.exists() else {"projects": []}
|
|
|
|
total_cost = sum(float(row.get("cost_estimate", 0)) for row in metrics)
|
|
avg_latency = round(sum(int(row.get("latency_ms", 0)) for row in metrics) / max(len(metrics), 1), 2)
|
|
failures = sum(1 for row in metrics if row.get("status") == "failed")
|
|
fallback_routes = sum(1 for row in fallback if row.get("route") == "fallback")
|
|
tool_denies = sum(1 for row in tools if row.get("decision") == "denied")
|
|
provider_tokens = sum(int(row.get("total_tokens", 0)) for row in provider_usage)
|
|
provider_cost = sum(float(row.get("cost_usd", 0)) for row in provider_usage)
|
|
registered_projects = len(project_registry.get("projects", []))
|
|
|
|
DASHBOARD.parent.mkdir(parents=True, exist_ok=True)
|
|
DASHBOARD.write_text(
|
|
f"""<!doctype html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="utf-8">
|
|
<title>CASAN Central AgentOps Dashboard</title>
|
|
<style>
|
|
body {{ font-family: Arial, sans-serif; margin: 32px; color: #172033; }}
|
|
.grid {{ display: grid; grid-template-columns: repeat(5, 1fr); gap: 12px; }}
|
|
.card {{ border: 1px solid #d8dee9; border-radius: 8px; padding: 16px; }}
|
|
.value {{ font-size: 28px; font-weight: 700; color: #0f766e; }}
|
|
table {{ border-collapse: collapse; width: 100%; margin-top: 20px; }}
|
|
td, th {{ border: 1px solid #d8dee9; padding: 8px; text-align: left; }}
|
|
th {{ background: #f1f5f9; }}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<h1>CASAN Level 5 Central AgentOps Dashboard</h1>
|
|
<p>Generated: {datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')}</p>
|
|
<div class="grid">
|
|
<div class="card"><div>Total Runs</div><div class="value">{len(metrics)}</div></div>
|
|
<div class="card"><div>Average Latency</div><div class="value">{avg_latency}ms</div></div>
|
|
<div class="card"><div>Estimated Cost</div><div class="value">${total_cost:.6f}</div></div>
|
|
<div class="card"><div>Failures</div><div class="value">{failures}</div></div>
|
|
<div class="card"><div>Fallback Routes</div><div class="value">{fallback_routes}</div></div>
|
|
</div>
|
|
<h2>Provider Usage Telemetry</h2>
|
|
<div class="grid">
|
|
<div class="card"><div>Provider Runs</div><div class="value">{len(provider_usage)}</div></div>
|
|
<div class="card"><div>Provider Tokens</div><div class="value">{provider_tokens}</div></div>
|
|
<div class="card"><div>Provider Cost</div><div class="value">${provider_cost:.5f}</div></div>
|
|
<div class="card"><div>Registered Projects</div><div class="value">{registered_projects}</div></div>
|
|
<div class="card"><div>Tool Denials</div><div class="value">{tool_denies}</div></div>
|
|
</div>
|
|
<h2>Governance Signals</h2>
|
|
<table>
|
|
<tr><th>Signal</th><th>Value</th></tr>
|
|
<tr><td>Tool registry denials</td><td>{tool_denies}</td></tr>
|
|
<tr><td>Fallback records</td><td>{len(fallback)}</td></tr>
|
|
<tr><td>Tool registry records</td><td>{len(tools)}</td></tr>
|
|
<tr><td>Provider telemetry records</td><td>{len(provider_usage)}</td></tr>
|
|
<tr><td>Registered harness projects</td><td>{registered_projects}</td></tr>
|
|
</table>
|
|
<h2>Recent AgentOps Metrics</h2>
|
|
<table>
|
|
<tr><th>Trace</th><th>Agent</th><th>Step</th><th>Status</th><th>Latency</th><th>Tokens</th><th>Cost</th></tr>
|
|
{''.join(f"<tr><td>{m.get('trace_id')}</td><td>{m.get('agent')}</td><td>{m.get('step')}</td><td>{m.get('status')}</td><td>{m.get('latency_ms')}</td><td>{m.get('total_tokens')}</td><td>{m.get('cost_estimate')}</td></tr>" for m in metrics[-10:])}
|
|
</table>
|
|
</body>
|
|
</html>
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
LEGACY_DASHBOARD.write_text(DASHBOARD.read_text(encoding="utf-8"), encoding="utf-8")
|
|
print(f"DASHBOARD_GENERATED {DASHBOARD}")
|