#!/usr/bin/env python3 from __future__ import annotations import json import pathlib from datetime import datetime, timezone ROOT = pathlib.Path(__file__).resolve().parents[2] METRICS = ROOT / ".specify" / "logs" / "cost" / "metrics.jsonl" FALLBACK = ROOT / ".specify" / "logs" / "level5" / "fallback.jsonl" TOOL = ROOT / ".specify" / "logs" / "level5" / "tool-registry.jsonl" PROVIDER = ROOT / ".specify" / "logs" / "level5" / "provider-usage.jsonl" PROJECT_REGISTRY = ROOT / ".specify" / "level5" / "project-registry.json" DASHBOARD = ROOT / "docs" / "output" / "casan" / "central-agentops-dashboard.html" LEGACY_DASHBOARD = ROOT / "docs" / "output" / "casan" / "agentops-dashboard.html" def read_jsonl(path: pathlib.Path) -> list[dict]: if not path.exists(): return [] rows = [] for line in path.read_text(encoding="utf-8").splitlines(): if line.strip(): rows.append(json.loads(line)) return rows metrics = read_jsonl(METRICS) fallback = read_jsonl(FALLBACK) tools = read_jsonl(TOOL) provider_usage = read_jsonl(PROVIDER) project_registry = json.loads(PROJECT_REGISTRY.read_text(encoding="utf-8")) if PROJECT_REGISTRY.exists() else {"projects": []} total_cost = sum(float(row.get("cost_estimate", 0)) for row in metrics) avg_latency = round(sum(int(row.get("latency_ms", 0)) for row in metrics) / max(len(metrics), 1), 2) failures = sum(1 for row in metrics if row.get("status") == "failed") fallback_routes = sum(1 for row in fallback if row.get("route") == "fallback") tool_denies = sum(1 for row in tools if row.get("decision") == "denied") provider_tokens = sum(int(row.get("total_tokens", 0)) for row in provider_usage) provider_cost = sum(float(row.get("cost_usd", 0)) for row in provider_usage) registered_projects = len(project_registry.get("projects", [])) hallucination_signals = sum(int(row.get("hallucination_signals", 0)) for row in metrics) # --- Harness maturity: rubric assessment (công tâm), khớp evidence/scoring-run-report.md --- ASSESS_DATE = "2026-07-05" HARNESS = [ ("H1", "Context", 84), ("H2", "Tool", 80), ("H3", "Evaluation", 82), ("H4", "Security", 80), ("H5", "Governance", 80), ("H6", "AgentOps", 80), ("H7", "Orchestration", 80), ] avg_score = round(sum(s for _, _, s in HARNESS) / len(HARNESS), 1) lowest_score = min(s for _, _, s in HARNESS) def _band(s): if s >= 81: return ("#16a34a", "Strong") if s >= 61: return ("#0f766e", "Good") if s >= 31: return ("#d97706", "Partial") return ("#dc2626", "GAP") harness_rows = "".join( f'
{hid} · {name}' f'' f'{s}/100 · {_band(s)[1]}
' for hid, name, s in HARNESS ) DASHBOARD.parent.mkdir(parents=True, exist_ok=True) DASHBOARD.write_text( f""" CASAN Level 4 · AgentOps Dashboard

CASAN Level 4 Central AgentOps Dashboard

7-harness security posture · Level-5 controls demonstrated locally · điểm công tâm theo rubric (evidence/scoring-run-report.md) · Generated: {datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')}

CASAN Level 4 — chứng minh bằng tấn công Average {avg_score}/100 Harness thấp nhất {lowest_score} 218 core tests · 0 fail Recall model 0.85 > regex 0.00

Đánh giá trưởng thành 7 Harness · rubric công tâm ({ASSESS_DATE})

{harness_rows}
Average {avg_score}/100 · Harness thấp nhất {lowest_score} → CASAN Level 4 (chưa lên "Strong/production" — bản production của IdP/WORM-store/HSM/sandbox-isolation còn planned).
Strong 81–100 (production) Good 61–80 Partial 31–60 GAP 0–30

Bảo mật & Governance đã kiểm chứng (test đối kháng thật)

Kiểm thử đối kháng 218 / 0 fail H4 recall model 0.85 > regex 0.00 Benign FP 0.00% · block 100.00% Audit hash-chain + ký KMS (rotate/non-exportable) WORM audit ngoài (gap/tamper detected) Approval ký-danh-tính (chống giả/replay/tự-duyệt) Cost-spike 4 chế độ · drift · hallucination scan Alert live: webhook · dead-letter Unicode/base64 normalize · tool-output scan action / supply-chain / data-exfil gate

Telemetry trực tiếp (live) · pipeline harness

Total Runs
{len(metrics)}
Average Latency
{avg_latency} ms
Estimated Cost
${total_cost:.6f}
Failures
{failures}
Fallback Routes
{fallback_routes}
Provider Runs
{len(provider_usage)}
Provider Tokens (thật)
{provider_tokens}
Provider Cost
${provider_cost:.5f}
Tool Denials
{tool_denies}
Hallucination Signals
{hallucination_signals}

Governance Signals

SignalValue
Tool registry denials{tool_denies}
Fallback records{len(fallback)}
Tool registry records{len(tools)}
Provider telemetry records{len(provider_usage)}
Registered harness projects{registered_projects}

Recent AgentOps Metrics (live)

{''.join(f"" for m in metrics[-10:])}
TraceAgentStepStatusLatencyTokensCost
{str(m.get('trace_id'))[:8]}…{m.get('agent')}{m.get('step')}{m.get('status')}{m.get('latency_ms')}ms{m.get('total_tokens')}${m.get('cost_estimate')}
""", encoding="utf-8", ) LEGACY_DASHBOARD.write_text(DASHBOARD.read_text(encoding="utf-8"), encoding="utf-8") print(f"DASHBOARD_GENERATED {DASHBOARD}")