update doc/log
This commit is contained in:
thanhnv
@@ -37,6 +37,30 @@ tool_denies = sum(1 for row in tools if row.get("decision") == "denied")
|
||||
provider_tokens = sum(int(row.get("total_tokens", 0)) for row in provider_usage)
|
||||
provider_cost = sum(float(row.get("cost_usd", 0)) for row in provider_usage)
|
||||
registered_projects = len(project_registry.get("projects", []))
|
||||
hallucination_signals = sum(int(row.get("hallucination_signals", 0)) for row in metrics)
|
||||
|
||||
# --- Harness maturity: rubric assessment (công tâm), khớp evidence/scoring-run-report.md ---
|
||||
ASSESS_DATE = "2026-07-05"
|
||||
HARNESS = [
|
||||
("H1", "Context", 84), ("H2", "Tool", 80), ("H3", "Evaluation", 82),
|
||||
("H4", "Security", 80), ("H5", "Governance", 80),
|
||||
("H6", "AgentOps", 80), ("H7", "Orchestration", 80),
|
||||
]
|
||||
avg_score = round(sum(s for _, _, s in HARNESS) / len(HARNESS), 1)
|
||||
lowest_score = min(s for _, _, s in HARNESS)
|
||||
|
||||
def _band(s):
|
||||
if s >= 81: return ("#16a34a", "Strong")
|
||||
if s >= 61: return ("#0f766e", "Good")
|
||||
if s >= 31: return ("#d97706", "Partial")
|
||||
return ("#dc2626", "GAP")
|
||||
|
||||
harness_rows = "".join(
|
||||
f'<div class="hrow"><span class="hlabel">{hid} · {name}</span>'
|
||||
f'<span class="meter"><i style="width:{s}%;background:{_band(s)[0]}"></i></span>'
|
||||
f'<span class="hscore" style="color:{_band(s)[0]}">{s}<small>/100 · {_band(s)[1]}</small></span></div>'
|
||||
for hid, name, s in HARNESS
|
||||
)
|
||||
|
||||
DASHBOARD.parent.mkdir(parents=True, exist_ok=True)
|
||||
DASHBOARD.write_text(
|
||||
@@ -44,35 +68,98 @@ DASHBOARD.write_text(
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>CASAN Central AgentOps Dashboard</title>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<meta name="color-scheme" content="light">
|
||||
<title>CASAN Level 4 · AgentOps Dashboard</title>
|
||||
<style>
|
||||
body {{ font-family: Arial, sans-serif; margin: 32px; color: #172033; }}
|
||||
:root {{ color-scheme: light; }}
|
||||
* {{ box-sizing: border-box; }}
|
||||
body {{ font-family: -apple-system, "Segoe UI", Arial, sans-serif; margin: 0; padding: 30px 34px 60px; background: #eef1f6; color: #16233a; max-width: 1180px; }}
|
||||
h1 {{ font-size: 27px; color: #1f3b6e; margin: 0 0 4px; letter-spacing: -.015em; }}
|
||||
.subtitle {{ color: #5a6b80; font-size: 13px; margin: 0 0 16px; }}
|
||||
h2 {{ color: #1f3b6e; font-size: 16px; margin: 26px 0 12px; letter-spacing: -.01em; }}
|
||||
.badges {{ display: flex; flex-wrap: wrap; gap: 9px; margin: 0 0 8px; }}
|
||||
.badge {{ background: #fff; border: 1px solid #d8dee9; border-radius: 999px; padding: 6px 13px; font-size: 12.5px; font-weight: 700; color: #1f3b6e; }}
|
||||
.badge.lv {{ background: #eaf6ef; border-color: #bfe0cd; color: #16794f; }}
|
||||
.panel {{ background: #fff; border: 1px solid #dbe2ec; border-radius: 13px; padding: 18px 20px; box-shadow: 0 1px 2px rgba(16,35,58,.05); }}
|
||||
.grid {{ display: grid; grid-template-columns: repeat(5, 1fr); gap: 12px; }}
|
||||
.card {{ border: 1px solid #d8dee9; border-radius: 8px; padding: 16px; }}
|
||||
.value {{ font-size: 28px; font-weight: 700; color: #0f766e; }}
|
||||
table {{ border-collapse: collapse; width: 100%; margin-top: 20px; }}
|
||||
td, th {{ border: 1px solid #d8dee9; padding: 8px; text-align: left; }}
|
||||
th {{ background: #f1f5f9; }}
|
||||
.card {{ background: #fff; border: 1px solid #dbe2ec; border-radius: 11px; padding: 15px 16px; box-shadow: 0 1px 2px rgba(16,35,58,.04); }}
|
||||
.card .k {{ color: #5a6b80; font-size: 11.5px; font-weight: 600; letter-spacing: .01em; }}
|
||||
.value {{ font-size: 26px; font-weight: 800; color: #0f766e; margin-top: 5px; }}
|
||||
.hrow {{ display: grid; grid-template-columns: 165px 1fr 128px; align-items: center; gap: 14px; padding: 6px 0; }}
|
||||
.hlabel {{ font-size: 13.5px; font-weight: 600; color: #28405c; }}
|
||||
.meter {{ background: #e6ebf2; border-radius: 6px; height: 13px; overflow: hidden; }}
|
||||
.meter > i {{ display: block; height: 100%; border-radius: 6px; }}
|
||||
.hscore {{ font-size: 15px; font-weight: 800; text-align: right; white-space: nowrap; }}
|
||||
.hscore small {{ font-size: 9.5px; color: #8a94a0; font-weight: 600; }}
|
||||
.hsum {{ margin-top: 13px; padding-top: 12px; border-top: 1px solid #eef1f6; font-size: 13px; color: #41566f; }}
|
||||
.hsum b {{ color: #1f3b6e; }}
|
||||
.bandlg {{ display: flex; flex-wrap: wrap; gap: 14px; margin-top: 9px; font-size: 11.5px; color: #6b7888; }}
|
||||
.bandlg i {{ width: 11px; height: 11px; border-radius: 3px; display: inline-block; margin-right: 5px; vertical-align: -1px; }}
|
||||
.chips {{ display: flex; flex-wrap: wrap; gap: 9px; }}
|
||||
.chip {{ display: inline-flex; align-items: center; gap: 8px; font-size: 12.5px; color: #28405c; background: #f5f8fc; border: 1px solid #dbe4ef; border-radius: 999px; padding: 8px 14px; }}
|
||||
.chip .d {{ width: 8px; height: 8px; border-radius: 99px; background: #16a34a; flex: none; }}
|
||||
.chip b {{ color: #0f766e; }}
|
||||
table {{ border-collapse: collapse; width: 100%; margin-top: 4px; background: #fff; }}
|
||||
td, th {{ border: 1px solid #e2e8f1; padding: 8px 10px; text-align: left; font-size: 12.5px; color: #16233a; }}
|
||||
th {{ background: #eef2f8; color: #1f3b6e; font-weight: 700; }}
|
||||
tr:nth-child(even) td {{ background: #f7f9fc; }}
|
||||
.ok {{ color: #16a34a; font-weight: 700; }}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>CASAN Level 5 Central AgentOps Dashboard</h1>
|
||||
<p>Generated: {datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')}</p>
|
||||
<div class="grid">
|
||||
<div class="card"><div>Total Runs</div><div class="value">{len(metrics)}</div></div>
|
||||
<div class="card"><div>Average Latency</div><div class="value">{avg_latency}ms</div></div>
|
||||
<div class="card"><div>Estimated Cost</div><div class="value">${total_cost:.6f}</div></div>
|
||||
<div class="card"><div>Failures</div><div class="value">{failures}</div></div>
|
||||
<div class="card"><div>Fallback Routes</div><div class="value">{fallback_routes}</div></div>
|
||||
<h1>CASAN Level 4 Central AgentOps Dashboard</h1>
|
||||
<p class="subtitle">7-harness security posture · Level-5 controls demonstrated locally · điểm công tâm theo rubric (evidence/scoring-run-report.md) · Generated: {datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')}</p>
|
||||
<div class="badges">
|
||||
<span class="badge lv">CASAN Level 4 — chứng minh bằng tấn công</span>
|
||||
<span class="badge">Average {avg_score}/100</span>
|
||||
<span class="badge">Harness thấp nhất {lowest_score}</span>
|
||||
<span class="badge">175 test · 0 fail</span>
|
||||
<span class="badge">Recall model 0.85 > regex 0.00</span>
|
||||
</div>
|
||||
<h2>Provider Usage Telemetry</h2>
|
||||
<div class="grid">
|
||||
<div class="card"><div>Provider Runs</div><div class="value">{len(provider_usage)}</div></div>
|
||||
<div class="card"><div>Provider Tokens</div><div class="value">{provider_tokens}</div></div>
|
||||
<div class="card"><div>Provider Cost</div><div class="value">${provider_cost:.5f}</div></div>
|
||||
<div class="card"><div>Registered Projects</div><div class="value">{registered_projects}</div></div>
|
||||
<div class="card"><div>Tool Denials</div><div class="value">{tool_denies}</div></div>
|
||||
|
||||
<h2>Đánh giá trưởng thành 7 Harness · rubric công tâm ({ASSESS_DATE})</h2>
|
||||
<div class="panel">
|
||||
{harness_rows}
|
||||
<div class="hsum">Average <b>{avg_score}/100</b> · Harness thấp nhất <b>{lowest_score}</b> → <b>CASAN Level 4</b> (chưa lên "Strong/production" — bản production của IdP/WORM-store/HSM/sandbox-isolation còn planned).</div>
|
||||
<div class="bandlg">
|
||||
<span><i style="background:#16a34a"></i>Strong 81–100 (production)</span>
|
||||
<span><i style="background:#0f766e"></i>Good 61–80</span>
|
||||
<span><i style="background:#d97706"></i>Partial 31–60</span>
|
||||
<span><i style="background:#dc2626"></i>GAP 0–30</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<h2>Bảo mật & Governance đã kiểm chứng (test đối kháng thật)</h2>
|
||||
<div class="panel"><div class="chips">
|
||||
<span class="chip"><span class="d"></span>Kiểm thử đối kháng <b>175 / 0 fail</b></span>
|
||||
<span class="chip"><span class="d"></span>H4 recall model <b>0.85</b> > regex 0.00</span>
|
||||
<span class="chip"><span class="d"></span>Benign FP <b>0.00%</b> · block <b>100.00%</b></span>
|
||||
<span class="chip"><span class="d"></span>Audit hash-chain + ký KMS (rotate/non-exportable)</span>
|
||||
<span class="chip"><span class="d"></span>WORM audit ngoài (gap/tamper detected)</span>
|
||||
<span class="chip"><span class="d"></span>Approval ký-danh-tính (chống giả/replay/tự-duyệt)</span>
|
||||
<span class="chip"><span class="d"></span>Cost-spike 4 chế độ · drift · hallucination scan</span>
|
||||
<span class="chip"><span class="d"></span>Alert live: webhook · dead-letter</span>
|
||||
<span class="chip"><span class="d"></span>Unicode/base64 normalize · tool-output scan</span>
|
||||
<span class="chip"><span class="d"></span>action / supply-chain / data-exfil gate</span>
|
||||
</div></div>
|
||||
|
||||
<h2>Telemetry trực tiếp (live) · pipeline harness</h2>
|
||||
<div class="grid">
|
||||
<div class="card"><div class="k">Total Runs</div><div class="value">{len(metrics)}</div></div>
|
||||
<div class="card"><div class="k">Average Latency</div><div class="value">{avg_latency}<small style="font-size:14px"> ms</small></div></div>
|
||||
<div class="card"><div class="k">Estimated Cost</div><div class="value">${total_cost:.6f}</div></div>
|
||||
<div class="card"><div class="k">Failures</div><div class="value">{failures}</div></div>
|
||||
<div class="card"><div class="k">Fallback Routes</div><div class="value">{fallback_routes}</div></div>
|
||||
</div>
|
||||
<div class="grid" style="margin-top:12px">
|
||||
<div class="card"><div class="k">Provider Runs</div><div class="value">{len(provider_usage)}</div></div>
|
||||
<div class="card"><div class="k">Provider Tokens (thật)</div><div class="value">{provider_tokens}</div></div>
|
||||
<div class="card"><div class="k">Provider Cost</div><div class="value">${provider_cost:.5f}</div></div>
|
||||
<div class="card"><div class="k">Tool Denials</div><div class="value">{tool_denies}</div></div>
|
||||
<div class="card"><div class="k">Hallucination Signals</div><div class="value">{hallucination_signals}</div></div>
|
||||
</div>
|
||||
|
||||
<h2>Governance Signals</h2>
|
||||
<table>
|
||||
<tr><th>Signal</th><th>Value</th></tr>
|
||||
@@ -82,10 +169,11 @@ th {{ background: #f1f5f9; }}
|
||||
<tr><td>Provider telemetry records</td><td>{len(provider_usage)}</td></tr>
|
||||
<tr><td>Registered harness projects</td><td>{registered_projects}</td></tr>
|
||||
</table>
|
||||
<h2>Recent AgentOps Metrics</h2>
|
||||
|
||||
<h2>Recent AgentOps Metrics (live)</h2>
|
||||
<table>
|
||||
<tr><th>Trace</th><th>Agent</th><th>Step</th><th>Status</th><th>Latency</th><th>Tokens</th><th>Cost</th></tr>
|
||||
{''.join(f"<tr><td>{m.get('trace_id')}</td><td>{m.get('agent')}</td><td>{m.get('step')}</td><td>{m.get('status')}</td><td>{m.get('latency_ms')}</td><td>{m.get('total_tokens')}</td><td>{m.get('cost_estimate')}</td></tr>" for m in metrics[-10:])}
|
||||
{''.join(f"<tr><td>{str(m.get('trace_id'))[:8]}…</td><td>{m.get('agent')}</td><td>{m.get('step')}</td><td class='{'ok' if m.get('status')=='success' else ''}'>{m.get('status')}</td><td>{m.get('latency_ms')}ms</td><td>{m.get('total_tokens')}</td><td>${m.get('cost_estimate')}</td></tr>" for m in metrics[-10:])}
|
||||
</table>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
Reference in New Issue
Block a user