thanhnvandClaude Opus 4.8
4bb184b935
Wave 0 + Wave 1 core of the transparent agentic-client integration: a
developer types prompts normally in Claude Code / Codex while every
certified turn still carries a full H1->H7 trace and an H6 record.
- agentic_bridge.py: stdlib-only lifecycle state machine (begin/pre-tool/
post-tool/telemetry/finalize/abort + report/doctor). Single-model
invariant (never calls a model), fail-closed at the side-effect point,
admission TTL + canonical-project/session binding, atomic state under
.specify/state/agentic-sessions/, secret redaction, null-not-zero H6.
- agentic-lifecycle.schema.json: client-agnostic JSON contract.
- adapters/claude-code + adapters/codex: thin hook renderers + config
templates that call the core bridge.
- phase-agentic-bridge-tests.sh: C1-C12 acceptance + threat suite (30/30).
- devkit templates/{claude,codex} + windows/install-agentic.ps1
(install/doctor/uninstall with manifest, path-safe).
- docs/casan Windows + security/bypass guides; plan status -> IMPLEMENTED.
- harden generate-agentops-dashboard.py aggregation against null H6 costs.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
210 lines
12 KiB
Python
Executable File
210 lines
12 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import os
|
||
import pathlib
|
||
from datetime import datetime, timezone
|
||
|
||
def _app_root(start):
|
||
# Plan-01: `.resolve()` follows the compat symlink into packages/casan-harness;
|
||
# runtime state lives at the app's `.specify`, so walk UP for that marker.
|
||
d = pathlib.Path(start).resolve()
|
||
for p in (d, *d.parents):
|
||
if (p / ".specify").is_dir() or (p / "packages" / "casan-harness").is_dir():
|
||
return p
|
||
return d.parents[2]
|
||
|
||
|
||
ROOT = _app_root(__file__)
|
||
METRICS = pathlib.Path(os.environ.get(
|
||
"CASAN_TELEMETRY_METRICS_LOG", os.environ.get(
|
||
"CASAN_DASHBOARD_METRICS", ROOT / ".specify" / "logs" / "cost" / "metrics.jsonl")))
|
||
FALLBACK = ROOT / ".specify" / "logs" / "level5" / "fallback.jsonl"
|
||
TOOL = ROOT / ".specify" / "logs" / "level5" / "tool-registry.jsonl"
|
||
PROVIDER = pathlib.Path(os.environ.get(
|
||
"CASAN_TELEMETRY_PROVIDER_LOG", os.environ.get(
|
||
"CASAN_CP_PROVIDER_USAGE", ROOT / ".specify" / "logs" / "level5" / "provider-usage.jsonl")))
|
||
PROJECT_REGISTRY = ROOT / ".specify" / "level5" / "project-registry.json"
|
||
DASHBOARD = ROOT / "docs" / "output" / "casan" / "central-agentops-dashboard.html"
|
||
LEGACY_DASHBOARD = ROOT / "docs" / "output" / "casan" / "agentops-dashboard.html"
|
||
|
||
def read_jsonl(path: pathlib.Path) -> list[dict]:
|
||
if not path.exists():
|
||
return []
|
||
rows = []
|
||
for line in path.read_text(encoding="utf-8").splitlines():
|
||
if line.strip():
|
||
rows.append(json.loads(line))
|
||
return rows
|
||
|
||
metrics = read_jsonl(METRICS)
|
||
fallback = read_jsonl(FALLBACK)
|
||
tools = read_jsonl(TOOL)
|
||
provider_usage = read_jsonl(PROVIDER)
|
||
project_registry = json.loads(PROJECT_REGISTRY.read_text(encoding="utf-8")) if PROJECT_REGISTRY.exists() else {"projects": []}
|
||
|
||
# Plan-20 §5: agentic (H6-agentic) records deliberately store token/cost as `null`
|
||
# when the client gave no reliable source (never coerced to 0 in the record). Sums
|
||
# below must therefore treat a MISSING/null number as 0 for aggregation without
|
||
# crashing — the null still surfaces as-is in the per-row table.
|
||
def _num(v, cast):
|
||
try:
|
||
return cast(v)
|
||
except (TypeError, ValueError):
|
||
return cast(0)
|
||
|
||
total_cost = sum(_num(row.get("cost_estimate", 0), float) for row in metrics)
|
||
avg_latency = round(sum(_num(row.get("latency_ms", 0), int) for row in metrics) / max(len(metrics), 1), 2)
|
||
failures = sum(1 for row in metrics if row.get("status") == "failed")
|
||
fallback_routes = sum(1 for row in fallback if row.get("route") == "fallback")
|
||
tool_denies = sum(1 for row in tools if row.get("decision") == "denied")
|
||
provider_tokens = sum(_num(row.get("total_tokens", 0), int) for row in provider_usage)
|
||
provider_cost = sum(_num(row.get("cost_usd", 0), float) for row in provider_usage)
|
||
registered_projects = len(project_registry.get("projects", []))
|
||
hallucination_signals = sum(_num(row.get("hallucination_signals", 0), int) for row in metrics)
|
||
|
||
# --- Harness maturity: rubric assessment (công tâm), khớp evidence/scoring-run-report.md ---
|
||
ASSESS_DATE = "2026-07-05"
|
||
HARNESS = [
|
||
("H1", "Context", 84), ("H2", "Tool", 80), ("H3", "Evaluation", 82),
|
||
("H4", "Security", 80), ("H5", "Governance", 80),
|
||
("H6", "AgentOps", 80), ("H7", "Orchestration", 80),
|
||
]
|
||
avg_score = round(sum(s for _, _, s in HARNESS) / len(HARNESS), 1)
|
||
lowest_score = min(s for _, _, s in HARNESS)
|
||
|
||
def _band(s):
|
||
if s >= 81: return ("#16a34a", "Strong")
|
||
if s >= 61: return ("#0f766e", "Good")
|
||
if s >= 31: return ("#d97706", "Partial")
|
||
return ("#dc2626", "GAP")
|
||
|
||
harness_rows = "".join(
|
||
f'<div class="hrow"><span class="hlabel">{hid} · {name}</span>'
|
||
f'<span class="meter"><i style="width:{s}%;background:{_band(s)[0]}"></i></span>'
|
||
f'<span class="hscore" style="color:{_band(s)[0]}">{s}<small>/100 · {_band(s)[1]}</small></span></div>'
|
||
for hid, name, s in HARNESS
|
||
)
|
||
|
||
DASHBOARD.parent.mkdir(parents=True, exist_ok=True)
|
||
DASHBOARD.write_text(
|
||
f"""<!doctype html>
|
||
<html lang="en">
|
||
<head>
|
||
<meta charset="utf-8">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||
<meta name="color-scheme" content="light">
|
||
<title>CASAN Level 4 · AgentOps Dashboard</title>
|
||
<style>
|
||
:root {{ color-scheme: light; }}
|
||
* {{ box-sizing: border-box; }}
|
||
body {{ font-family: -apple-system, "Segoe UI", Arial, sans-serif; margin: 0; padding: 30px 34px 60px; background: #eef1f6; color: #16233a; max-width: 1180px; }}
|
||
h1 {{ font-size: 27px; color: #1f3b6e; margin: 0 0 4px; letter-spacing: -.015em; }}
|
||
.subtitle {{ color: #5a6b80; font-size: 13px; margin: 0 0 16px; }}
|
||
h2 {{ color: #1f3b6e; font-size: 16px; margin: 26px 0 12px; letter-spacing: -.01em; }}
|
||
.badges {{ display: flex; flex-wrap: wrap; gap: 9px; margin: 0 0 8px; }}
|
||
.badge {{ background: #fff; border: 1px solid #d8dee9; border-radius: 999px; padding: 6px 13px; font-size: 12.5px; font-weight: 700; color: #1f3b6e; }}
|
||
.badge.lv {{ background: #eaf6ef; border-color: #bfe0cd; color: #16794f; }}
|
||
.panel {{ background: #fff; border: 1px solid #dbe2ec; border-radius: 13px; padding: 18px 20px; box-shadow: 0 1px 2px rgba(16,35,58,.05); }}
|
||
.grid {{ display: grid; grid-template-columns: repeat(5, 1fr); gap: 12px; }}
|
||
.card {{ background: #fff; border: 1px solid #dbe2ec; border-radius: 11px; padding: 15px 16px; box-shadow: 0 1px 2px rgba(16,35,58,.04); }}
|
||
.card .k {{ color: #5a6b80; font-size: 11.5px; font-weight: 600; letter-spacing: .01em; }}
|
||
.value {{ font-size: 26px; font-weight: 800; color: #0f766e; margin-top: 5px; }}
|
||
.hrow {{ display: grid; grid-template-columns: 165px 1fr 128px; align-items: center; gap: 14px; padding: 6px 0; }}
|
||
.hlabel {{ font-size: 13.5px; font-weight: 600; color: #28405c; }}
|
||
.meter {{ background: #e6ebf2; border-radius: 6px; height: 13px; overflow: hidden; }}
|
||
.meter > i {{ display: block; height: 100%; border-radius: 6px; }}
|
||
.hscore {{ font-size: 15px; font-weight: 800; text-align: right; white-space: nowrap; }}
|
||
.hscore small {{ font-size: 9.5px; color: #8a94a0; font-weight: 600; }}
|
||
.hsum {{ margin-top: 13px; padding-top: 12px; border-top: 1px solid #eef1f6; font-size: 13px; color: #41566f; }}
|
||
.hsum b {{ color: #1f3b6e; }}
|
||
.bandlg {{ display: flex; flex-wrap: wrap; gap: 14px; margin-top: 9px; font-size: 11.5px; color: #6b7888; }}
|
||
.bandlg i {{ width: 11px; height: 11px; border-radius: 3px; display: inline-block; margin-right: 5px; vertical-align: -1px; }}
|
||
.chips {{ display: flex; flex-wrap: wrap; gap: 9px; }}
|
||
.chip {{ display: inline-flex; align-items: center; gap: 8px; font-size: 12.5px; color: #28405c; background: #f5f8fc; border: 1px solid #dbe4ef; border-radius: 999px; padding: 8px 14px; }}
|
||
.chip .d {{ width: 8px; height: 8px; border-radius: 99px; background: #16a34a; flex: none; }}
|
||
.chip b {{ color: #0f766e; }}
|
||
table {{ border-collapse: collapse; width: 100%; margin-top: 4px; background: #fff; }}
|
||
td, th {{ border: 1px solid #e2e8f1; padding: 8px 10px; text-align: left; font-size: 12.5px; color: #16233a; }}
|
||
th {{ background: #eef2f8; color: #1f3b6e; font-weight: 700; }}
|
||
tr:nth-child(even) td {{ background: #f7f9fc; }}
|
||
.ok {{ color: #16a34a; font-weight: 700; }}
|
||
</style>
|
||
</head>
|
||
<body>
|
||
<h1>CASAN Level 4 Central AgentOps Dashboard</h1>
|
||
<p class="subtitle">7-harness security posture · Level-5 controls demonstrated locally · điểm công tâm theo rubric (evidence/scoring-run-report.md) · Generated: {datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')}</p>
|
||
<div class="badges">
|
||
<span class="badge lv">CASAN Level 4 — chứng minh bằng tấn công</span>
|
||
<span class="badge">Average {avg_score}/100</span>
|
||
<span class="badge">Harness thấp nhất {lowest_score}</span>
|
||
<span class="badge">218 core tests · 0 fail</span>
|
||
<span class="badge">Recall model 0.85 > regex 0.00</span>
|
||
</div>
|
||
|
||
<h2>Đánh giá trưởng thành 7 Harness · rubric công tâm ({ASSESS_DATE})</h2>
|
||
<div class="panel">
|
||
{harness_rows}
|
||
<div class="hsum">Average <b>{avg_score}/100</b> · Harness thấp nhất <b>{lowest_score}</b> → <b>CASAN Level 4</b> (chưa lên "Strong/production" — bản production của IdP/WORM-store/HSM/sandbox-isolation còn planned).</div>
|
||
<div class="bandlg">
|
||
<span><i style="background:#16a34a"></i>Strong 81–100 (production)</span>
|
||
<span><i style="background:#0f766e"></i>Good 61–80</span>
|
||
<span><i style="background:#d97706"></i>Partial 31–60</span>
|
||
<span><i style="background:#dc2626"></i>GAP 0–30</span>
|
||
</div>
|
||
</div>
|
||
|
||
<h2>Bảo mật & Governance đã kiểm chứng (test đối kháng thật)</h2>
|
||
<div class="panel"><div class="chips">
|
||
<span class="chip"><span class="d"></span>Kiểm thử đối kháng <b>218 / 0 fail</b></span>
|
||
<span class="chip"><span class="d"></span>H4 recall model <b>0.85</b> > regex 0.00</span>
|
||
<span class="chip"><span class="d"></span>Benign FP <b>0.00%</b> · block <b>100.00%</b></span>
|
||
<span class="chip"><span class="d"></span>Audit hash-chain + ký KMS (rotate/non-exportable)</span>
|
||
<span class="chip"><span class="d"></span>WORM audit ngoài (gap/tamper detected)</span>
|
||
<span class="chip"><span class="d"></span>Approval ký-danh-tính (chống giả/replay/tự-duyệt)</span>
|
||
<span class="chip"><span class="d"></span>Cost-spike 4 chế độ · drift · hallucination scan</span>
|
||
<span class="chip"><span class="d"></span>Alert live: webhook · dead-letter</span>
|
||
<span class="chip"><span class="d"></span>Unicode/base64 normalize · tool-output scan</span>
|
||
<span class="chip"><span class="d"></span>action / supply-chain / data-exfil gate</span>
|
||
</div></div>
|
||
|
||
<h2>Telemetry trực tiếp (live) · pipeline harness</h2>
|
||
<div class="grid">
|
||
<div class="card"><div class="k">Total Runs</div><div class="value">{len(metrics)}</div></div>
|
||
<div class="card"><div class="k">Average Latency</div><div class="value">{avg_latency}<small style="font-size:14px"> ms</small></div></div>
|
||
<div class="card"><div class="k">Estimated Cost</div><div class="value">${total_cost:.6f}</div></div>
|
||
<div class="card"><div class="k">Failures</div><div class="value">{failures}</div></div>
|
||
<div class="card"><div class="k">Fallback Routes</div><div class="value">{fallback_routes}</div></div>
|
||
</div>
|
||
<div class="grid" style="margin-top:12px">
|
||
<div class="card"><div class="k">Provider Runs</div><div class="value">{len(provider_usage)}</div></div>
|
||
<div class="card"><div class="k">Provider Tokens (thật)</div><div class="value">{provider_tokens}</div></div>
|
||
<div class="card"><div class="k">Provider Cost</div><div class="value">${provider_cost:.5f}</div></div>
|
||
<div class="card"><div class="k">Tool Denials</div><div class="value">{tool_denies}</div></div>
|
||
<div class="card"><div class="k">Hallucination Signals</div><div class="value">{hallucination_signals}</div></div>
|
||
</div>
|
||
|
||
<h2>Governance Signals</h2>
|
||
<table>
|
||
<tr><th>Signal</th><th>Value</th></tr>
|
||
<tr><td>Tool registry denials</td><td>{tool_denies}</td></tr>
|
||
<tr><td>Fallback records</td><td>{len(fallback)}</td></tr>
|
||
<tr><td>Tool registry records</td><td>{len(tools)}</td></tr>
|
||
<tr><td>Provider telemetry records</td><td>{len(provider_usage)}</td></tr>
|
||
<tr><td>Registered harness projects</td><td>{registered_projects}</td></tr>
|
||
</table>
|
||
|
||
<h2>Recent AgentOps Metrics (live)</h2>
|
||
<table>
|
||
<tr><th>Trace</th><th>Agent</th><th>Step</th><th>Status</th><th>Latency</th><th>Tokens</th><th>Cost</th></tr>
|
||
{''.join(f"<tr><td>{str(m.get('trace_id'))[:8]}…</td><td>{m.get('agent')}</td><td>{m.get('step')}</td><td class='{'ok' if m.get('status')=='success' else ''}'>{m.get('status')}</td><td>{m.get('latency_ms')}ms</td><td>{m.get('total_tokens')}</td><td>${m.get('cost_estimate')}</td></tr>" for m in metrics[-10:])}
|
||
</table>
|
||
</body>
|
||
</html>
|
||
""",
|
||
encoding="utf-8",
|
||
)
|
||
LEGACY_DASHBOARD.write_text(DASHBOARD.read_text(encoding="utf-8"), encoding="utf-8")
|
||
print(f"DASHBOARD_GENERATED {DASHBOARD}")
|