feat: add production assurance dashboard flow
This commit is contained in:
thanhnv
@@ -139,6 +139,12 @@ def trace_dir():
|
||||
return d
|
||||
|
||||
|
||||
def trace_event_dir():
|
||||
d = os.path.join(state_root(), "logs", "trace-events")
|
||||
os.makedirs(d, exist_ok=True)
|
||||
return d
|
||||
|
||||
|
||||
def metrics_log():
|
||||
override = os.environ.get("CASAN_TELEMETRY_METRICS_LOG") or os.environ.get(
|
||||
"CASAN_METRICS_LOG"
|
||||
@@ -151,6 +157,178 @@ def metrics_log():
|
||||
return path
|
||||
|
||||
|
||||
def project_config():
|
||||
path = os.path.join(app_root(), ".casan", "config.json")
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as fh:
|
||||
value = json.load(fh)
|
||||
return value if isinstance(value, dict) else {}
|
||||
except (OSError, ValueError):
|
||||
return {}
|
||||
|
||||
|
||||
def dashboard_url(trace_id):
|
||||
"""Return a safe Control Plane deep link, when the project was enrolled.
|
||||
|
||||
Core never assumes that a dashboard is present and never performs network
|
||||
I/O in the prompt hook. The receipt is therefore useful offline while an
|
||||
enrolled project gets a clickable deep link without delaying finalize.
|
||||
"""
|
||||
configured = os.environ.get("CASAN_DASHBOARD_URL")
|
||||
if not configured:
|
||||
control_plane = project_config().get("control_plane")
|
||||
if isinstance(control_plane, dict):
|
||||
configured = control_plane.get("dashboard_url")
|
||||
base = str(configured or "").strip().rstrip("/")
|
||||
if not re.match(r"^https?://[a-zA-Z0-9]", base):
|
||||
return None
|
||||
return "%s/runs?trace=%s" % (base, str(trace_id or ""))
|
||||
|
||||
|
||||
def latest_run_path():
|
||||
return os.path.join(state_root(), "state", "latest-run.json")
|
||||
|
||||
|
||||
def delivery_spool_dir():
|
||||
path = os.path.join(state_root(), "spool", "control-plane", "pending")
|
||||
os.makedirs(path, exist_ok=True)
|
||||
return path
|
||||
|
||||
|
||||
def spool_run_envelope(rec, receipt, metric, trace_path):
|
||||
event_path = os.path.join(
|
||||
trace_event_dir(), "%s.jsonl" % rec.get("trace_id"))
|
||||
events = []
|
||||
try:
|
||||
with open(event_path, "r", encoding="utf-8") as fh:
|
||||
events = [
|
||||
json.loads(line)
|
||||
for line in fh
|
||||
if line.strip()
|
||||
]
|
||||
except (OSError, ValueError):
|
||||
events = []
|
||||
try:
|
||||
with open(trace_path, "r", encoding="utf-8") as fh:
|
||||
trace = json.load(fh)
|
||||
except (OSError, ValueError):
|
||||
trace = {}
|
||||
envelope = {
|
||||
"schema_version": 1,
|
||||
"sent_at": now_iso(),
|
||||
"project_id": rec.get("project_id"),
|
||||
"trace_id": rec.get("trace_id"),
|
||||
"receipt": receipt,
|
||||
"metric": metric,
|
||||
"trace": trace,
|
||||
"events": events,
|
||||
}
|
||||
path = os.path.join(
|
||||
delivery_spool_dir(), "%s.json" % rec.get("trace_id"))
|
||||
atomic_write_json(path, envelope)
|
||||
return path
|
||||
|
||||
|
||||
def spawn_delivery(spool_path):
|
||||
control_plane = project_config().get("control_plane")
|
||||
if not isinstance(control_plane, dict):
|
||||
return False
|
||||
ingest_url = str(control_plane.get("ingest_url") or "").strip()
|
||||
token_env = str(
|
||||
control_plane.get("token_env")
|
||||
or "CASAN_CONTROL_PLANE_TOKEN"
|
||||
).strip()
|
||||
if not re.match(r"^https?://[a-zA-Z0-9]", ingest_url):
|
||||
return False
|
||||
if not token_env or not os.environ.get(token_env):
|
||||
return False
|
||||
exporter = os.path.join(_self_dir(), "telemetry_exporter.py")
|
||||
if not os.path.isfile(exporter):
|
||||
return False
|
||||
try:
|
||||
subprocess.Popen(
|
||||
[sys.executable, exporter, "--spool", spool_path,
|
||||
"--url", ingest_url, "--token-env", token_env],
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
close_fds=True,
|
||||
start_new_session=True,
|
||||
)
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def write_run_receipt(rec, decision, quality, warnings, report_url):
|
||||
"""Persist a non-sensitive pointer for CLI/dashboard discovery.
|
||||
|
||||
This is a materialized receipt, not the evidence source of truth. Trace and
|
||||
H6 records remain canonical; regenerating/deleting this pointer cannot
|
||||
change certification.
|
||||
"""
|
||||
payload = {
|
||||
"schema_version": 1,
|
||||
"trace_id": rec.get("trace_id"),
|
||||
"project_id": rec.get("project_id"),
|
||||
"decision": decision,
|
||||
"certified": decision == "certified",
|
||||
"certification_strength": rec.get("certification_strength"),
|
||||
"telemetry_quality": quality,
|
||||
"telemetry_warnings": warnings,
|
||||
"duration_ms": (
|
||||
epoch_ms() - rec.get("started_ms")
|
||||
if rec.get("started_ms") else None
|
||||
),
|
||||
"tool_calls": rec.get("tool_calls", 0),
|
||||
"failures": rec.get("failures", 0),
|
||||
"finalized_at": rec.get("finalized_at") or now_iso(),
|
||||
"report_url": report_url,
|
||||
"trace_path": os.path.relpath(
|
||||
os.path.join(trace_dir(), "agentic-%s.json" % rec.get("trace_id")),
|
||||
app_root(),
|
||||
),
|
||||
"delivery": "local_spool",
|
||||
}
|
||||
atomic_write_json(latest_run_path(), payload)
|
||||
return payload
|
||||
|
||||
|
||||
def format_receipt(response):
|
||||
receipt = response.get("receipt") or {}
|
||||
certified = bool(receipt.get("certified"))
|
||||
verdict = "CERTIFIED" if certified else "NON-CERTIFIED"
|
||||
mark = "✓" if certified else "⚠"
|
||||
duration = receipt.get("duration_ms")
|
||||
duration_text = (
|
||||
"%.1fs" % (float(duration) / 1000.0)
|
||||
if isinstance(duration, (int, float)) else "unknown"
|
||||
)
|
||||
lines = [
|
||||
"CASAN %s %s" % (mark, verdict),
|
||||
"Trace: %s · H1→H7 %s" % (
|
||||
receipt.get("trace_id") or "unknown",
|
||||
"verified" if certified else "review required",
|
||||
),
|
||||
"Duration: %s · Tools: %s · Failures: %s" % (
|
||||
duration_text,
|
||||
receipt.get("tool_calls", 0),
|
||||
receipt.get("failures", 0),
|
||||
),
|
||||
"H6 telemetry: %s" % str(
|
||||
receipt.get("telemetry_quality") or "unknown"
|
||||
).upper(),
|
||||
]
|
||||
if receipt.get("report_url"):
|
||||
lines.append("View report → %s" % receipt["report_url"])
|
||||
else:
|
||||
lines.append(
|
||||
"View report → casan view %s" %
|
||||
(receipt.get("trace_id") or "")
|
||||
)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Small helpers
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
@@ -462,14 +640,76 @@ def h2_registry_gate(action, idempotency_key):
|
||||
# Evidence + telemetry writers
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def _gate_status(decision):
|
||||
normalized = str(decision or "").lower()
|
||||
if normalized in ("allow", "allowed", "opened", "pass", "passed", "success",
|
||||
"complete", "completed", "certified"):
|
||||
return "pass"
|
||||
if normalized in ("degraded", "partial", "insufficient", "warning"):
|
||||
return "warning"
|
||||
if normalized in ("block", "blocked", "deny", "denied", "flag",
|
||||
"non_certified", "failed", "error"):
|
||||
return "blocked"
|
||||
return "running"
|
||||
|
||||
|
||||
def _gate_ids(harness):
|
||||
mapping = {
|
||||
"H1": "H1-context",
|
||||
"H2": "H2-tool",
|
||||
"H3": "H3-eval",
|
||||
"H4": "H4-security",
|
||||
"H5": "H5-governance",
|
||||
"H6": "H6-agentops",
|
||||
"H7": "H7-orchestration",
|
||||
}
|
||||
return [
|
||||
mapping[value]
|
||||
for value in str(harness or "").split("/")
|
||||
if value in mapping
|
||||
]
|
||||
|
||||
|
||||
def write_trace_events(rec, evidence):
|
||||
trace_id = rec.get("trace_id")
|
||||
if not trace_id:
|
||||
return
|
||||
path = os.path.join(trace_event_dir(), "%s.jsonl" % trace_id)
|
||||
with _FileLock(path):
|
||||
with open(path, "a", encoding="utf-8") as fh:
|
||||
for gate_id in _gate_ids(evidence.get("h")):
|
||||
event = {
|
||||
"schema_version": 1,
|
||||
"timestamp": evidence.get("at"),
|
||||
"trace_id": trace_id,
|
||||
"gate_id": gate_id,
|
||||
"status": _gate_status(evidence.get("decision")),
|
||||
"reason": "%s: %s" % (
|
||||
evidence.get("kind"),
|
||||
evidence.get("detail"),
|
||||
),
|
||||
"evidence": {
|
||||
"kind": evidence.get("kind"),
|
||||
"decision": evidence.get("decision"),
|
||||
"detail": evidence.get("detail"),
|
||||
"certification_strength": rec.get(
|
||||
"certification_strength"),
|
||||
},
|
||||
}
|
||||
fh.write(json.dumps(
|
||||
event, ensure_ascii=False, separators=(",", ":")) + "\n")
|
||||
|
||||
|
||||
def add_evidence(rec, h, kind, decision, detail):
|
||||
rec.setdefault("evidence", []).append({
|
||||
evidence = {
|
||||
"h": h,
|
||||
"kind": kind,
|
||||
"decision": decision,
|
||||
"at": now_iso(),
|
||||
"detail": redact(detail, 160),
|
||||
})
|
||||
}
|
||||
rec.setdefault("evidence", []).append(evidence)
|
||||
write_trace_events(rec, evidence)
|
||||
|
||||
|
||||
def classify_telemetry(rec):
|
||||
@@ -853,6 +1093,20 @@ def op_finalize(req):
|
||||
stop_reason = req.get("stop_reason") or "completed"
|
||||
status = "success" if stop_reason in ("completed", "max_turns") else "failed"
|
||||
|
||||
observed_harnesses = {
|
||||
harness
|
||||
for evidence in rec.get("evidence", [])
|
||||
for harness in str(evidence.get("h") or "").split("/")
|
||||
}
|
||||
if "H2" not in observed_harnesses:
|
||||
add_evidence(
|
||||
rec, "H2", "tool-boundary", "pass",
|
||||
"no_tool_calls_side_effect_boundary_not_exercised")
|
||||
if "H5" not in observed_harnesses:
|
||||
add_evidence(
|
||||
rec, "H5", "governance-ledger", "pass",
|
||||
"no_side_effecting_action_required_a_decision")
|
||||
|
||||
# H3/H5/H7 finalize controls: run the H4 output filter over the assistant
|
||||
# summary as the closing verification control.
|
||||
reasons = []
|
||||
@@ -914,9 +1168,23 @@ def op_finalize(req):
|
||||
rec["certified"] = certified
|
||||
rec["finalized"] = True
|
||||
rec["finalized_at"] = now_iso()
|
||||
add_evidence(
|
||||
rec,
|
||||
"H7",
|
||||
"certification",
|
||||
"certified" if certified else "non_certified",
|
||||
",".join(reasons),
|
||||
)
|
||||
|
||||
quality, warnings, missing = classify_telemetry(rec)
|
||||
write_h6_record(rec, status, quality, warnings, missing)
|
||||
add_evidence(
|
||||
rec,
|
||||
"H6",
|
||||
"telemetry-quality",
|
||||
quality,
|
||||
",".join(warnings) if warnings else "provider_usage_complete",
|
||||
)
|
||||
h6_record = write_h6_record(rec, status, quality, warnings, missing)
|
||||
trace_path = write_trace_file(rec, certified, reasons)
|
||||
save_admission(rec)
|
||||
|
||||
@@ -926,6 +1194,15 @@ def op_finalize(req):
|
||||
reason=",".join(reasons))
|
||||
resp["warnings"].extend(warnings)
|
||||
resp["context"] = "trace=%s certified=%s" % (os.path.basename(trace_path), certified)
|
||||
report_url = dashboard_url(rec.get("trace_id"))
|
||||
resp["report_url"] = report_url
|
||||
resp["receipt"] = write_run_receipt(
|
||||
rec, resp["decision"], quality, warnings, report_url)
|
||||
spool_path = spool_run_envelope(
|
||||
rec, resp["receipt"], h6_record, trace_path)
|
||||
if spawn_delivery(spool_path):
|
||||
resp["receipt"]["delivery"] = "async_delivery_started"
|
||||
atomic_write_json(latest_run_path(), resp["receipt"])
|
||||
return resp
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user