feat: add live H1-H7 trace explorer

This commit is contained in:
thanhnv
2026-07-10 23:17:23 +09:00
parent 7cb592a32f
commit 1345d4c930
11 changed files with 496 additions and 28 deletions
+1
View File
@@ -42,6 +42,7 @@ yarn-error.log*
pnpm-debug.log* pnpm-debug.log*
.specify/logs/tmp/ .specify/logs/tmp/
.specify/logs/trace/ .specify/logs/trace/
.specify/logs/trace-events/
.specify/logs/chat/ .specify/logs/chat/
.specify/logs/idempotency/ .specify/logs/idempotency/
.specify/logs/level5/rollback-backups/ .specify/logs/level5/rollback-backups/
@@ -42,6 +42,7 @@ export const PATHS = {
incidents: env('CASAN_CP_INCIDENTS', '.specify/logs/level5/incidents.jsonl'), incidents: env('CASAN_CP_INCIDENTS', '.specify/logs/level5/incidents.jsonl'),
alerts: env('CASAN_DASHBOARD_ALERTS', '.specify/agentops/alerts.log'), alerts: env('CASAN_DASHBOARD_ALERTS', '.specify/agentops/alerts.log'),
traceDir: env('CASAN_CP_TRACE_DIR', '.specify/logs/trace'), traceDir: env('CASAN_CP_TRACE_DIR', '.specify/logs/trace'),
traceEventDir: env('CASAN_CP_TRACE_EVENT_DIR', '.specify/logs/trace-events'),
traceability: env('CASAN_CP_TRACEABILITY', 'docs/output/casan/traceability-matrix.json'), traceability: env('CASAN_CP_TRACEABILITY', 'docs/output/casan/traceability-matrix.json'),
drift: env('CASAN_CP_DRIFT', 'docs/output/casan/level5-evidence/09-drift-report.json'), drift: env('CASAN_CP_DRIFT', 'docs/output/casan/level5-evidence/09-drift-report.json'),
businessKpi: env('CASAN_CP_KPI', 'docs/output/casan/level5-evidence/14-business-kpi-report.json'), businessKpi: env('CASAN_CP_KPI', 'docs/output/casan/level5-evidence/14-business-kpi-report.json'),
@@ -1,6 +1,7 @@
// Read-only Ops Console API. Every handler returns the standard ok() envelope. No writes, // Read-only Ops Console API. Every handler returns the standard ok() envelope. No writes,
// no auth (loopback-bound, "Đọc ≠ Ghi"); management/RBAC is Plan-13 Track 2/3 (future). // no auth (loopback-bound, "Đọc ≠ Ghi"); management/RBAC is Plan-13 Track 2/3 (future).
import { Controller, Get, Inject, Param, Query } from '@nestjs/common'; import { Controller, Get, Inject, Param, Query, Req, Res } from '@nestjs/common';
import type { Request, Response } from 'express';
import { ok } from '../common/api-response.js'; import { ok } from '../common/api-response.js';
import { TelemetryService } from './telemetry.service.js'; import { TelemetryService } from './telemetry.service.js';
@@ -27,6 +28,38 @@ export class TelemetryController {
return ok(this.svc.run(traceId)); return ok(this.svc.run(traceId));
} }
@Get('runs/:traceId/graph')
traceGraph(@Param('traceId') traceId: string) {
return ok(this.svc.traceGraph(traceId));
}
@Get('runs/:traceId/events')
traceEvents(@Param('traceId') traceId: string, @Req() req: Request, @Res() res: Response) {
res.setHeader('Content-Type', 'text/event-stream');
res.setHeader('Cache-Control', 'no-cache, no-transform');
res.setHeader('Connection', 'keep-alive');
res.flushHeaders();
let previous = '';
const publish = () => {
const graph = this.svc.traceGraph(traceId);
const serialized = JSON.stringify(graph);
if (serialized !== previous) {
res.write(`event: trace\ndata: ${serialized}\n\n`);
previous = serialized;
} else {
res.write(': heartbeat\n\n');
}
};
const timer = setInterval(publish, 750);
const close = () => {
clearInterval(timer);
if (!res.writableEnded) res.end();
};
req.on('close', close);
publish();
}
@Get('governance') @Get('governance')
governance() { governance() {
return ok(this.svc.governance()); return ok(this.svc.governance());
@@ -9,6 +9,36 @@ import { APP_ROOT, PATHS, isStale, metricsAgeSeconds, STALE_AFTER_S } from '../c
import { readJsonl, readJson, readHead, readTrace } from './telemetry.reader.js'; import { readJsonl, readJson, readHead, readTrace } from './telemetry.reader.js';
type Row = Record<string, any>; type Row = Record<string, any>;
export type HarnessGateStatus = 'queued' | 'running' | 'pass' | 'warning' | 'blocked' | 'error' | 'skipped';
export interface HarnessTraceEvent {
timestamp: string;
trace_id: string;
gate_id: string;
status: HarnessGateStatus;
reason: string;
evidence: Record<string, unknown>;
}
export interface HarnessGateNode {
id: string;
title: string;
description: string;
status: HarnessGateStatus;
reason: string;
updated_at: string | null;
evidence: Record<string, unknown>;
events: HarnessTraceEvent[];
}
const HARNESS_GATES = [
{ id: 'H1-context', title: 'H1 · Context', description: 'Prompt contract, mode and risk classification' },
{ id: 'H2-tool', title: 'H2 · Tool', description: 'Allowlisted source and tool preparation' },
{ id: 'H3-eval', title: 'H3 · Eval', description: 'Grounded synthesis and quality evaluation' },
{ id: 'H4-security', title: 'H4 · Security', description: 'Input and output security boundary' },
{ id: 'H5-governance', title: 'H5 · Governance', description: 'Decision policy and append-only audit' },
{ id: 'H6-agentops', title: 'H6 · AgentOps', description: 'Runtime, token, cost and failure telemetry' },
{ id: 'H7-orchestration', title: 'H7 · Orchestration', description: 'Final governed outcome and certification' },
] as const;
type ChatReplayResult = { type ChatReplayResult = {
ok: boolean; ok: boolean;
decision: string; decision: string;
@@ -27,6 +57,22 @@ const recent = (rows: Row[], n: number) => rows.slice(-n).reverse();
const arr = (v: any): any[] => (Array.isArray(v) ? v : []); const arr = (v: any): any[] => (Array.isArray(v) ? v : []);
const pct = (part: number, total: number) => (total > 0 ? Math.round((part / total) * 1000) / 10 : 0); const pct = (part: number, total: number) => (total > 0 ? Math.round((part / total) * 1000) / 10 : 0);
function gateStatus(value: unknown): HarnessGateStatus {
const status = String(value ?? '').toLowerCase();
if (['success', 'pass', 'passed', 'allow', 'allowed', 'answered'].includes(status)) return 'pass';
if (['warn', 'warning'].includes(status)) return 'warning';
if (['block', 'blocked', 'deny', 'denied'].includes(status)) return 'blocked';
if (['fail', 'failed', 'error'].includes(status)) return 'error';
if (status === 'running') return 'running';
if (status === 'skipped') return 'skipped';
return 'queued';
}
function safeTraceId(traceId: string): string {
if (!/^[a-zA-Z0-9][a-zA-Z0-9._-]{0,127}$/.test(traceId)) return '';
return traceId;
}
function artifact(path: string, source: string, verifiedWhenPresent = true) { function artifact(path: string, source: string, verifiedWhenPresent = true) {
const present = existsSync(path); const present = existsSync(path);
const runAt = present ? statSync(path).mtime.toISOString() : null; const runAt = present ? statSync(path).mtime.toISOString() : null;
@@ -227,6 +273,82 @@ export class TelemetryService {
return trace ? { found: true, trace } : { found: false, trace: null }; return trace ? { found: true, trace } : { found: false, trace: null };
} }
traceGraph(traceId: string) {
const safeId = safeTraceId(traceId);
const events = safeId
? readJsonl<HarnessTraceEvent>(join(PATHS.traceEventDir, `${safeId}.jsonl`))
.filter((event) => event.trace_id === safeId && HARNESS_GATES.some((gate) => gate.id === event.gate_id))
.map((event) => ({
timestamp: String(event.timestamp ?? ''),
trace_id: safeId,
gate_id: String(event.gate_id ?? ''),
status: gateStatus(event.status),
reason: String(event.reason ?? ''),
evidence: event.evidence && typeof event.evidence === 'object' ? event.evidence : {},
}))
: [];
if (safeId && events.length === 0) {
const legacyTrace = readTrace(PATHS.traceDir, safeId);
if (legacyTrace) {
const gateId = String(legacyTrace.harness ?? '');
if (HARNESS_GATES.some((gate) => gate.id === gateId)) {
events.push({
timestamp: String(legacyTrace.timestamp ?? ''),
trace_id: safeId,
gate_id: gateId,
status: gateStatus(legacyTrace.status ?? legacyTrace.action),
reason: `Legacy ${gateId} trace`,
evidence: {
mode: legacyTrace.mode ?? null,
action: legacyTrace.action ?? null,
risk_level: legacyTrace.risk_level ?? null,
},
});
}
}
for (const metric of readJsonl<Row>(PATHS.metrics).filter((row) => row.trace_id === safeId)) {
events.push({
timestamp: String(metric.timestamp ?? ''),
trace_id: safeId,
gate_id: 'H6-agentops',
status: gateStatus(metric.status),
reason: String(metric.step ?? 'Legacy runtime metric'),
evidence: {
latency_ms: metric.latency_ms ?? null,
total_tokens: metric.total_tokens ?? null,
cost_estimate: metric.cost_estimate ?? null,
synthesis_mode: metric.synthesis_mode ?? null,
},
});
}
}
const nodes: HarnessGateNode[] = HARNESS_GATES.map((gate) => {
const gateEvents = events.filter((event) => event.gate_id === gate.id);
const latest = gateEvents[gateEvents.length - 1];
return {
...gate,
status: latest?.status ?? 'queued',
reason: latest?.reason ?? 'Waiting for evidence',
updated_at: latest?.timestamp || null,
evidence: latest?.evidence ?? {},
events: gateEvents,
};
});
const lastEvent = events[events.length - 1];
const outcome = nodes.find((node) => node.id === 'H7-orchestration');
return {
found: events.length > 0,
trace_id: safeId || traceId,
updated_at: lastEvent?.timestamp || null,
terminal: outcome?.status === 'pass' || outcome?.status === 'blocked' || outcome?.status === 'error',
progress: nodes.filter((node) => node.status !== 'queued').length,
nodes,
events,
};
}
governance() { governance() {
const audit = readJsonl(PATHS.audit); const audit = readJsonl(PATHS.audit);
const byDecision: Record<string, number> = {}; const byDecision: Record<string, number> = {};
@@ -0,0 +1,135 @@
import { useEffect, useMemo, useState } from 'react';
import { useQuery } from '@tanstack/react-query';
import { api, type HarnessGateNode, type HarnessGateStatus, type HarnessTraceGraph } from '../../lib/api';
import { Card, StatusBadge } from '../ui/Card';
const NODE_TONE: Record<HarnessGateStatus, string> = {
queued: 'border-slate-200 bg-slate-50 text-slate-500',
running: 'border-blue-300 bg-blue-50 text-blue-800 shadow-[0_0_0_4px_rgba(59,130,246,0.1)]',
pass: 'border-emerald-300 bg-emerald-50 text-emerald-800',
warning: 'border-amber-300 bg-amber-50 text-amber-800',
blocked: 'border-rose-400 bg-rose-50 text-rose-800 shadow-[0_0_0_4px_rgba(244,63,94,0.09)]',
error: 'border-rose-400 bg-rose-50 text-rose-800 shadow-[0_0_0_4px_rgba(244,63,94,0.09)]',
skipped: 'border-slate-200 bg-slate-100 text-slate-400',
};
const STATUS_MARK: Record<HarnessGateStatus, string> = {
queued: '○',
running: '●',
pass: '✓',
warning: '!',
blocked: '×',
error: '×',
skipped: '–',
};
function Evidence({ node }: { node: HarnessGateNode }) {
const entries = Object.entries(node.evidence);
return (
<div className="grid gap-4 lg:grid-cols-[minmax(0,1fr)_minmax(18rem,0.8fr)]">
<div>
<div className="flex flex-wrap items-center gap-2">
<h3 className="text-lg font-semibold text-slate-900">{node.title}</h3>
<StatusBadge value={node.status} />
</div>
<p className="mt-1 text-sm text-slate-500">{node.description}</p>
<div className="mt-4 rounded-xl border border-slate-200 bg-slate-50 p-4">
<div className="text-xs font-semibold uppercase tracking-[0.12em] text-slate-400">Latest decision</div>
<p className="mt-2 text-sm leading-6 text-slate-700">{node.reason}</p>
{node.updated_at && <p className="mt-2 text-xs text-slate-400">{node.updated_at.replace('T', ' ').replace('Z', '')}</p>}
</div>
</div>
<div className="rounded-xl border border-slate-800 bg-slate-950 p-4 text-slate-100">
<div className="text-xs font-semibold uppercase tracking-[0.12em] text-slate-400">Safe evidence</div>
{entries.length > 0 ? (
<dl className="mt-3 space-y-2 text-xs">
{entries.map(([key, value]) => (
<div key={key} className="grid grid-cols-[8rem_minmax(0,1fr)] gap-3 border-b border-slate-800 pb-2 last:border-0">
<dt className="font-medium text-slate-400">{key}</dt>
<dd className="break-all font-mono text-slate-200">{typeof value === 'string' ? value : JSON.stringify(value)}</dd>
</div>
))}
</dl>
) : <p className="mt-3 text-sm text-slate-500">No evidence has been emitted for this gate yet.</p>}
</div>
</div>
);
}
export function TraceExplorer({ traceId, onClose }: { traceId: string; onClose?: () => void }) {
const query = useQuery({
queryKey: ['trace-graph', traceId],
queryFn: () => api.traceGraph(traceId),
enabled: Boolean(traceId),
});
const [graph, setGraph] = useState<HarnessTraceGraph | null>(null);
const [selectedId, setSelectedId] = useState('H1-context');
useEffect(() => {
if (query.data) setGraph(query.data);
}, [query.data]);
useEffect(() => {
if (!traceId) return undefined;
return api.watchTrace(traceId, setGraph);
}, [traceId]);
const selected = useMemo(
() => graph?.nodes.find((node) => node.id === selectedId) ?? graph?.nodes[0] ?? null,
[graph, selectedId],
);
return (
<Card
title="Harness trace explorer"
right={onClose && <button type="button" onClick={onClose} className="rounded-lg border border-slate-200 px-3 py-1.5 text-xs font-medium text-slate-600 transition-colors hover:bg-slate-50">Close</button>}
className="overflow-hidden"
>
<div className="mb-5 flex flex-wrap items-end justify-between gap-3">
<div>
<p className="text-[11px] font-semibold uppercase tracking-[0.13em] text-slate-400">Trace ID</p>
<p className="mt-1 break-all font-mono text-sm text-slate-700">{traceId}</p>
</div>
<div className="flex items-center gap-2 text-xs text-slate-500">
<span className={`h-2 w-2 rounded-full ${graph?.terminal ? 'bg-emerald-500' : 'animate-pulse bg-blue-500'}`} />
{graph?.terminal ? 'Completed' : 'Live · waiting for events'}
<span className="text-slate-300">·</span>
{graph?.progress ?? 0}/7 gates
</div>
</div>
{query.isLoading && !graph ? <div className="py-10 text-center text-sm text-slate-500">Loading trace…</div> : (
<>
<div className="overflow-x-auto pb-5">
<div className="flex min-w-max items-center px-1 py-3">
{graph?.nodes.map((node, index) => {
const next = graph.nodes[index + 1];
const edgeActive = node.status === 'pass' || node.status === 'running' || node.status === 'warning' || next?.status === 'running';
return (
<div key={node.id} className="flex items-center">
<button
type="button"
aria-pressed={selectedId === node.id}
onClick={() => setSelectedId(node.id)}
className={`w-40 rounded-xl border p-3 text-left transition duration-200 hover:-translate-y-0.5 hover:shadow-md ${NODE_TONE[node.status]} ${selectedId === node.id ? 'ring-2 ring-slate-900/10' : ''}`}
>
<div className="flex items-center justify-between gap-2">
<span className="text-xs font-bold tracking-wide">{node.title.split(' · ')[0]}</span>
<span className={`flex h-6 w-6 items-center justify-center rounded-full border border-current text-xs ${node.status === 'running' ? 'animate-pulse' : ''}`}>{STATUS_MARK[node.status]}</span>
</div>
<div className="mt-3 text-sm font-semibold">{node.title.split(' · ')[1]}</div>
<div className="mt-1 truncate text-[11px] opacity-70">{node.reason}</div>
</button>
{next && <div className={`trace-edge mx-2 ${edgeActive ? 'trace-edge-active' : ''}`} aria-hidden="true" />}
</div>
);
})}
</div>
</div>
{selected && <Evidence node={selected} />}
{!graph?.found && <div className="mt-4 rounded-xl border border-amber-200 bg-amber-50 p-3 text-sm text-amber-800">This is a legacy or not-yet-started trace. New chat turns emit detailed H1–H7 events automatically.</div>}
</>
)}
</Card>
);
}
@@ -23,8 +23,9 @@ export function StatTile({ label, value, sub }: { label: string; value: ReactNod
} }
const TONE: Record<string, string> = { const TONE: Record<string, string> = {
ok: 'border-emerald-200 bg-emerald-50 text-emerald-700', pass: 'border-emerald-200 bg-emerald-50 text-emerald-700', success: 'border-emerald-200 bg-emerald-50 text-emerald-700', allow: 'border-emerald-200 bg-emerald-50 text-emerald-700', approved: 'border-emerald-200 bg-emerald-50 text-emerald-700', answered: 'border-emerald-200 bg-emerald-50 text-emerald-700', certified: 'border-emerald-200 bg-emerald-50 text-emerald-700', verified: 'border-emerald-200 bg-emerald-50 text-emerald-700', ok: 'border-emerald-200 bg-emerald-50 text-emerald-700', pass: 'border-emerald-200 bg-emerald-50 text-emerald-700', success: 'border-emerald-200 bg-emerald-50 text-emerald-700', allow: 'border-emerald-200 bg-emerald-50 text-emerald-700', approved: 'border-emerald-200 bg-emerald-50 text-emerald-700', answered: 'border-emerald-200 bg-emerald-50 text-emerald-700', certified: 'border-emerald-200 bg-emerald-50 text-emerald-700', verified: 'border-emerald-200 bg-emerald-50 text-emerald-700',
warn: 'border-amber-200 bg-amber-50 text-amber-700', stale: 'border-amber-200 bg-amber-50 text-amber-700', draft: 'border-amber-200 bg-amber-50 text-amber-700', pending: 'border-amber-200 bg-amber-50 text-amber-700', attention: 'border-amber-200 bg-amber-50 text-amber-700', running: 'border-blue-200 bg-blue-50 text-blue-700', streaming: 'border-blue-200 bg-blue-50 text-blue-700',
fail: 'border-rose-200 bg-rose-50 text-rose-700', failed: 'border-rose-200 bg-rose-50 text-rose-700', denied: 'border-rose-200 bg-rose-50 text-rose-700', blocked: 'border-rose-200 bg-rose-50 text-rose-700', deny: 'border-rose-200 bg-rose-50 text-rose-700', block: 'border-rose-200 bg-rose-50 text-rose-700', crit: 'border-rose-200 bg-rose-50 text-rose-700', breach: 'border-rose-200 bg-rose-50 text-rose-700', halted: 'border-rose-200 bg-rose-50 text-rose-700', warn: 'border-amber-200 bg-amber-50 text-amber-700', warning: 'border-amber-200 bg-amber-50 text-amber-700', stale: 'border-amber-200 bg-amber-50 text-amber-700', draft: 'border-amber-200 bg-amber-50 text-amber-700', pending: 'border-amber-200 bg-amber-50 text-amber-700', attention: 'border-amber-200 bg-amber-50 text-amber-700',
fail: 'border-rose-200 bg-rose-50 text-rose-700', failed: 'border-rose-200 bg-rose-50 text-rose-700', error: 'border-rose-200 bg-rose-50 text-rose-700', denied: 'border-rose-200 bg-rose-50 text-rose-700', blocked: 'border-rose-200 bg-rose-50 text-rose-700', deny: 'border-rose-200 bg-rose-50 text-rose-700', block: 'border-rose-200 bg-rose-50 text-rose-700', crit: 'border-rose-200 bg-rose-50 text-rose-700', breach: 'border-rose-200 bg-rose-50 text-rose-700', halted: 'border-rose-200 bg-rose-50 text-rose-700',
}; };
export function StatusBadge({ value }: { value: string }) { export function StatusBadge({ value }: { value: string }) {
const tone = TONE[String(value).toLowerCase()] ?? 'border-slate-200 bg-slate-50 text-slate-600'; const tone = TONE[String(value).toLowerCase()] ?? 'border-slate-200 bg-slate-50 text-slate-600';
@@ -27,3 +27,37 @@ button, input, select, textarea { font: inherit; }
::-webkit-scrollbar { width: 10px; height: 10px; } ::-webkit-scrollbar { width: 10px; height: 10px; }
::-webkit-scrollbar-thumb { background: #c8d0dd; border: 3px solid transparent; border-radius: 999px; background-clip: padding-box; } ::-webkit-scrollbar-thumb { background: #c8d0dd; border: 3px solid transparent; border-radius: 999px; background-clip: padding-box; }
::-webkit-scrollbar-track { background: transparent; } ::-webkit-scrollbar-track { background: transparent; }
@keyframes trace-flow {
from { transform: translateX(-150%); }
to { transform: translateX(350%); }
}
.trace-edge {
position: relative;
width: 2.75rem;
height: 2px;
flex: 0 0 2.75rem;
overflow: hidden;
border-radius: 999px;
background: #cbd5e1;
}
.trace-edge-active {
background: #93c5fd;
}
.trace-edge-active::after {
position: absolute;
inset-block: 0;
left: 0;
width: 35%;
border-radius: 999px;
background: linear-gradient(90deg, transparent, #2563eb, transparent);
content: "";
animation: trace-flow 1.2s linear infinite;
}
@media (prefers-reduced-motion: reduce) {
.trace-edge-active::after { animation: none; }
}
@@ -23,6 +23,17 @@ async function post<T>(path: string, body: unknown, headers: Record<string, stri
export interface Freshness { stale: boolean; age_s: number | null; stale_after_s: number } export interface Freshness { stale: boolean; age_s: number | null; stale_after_s: number }
export interface HarnessRunRecord {
timestamp?: string;
trace_id?: string;
harness?: string;
step?: string;
status?: string;
latency_ms?: number;
total_tokens?: number;
cost_estimate?: number;
}
export interface Overview extends Freshness { export interface Overview extends Freshness {
totals: { totals: {
runs: number; total_cost: number; avg_latency_ms: number; failures: number; runs: number; total_cost: number; avg_latency_ms: number; failures: number;
@@ -217,6 +228,38 @@ export interface ProviderAuthStatus {
authMethod: string; authMethod: string;
} }
export type HarnessGateStatus = 'queued' | 'running' | 'pass' | 'warning' | 'blocked' | 'error' | 'skipped';
export interface HarnessTraceEvent {
timestamp: string;
trace_id: string;
gate_id: string;
status: HarnessGateStatus;
reason: string;
evidence: Record<string, unknown>;
}
export interface HarnessGateNode {
id: string;
title: string;
description: string;
status: HarnessGateStatus;
reason: string;
updated_at: string | null;
evidence: Record<string, unknown>;
events: HarnessTraceEvent[];
}
export interface HarnessTraceGraph {
found: boolean;
trace_id: string;
updated_at: string | null;
terminal: boolean;
progress: number;
nodes: HarnessGateNode[];
events: HarnessTraceEvent[];
}
export interface ChatReplay { export interface ChatReplay {
ok: boolean; ok: boolean;
decision: 'MATCH' | 'DRIFT' | 'BREAK' | string; decision: 'MATCH' | 'DRIFT' | 'BREAK' | string;
@@ -278,7 +321,22 @@ function actorHeaders(actor: SettingsActor): Record<string, string> {
export const api = { export const api = {
overview: () => get<Overview>('overview'), overview: () => get<Overview>('overview'),
runs: (limit = 50) => get<Freshness & { count: number; runs: any[] }>(`runs?limit=${limit}`), runs: (limit = 50) => get<Freshness & { count: number; runs: HarnessRunRecord[] }>(`runs?limit=${limit}`),
traceGraph: (traceId: string) => get<HarnessTraceGraph>(`runs/${encodeURIComponent(traceId)}/graph`),
watchTrace: (traceId: string, onGraph: (graph: HarnessTraceGraph) => void, onError?: () => void) => {
const base = import.meta.env.VITE_API_BASE_URL ?? '/api/v1';
const stream = new EventSource(`${base}/runs/${encodeURIComponent(traceId)}/events`, { withCredentials: true });
const handler = (event: MessageEvent<string>) => {
try {
onGraph(JSON.parse(event.data) as HarnessTraceGraph);
} catch {
onError?.();
}
};
stream.addEventListener('trace', handler as EventListener);
stream.onerror = () => onError?.();
return () => stream.close();
},
governance: () => get<Freshness & { records: number; by_decision: Record<string, number>; head: string | null; recent: any[] }>('governance'), governance: () => get<Freshness & { records: number; by_decision: Record<string, number>; head: string | null; recent: any[] }>('governance'),
security: () => get<Freshness & { verdicts: number; by_status: Record<string, number>; benign_fp: any; recent: any[] }>('security'), security: () => get<Freshness & { verdicts: number; by_status: Record<string, number>; benign_fp: any; recent: any[] }>('security'),
incidents: () => get<Freshness & { total: number; kill_switch_scopes: string[]; incidents: any[] }>('incidents'), incidents: () => get<Freshness & { total: number; kill_switch_scopes: string[]; incidents: any[] }>('incidents'),
@@ -1,5 +1,6 @@
import { type KeyboardEvent, useMemo, useState } from 'react'; import { type KeyboardEvent, useMemo, useState } from 'react';
import { useMutation, useQuery } from '@tanstack/react-query'; import { useMutation, useQuery } from '@tanstack/react-query';
import { Link } from 'react-router-dom';
import { api, ChatAction, ChatAgent, ChatAnswer, ChatHistoryTurn, ChatModelProvider, ChatStreamPhase, SettingsActor } from '../lib/api'; import { api, ChatAction, ChatAgent, ChatAnswer, ChatHistoryTurn, ChatModelProvider, ChatStreamPhase, SettingsActor } from '../lib/api';
import { Card, StatusBadge } from '../components/ui/Card'; import { Card, StatusBadge } from '../components/ui/Card';
import { ModelConnectionPanel } from '../components/chat/ModelConnectionPanel'; import { ModelConnectionPanel } from '../components/chat/ModelConnectionPanel';
@@ -295,7 +296,7 @@ export function Chat() {
<div className="mt-5 text-[10px] font-bold uppercase tracking-[0.15em] text-slate-400">Latest verification</div> <div className="mt-5 text-[10px] font-bold uppercase tracking-[0.15em] text-slate-400">Latest verification</div>
<div className="mt-2 space-y-2"> <div className="mt-2 space-y-2">
<div className="rounded-xl border border-slate-200 bg-white p-3"><div className="text-xs text-slate-400">Audit anchor</div><div className="mt-1 break-all font-mono text-[11px] text-slate-700">{auditHash(last).slice(0, 22)}{auditHash(last) !== 'n/a' ? '…' : ''}</div></div> <div className="rounded-xl border border-slate-200 bg-white p-3"><div className="text-xs text-slate-400">Audit anchor</div><div className="mt-1 break-all font-mono text-[11px] text-slate-700">{auditHash(last).slice(0, 22)}{auditHash(last) !== 'n/a' ? '…' : ''}</div></div>
{last && <><div className="rounded-xl border border-slate-200 bg-white p-3"><div className="flex flex-wrap gap-1.5"><StatusBadge value={last.mode} /><StatusBadge value={last.risk} /><StatusBadge value={last.decision} /><StatusBadge value={last.certified ? 'certified' : 'held'} /></div><div className="mt-2 text-xs text-slate-500">{last.router?.reason ?? 'Policy route verified'}{last.synthesis?.fallback_from ? ` · fell back from ${last.synthesis.fallback_from} to local` : ''}</div></div><Card title="Evidence" className="p-3.5"><div className="space-y-2">{last.sources.slice(0, 3).map((source) => <div key={`${source.path}-${source.line ?? ''}`} className="rounded-lg bg-slate-50 p-2.5"><div className="truncate text-xs font-medium text-slate-700">{source.title || source.path}</div><div className="mt-1 text-[11px] leading-4 text-slate-500">{source.preview || source.excerpt || 'Verified source'}</div></div>)}{last.sources.length === 0 && <div className="text-xs text-slate-500">No source was returned for this decision.</div>}</div></Card></>} {last && <><div className="rounded-xl border border-slate-200 bg-white p-3"><div className="flex flex-wrap gap-1.5"><StatusBadge value={last.mode} /><StatusBadge value={last.risk} /><StatusBadge value={last.decision} /><StatusBadge value={last.certified ? 'certified' : 'held'} /></div><div className="mt-2 text-xs text-slate-500">{last.router?.reason ?? 'Policy route verified'}{last.synthesis?.fallback_from ? ` · fell back from ${last.synthesis.fallback_from} to local` : ''}</div>{last.trace_id && <Link to={`/runs?trace=${encodeURIComponent(last.trace_id)}`} className="mt-3 flex w-full items-center justify-center gap-2 rounded-lg border border-blue-200 bg-blue-50 px-3 py-2 text-xs font-semibold text-blue-700 transition-colors hover:bg-blue-100">Open live H1–H7 trace <Glyph name="chevron" /></Link>}</div><Card title="Evidence" className="p-3.5"><div className="space-y-2">{last.sources.slice(0, 3).map((source) => <div key={`${source.path}-${source.line ?? ''}`} className="rounded-lg bg-slate-50 p-2.5"><div className="truncate text-xs font-medium text-slate-700">{source.title || source.path}</div><div className="mt-1 text-[11px] leading-4 text-slate-500">{source.preview || source.excerpt || 'Verified source'}</div></div>)}{last.sources.length === 0 && <div className="text-xs text-slate-500">No source was returned for this decision.</div>}</div></Card></>}
</div> </div>
<div className="mt-5 text-[10px] font-bold uppercase tracking-[0.15em] text-slate-400">Registered actions</div> <div className="mt-5 text-[10px] font-bold uppercase tracking-[0.15em] text-slate-400">Registered actions</div>
@@ -1,31 +1,53 @@
import { useQuery } from '@tanstack/react-query'; import { useQuery } from '@tanstack/react-query';
import { api } from '../lib/api'; import { useSearchParams } from 'react-router-dom';
import { api, type HarnessRunRecord } from '../lib/api';
import { Card, StatusBadge } from '../components/ui/Card'; import { Card, StatusBadge } from '../components/ui/Card';
import { TraceExplorer } from '../components/trace/TraceExplorer';
export function Runs() { export function Runs() {
const { data, isLoading } = useQuery({ queryKey: ['runs'], queryFn: () => api.runs(100) }); const { data, isLoading } = useQuery({ queryKey: ['runs'], queryFn: () => api.runs(100) });
if (isLoading || !data) return <div className="text-gray-500">Loading…</div>; const [searchParams, setSearchParams] = useSearchParams();
const selectedTrace = searchParams.get('trace') ?? '';
if (isLoading || !data) return <div className="text-slate-500">Loading…</div>;
return ( return (
<Card title={`Recent runs (${data.count})`}> <div className="space-y-5">
<div className="overflow-x-auto"> {selectedTrace && <TraceExplorer traceId={selectedTrace} onClose={() => setSearchParams({})} />}
<table className="w-full text-sm"> <Card
<thead><tr className="text-left text-gray-500 border-b border-gray-200"> title={`Recent runs (${data.count})`}
<th className="py-2">time</th><th>step</th><th>status</th><th>latency</th><th>tokens</th><th>cost</th> right={<span className="text-xs font-normal normal-case tracking-normal text-slate-400">Select a run to inspect H1–H7</span>}
</tr></thead> >
<tbody> <div className="overflow-x-auto">
{data.runs.map((r: any, i: number) => ( <table className="w-full text-sm">
<tr key={i} className="border-b border-gray-100"> <thead><tr className="border-b border-slate-200 text-left text-slate-500">
<td className="py-2 text-gray-500">{r.timestamp?.replace('T', ' ').replace('Z', '')}</td> <th className="py-2">time</th><th>step</th><th>status</th><th>latency</th><th>tokens</th><th>cost</th><th className="text-right">trace</th>
<td className="text-gray-700">{r.step ?? r.harness}</td> </tr></thead>
<td><StatusBadge value={r.status ?? '—'} /></td> <tbody>
<td>{r.latency_ms ?? '—'} ms</td> {data.runs.map((run: HarnessRunRecord, index: number) => (
<td>{r.total_tokens ?? '—'}</td> <tr key={`${run.trace_id ?? 'run'}-${index}`} className="border-b border-slate-100 transition-colors hover:bg-slate-50/80">
<td>${Number(r.cost_estimate ?? 0).toFixed(5)}</td> <td className="py-2 text-slate-500">{run.timestamp?.replace('T', ' ').replace('Z', '')}</td>
</tr> <td className="text-slate-700">{run.step ?? run.harness}</td>
))} <td><StatusBadge value={run.status ?? '—'} /></td>
</tbody> <td>{run.latency_ms ?? '—'} ms</td>
</table> <td>{run.total_tokens ?? '—'}</td>
</div> <td>${Number(run.cost_estimate ?? 0).toFixed(5)}</td>
</Card> <td className="py-2 text-right">
{run.trace_id ? (
<button
type="button"
onClick={() => setSearchParams({ trace: run.trace_id ?? '' })}
className="rounded-lg border border-blue-200 bg-blue-50 px-3 py-1.5 text-xs font-semibold text-blue-700 transition-colors hover:bg-blue-100"
>
Open H1–H7
</button>
) : <span className="text-slate-300">—</span>}
</td>
</tr>
))}
</tbody>
</table>
</div>
</Card>
</div>
); );
} }
@@ -92,6 +92,12 @@ def metrics_path() -> str:
return guarded_override(os.environ["CASAN_CHAT_METRICS_LOG"]) if os.environ.get("CASAN_CHAT_METRICS_LOG") else tenant_path("telemetry/cost/metrics.jsonl", "logs/cost/metrics.jsonl") return guarded_override(os.environ["CASAN_CHAT_METRICS_LOG"]) if os.environ.get("CASAN_CHAT_METRICS_LOG") else tenant_path("telemetry/cost/metrics.jsonl", "logs/cost/metrics.jsonl")
def trace_events_path(trace_id: str) -> str:
override = os.environ.get("CASAN_TRACE_EVENTS_DIR")
directory = guarded_override(override) if override else tenant_path("telemetry/trace-events", "logs/trace-events")
return os.path.join(directory, f"{trace_id}.jsonl")
def now_iso() -> str: def now_iso() -> str:
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
@@ -365,6 +371,22 @@ def append_jsonl(path: str, rec):
fh.write(json.dumps(rec, ensure_ascii=False) + "\n") fh.write(json.dumps(rec, ensure_ascii=False) + "\n")
def record_trace_event(trace_id: str, gate_id: str, status: str, reason: str, evidence=None):
"""Append a privacy-minimised event used by the live H1-H7 explorer."""
try:
append_jsonl(trace_events_path(trace_id), {
"timestamp": now_iso(),
"trace_id": trace_id,
"gate_id": gate_id,
"status": status,
"reason": reason,
"evidence": evidence or {},
})
except OSError:
# Observability must never turn an otherwise safe read-only answer into a failure.
pass
def load_head() -> str: def load_head() -> str:
try: try:
return open(head_path(), encoding="utf-8").read().strip() or GENESIS_HASH return open(head_path(), encoding="utf-8").read().strip() or GENESIS_HASH
@@ -486,10 +508,16 @@ def ask(args):
chat_id = args.chat_id or "chat-default" chat_id = args.chat_id or "chat-default"
turn_id = args.turn_id or f"turn-{trace_id[:12]}" turn_id = args.turn_id or f"turn-{trace_id[:12]}"
tenant_id = args.tenant or "default" tenant_id = args.tenant or "default"
record_trace_event(trace_id, "H1-context", "running", "Classifying prompt contract")
record_trace_event(trace_id, "H1-context", "pass" if router.get("mode") in ("READ_ONLY", "ANALYSIS") else "blocked", router.get("reason", "Prompt classified"), {
"mode": router.get("mode", "BLOCK"),
"risk": router.get("risk", "high"),
})
def finish(decision: str, answer: str, sources=None, safe_message="", synthesis=None): def finish(decision: str, answer: str, sources=None, safe_message="", synthesis=None):
elapsed = int((datetime.now(timezone.utc) - started).total_seconds() * 1000) elapsed = int((datetime.now(timezone.utc) - started).total_seconds() * 1000)
sources = sources or [] sources = sources or []
record_trace_event(trace_id, "H5-governance", "running", "Writing append-only decision audit")
rec = record_turn({ rec = record_turn({
"timestamp": now_iso(), "timestamp": now_iso(),
"trace_id": trace_id, "trace_id": trace_id,
@@ -508,7 +536,22 @@ def ask(args):
"synthesis_mode": (synthesis or {}).get("mode", "deterministic"), "synthesis_mode": (synthesis or {}).get("mode", "deterministic"),
"sources": [{"path": s.get("path"), "line": s.get("line")} for s in sources], "sources": [{"path": s.get("path"), "line": s.get("line")} for s in sources],
}) })
record_trace_event(trace_id, "H5-governance", "pass", "Decision audit recorded", {
"decision": decision,
"audit_seq": rec["seq"],
"audit_hash": rec["record_hash"],
})
record_trace_event(trace_id, "H6-agentops", "running", "Recording runtime metrics")
record_metrics(trace_id, safe_message or message, answer, "success" if decision == "ANSWERED" else "failed", elapsed, synthesis) record_metrics(trace_id, safe_message or message, answer, "success" if decision == "ANSWERED" else "failed", elapsed, synthesis)
record_trace_event(trace_id, "H6-agentops", "pass" if decision == "ANSWERED" else "error", "Runtime metrics recorded", {
"latency_ms": elapsed,
"status": "success" if decision == "ANSWERED" else "failed",
"synthesis_mode": (synthesis or {}).get("mode", "deterministic"),
})
record_trace_event(trace_id, "H7-orchestration", "pass" if decision == "ANSWERED" else "blocked", "Harness turn completed" if decision == "ANSWERED" else "Harness stopped with governed outcome", {
"decision": decision,
"certified": decision == "ANSWERED",
})
return { return {
"success": decision == "ANSWERED", "success": decision == "ANSWERED",
"chat_id": chat_id, "chat_id": chat_id,
@@ -532,8 +575,10 @@ def ask(args):
role = "analysis" if router.get("mode") == "ANALYSIS" else "read_only" role = "analysis" if router.get("mode") == "ANALYSIS" else "read_only"
record_trace_event(trace_id, "H4-security", "running", "Scanning input boundary")
rc, safe_input, scan_msg = run_security(message, "input") rc, safe_input, scan_msg = run_security(message, "input")
if rc != 0: if rc != 0:
record_trace_event(trace_id, "H4-security", "blocked", "Input rejected by security boundary", {"scan": scan_msg})
router["mode"] = "BLOCK" router["mode"] = "BLOCK"
router["reason"] = "h4_input_denied" router["reason"] = "h4_input_denied"
router["matched_rules"] = router.get("matched_rules", []) + [scan_msg] router["matched_rules"] = router.get("matched_rules", []) + [scan_msg]
@@ -541,8 +586,14 @@ def ask(args):
print(json.dumps(finish("DENIED", answer), ensure_ascii=False)) print(json.dumps(finish("DENIED", answer), ensure_ascii=False))
return 2 return 2
record_trace_event(trace_id, "H4-security", "running", "Input passed; output scan pending", {"input_scan": "pass"})
record_trace_event(trace_id, "H2-tool", "running", "Retrieving allowlisted evidence")
sources = collect_sources(safe_input) sources = collect_sources(safe_input)
history = load_history(chat_id, tenant_id) history = load_history(chat_id, tenant_id)
record_trace_event(trace_id, "H2-tool", "pass", "Allowlisted evidence prepared", {
"source_count": len(sources),
"history_available": bool(history),
})
# Item 3: streaming — emit a SAFE deterministic draft (whitelist-only, no model # Item 3: streaming — emit a SAFE deterministic draft (whitelist-only, no model
# text, no side-effect) tagged UNCERTIFIED, then continue to the certified final. # text, no side-effect) tagged UNCERTIFIED, then continue to the certified final.
@@ -560,9 +611,17 @@ def ask(args):
"synthesis": {"mode": "deterministic", "reason": "stream_draft"}, "synthesis": {"mode": "deterministic", "reason": "stream_draft"},
}, ensure_ascii=False), flush=True) }, ensure_ascii=False), flush=True)
record_trace_event(trace_id, "H3-eval", "running", "Synthesizing grounded answer")
answer, synthesis = synthesize_answer(safe_input, sources, role, history) answer, synthesis = synthesize_answer(safe_input, sources, role, history)
record_trace_event(trace_id, "H3-eval", "pass", "Grounded synthesis completed", {
"mode": synthesis.get("mode", "deterministic"),
"provider": synthesis.get("provider", "deterministic"),
"model": synthesis.get("model", "none"),
"source_count": len(sources),
})
rc, safe_answer, scan_msg = run_security(answer, "output") rc, safe_answer, scan_msg = run_security(answer, "output")
if rc != 0: if rc != 0:
record_trace_event(trace_id, "H4-security", "blocked", "Output rejected by security boundary", {"scan": scan_msg})
router["mode"] = "BLOCK" router["mode"] = "BLOCK"
router["reason"] = "h4_output_denied" router["reason"] = "h4_output_denied"
router["matched_rules"] = router.get("matched_rules", []) + [scan_msg] router["matched_rules"] = router.get("matched_rules", []) + [scan_msg]
@@ -572,6 +631,7 @@ def ask(args):
print(json.dumps(result, ensure_ascii=False)) print(json.dumps(result, ensure_ascii=False))
return 2 return 2
record_trace_event(trace_id, "H4-security", "pass", "Input and output security scans passed")
result = finish("ANSWERED", safe_answer, sources, safe_input, synthesis) result = finish("ANSWERED", safe_answer, sources, safe_input, synthesis)
if getattr(args, "stream", False): if getattr(args, "stream", False):
result["phase"] = "final" result["phase"] = "final"