629 lines
26 KiB
TypeScript
629 lines
26 KiB
TypeScript
// Read-only aggregations over CASAN harness telemetry. Formulas mirror
|
|
// packages/casan-harness/tests/generate-agentops-dashboard.py; all numbers come from real
|
|
// on-disk feeds. Missing feeds degrade to zero/empty + a `stale` flag — never fabricated.
|
|
import { existsSync, readFileSync, statSync } from 'node:fs';
|
|
import { execFileSync } from 'node:child_process';
|
|
import { join, relative } from 'node:path';
|
|
import { Injectable } from '@nestjs/common';
|
|
import { APP_ROOT, PATHS, isStale, metricsAgeSeconds, STALE_AFTER_S, telemetryFreshness } from '../common/app-root.js';
|
|
import { readJsonl, readJson, readHead, readTrace } from './telemetry.reader.js';
|
|
|
|
type Row = Record<string, any>;
|
|
export type HarnessGateStatus = 'queued' | 'running' | 'pass' | 'warning' | 'blocked' | 'error' | 'skipped';
|
|
export interface HarnessTraceEvent {
|
|
timestamp: string;
|
|
trace_id: string;
|
|
gate_id: string;
|
|
status: HarnessGateStatus;
|
|
reason: string;
|
|
evidence: Record<string, unknown>;
|
|
}
|
|
|
|
export interface HarnessGateNode {
|
|
id: string;
|
|
title: string;
|
|
description: string;
|
|
status: HarnessGateStatus;
|
|
reason: string;
|
|
updated_at: string | null;
|
|
evidence: Record<string, unknown>;
|
|
events: HarnessTraceEvent[];
|
|
}
|
|
|
|
const HARNESS_GATES = [
|
|
{ id: 'H1-context', title: 'H1 · Context', description: 'Prompt contract, mode and risk classification' },
|
|
{ id: 'H2-tool', title: 'H2 · Tool', description: 'Allowlisted source and tool preparation' },
|
|
{ id: 'H3-eval', title: 'H3 · Eval', description: 'Grounded synthesis and quality evaluation' },
|
|
{ id: 'H4-security', title: 'H4 · Security', description: 'Input and output security boundary' },
|
|
{ id: 'H5-governance', title: 'H5 · Governance', description: 'Decision policy and append-only audit' },
|
|
{ id: 'H6-agentops', title: 'H6 · AgentOps', description: 'Runtime, token, cost and failure telemetry' },
|
|
{ id: 'H7-orchestration', title: 'H7 · Orchestration', description: 'Final governed outcome and certification' },
|
|
] as const;
|
|
type ChatReplayResult = {
|
|
ok: boolean;
|
|
decision: string;
|
|
records: number;
|
|
loop_replayed?: number;
|
|
diffs?: any[];
|
|
audit_path?: string;
|
|
error?: string;
|
|
};
|
|
|
|
const CHAT_REPLAY_CLI = join(APP_ROOT, 'packages', 'casan-harness', 'scripts', 'bash', 'chat-replay.py');
|
|
const num = (v: any) => (typeof v === 'number' && isFinite(v) ? v : 0);
|
|
const sum = (rows: Row[], k: string) => rows.reduce((a, r) => a + num(r[k]), 0);
|
|
const count = (rows: Row[], pred: (r: Row) => boolean) => rows.reduce((a, r) => a + (pred(r) ? 1 : 0), 0);
|
|
const recent = (rows: Row[], n: number) => rows.slice(-n).reverse();
|
|
const arr = (v: any): any[] => (Array.isArray(v) ? v : []);
|
|
const pct = (part: number, total: number) => (total > 0 ? Math.round((part / total) * 1000) / 10 : 0);
|
|
|
|
function gateStatus(value: unknown): HarnessGateStatus {
|
|
const status = String(value ?? '').toLowerCase();
|
|
if (['success', 'pass', 'passed', 'allow', 'allowed', 'answered'].includes(status)) return 'pass';
|
|
if (['warn', 'warning'].includes(status)) return 'warning';
|
|
if (['block', 'blocked', 'deny', 'denied'].includes(status)) return 'blocked';
|
|
if (['fail', 'failed', 'error'].includes(status)) return 'error';
|
|
if (status === 'running') return 'running';
|
|
if (status === 'skipped') return 'skipped';
|
|
return 'queued';
|
|
}
|
|
|
|
function safeTraceId(traceId: string): string {
|
|
if (!/^[a-zA-Z0-9][a-zA-Z0-9._-]{0,127}$/.test(traceId)) return '';
|
|
return traceId;
|
|
}
|
|
|
|
function artifact(path: string, source: string, verifiedWhenPresent = true) {
|
|
const present = existsSync(path);
|
|
const runAt = present ? statSync(path).mtime.toISOString() : null;
|
|
return {
|
|
source,
|
|
artifact_path: relative(APP_ROOT, path) || '.',
|
|
commit: gitCommit(),
|
|
run_at: runAt,
|
|
verified: present && verifiedWhenPresent,
|
|
status: present ? (verifiedWhenPresent ? 'verified' : 'present_unverified') : 'missing',
|
|
};
|
|
}
|
|
|
|
function gitCommit(): string | null {
|
|
try {
|
|
const head = readFileSync(join(APP_ROOT, '.git', 'HEAD'), 'utf8').trim();
|
|
if (/^[0-9a-f]{40}$/i.test(head)) return head;
|
|
const ref = head.match(/^ref:\s+(.+)$/)?.[1];
|
|
if (!ref) return null;
|
|
const value = readFileSync(join(APP_ROOT, '.git', ref), 'utf8').trim();
|
|
return /^[0-9a-f]{40}$/i.test(value) ? value : null;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function widget(
|
|
id: string,
|
|
title: string,
|
|
status: string,
|
|
summary: string,
|
|
metrics: Record<string, any>,
|
|
envelope: ReturnType<typeof artifact>,
|
|
evidence: Record<string, any> = {},
|
|
) {
|
|
return { id, title, status, summary, metrics, envelope, evidence };
|
|
}
|
|
|
|
function parseJson<T>(raw: string): T | null {
|
|
if (!raw) return null;
|
|
try {
|
|
return JSON.parse(raw) as T;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function runChatReplay(chatAuditPath: string): ChatReplayResult {
|
|
if (!existsSync(chatAuditPath)) {
|
|
return { ok: true, decision: 'NO_DATA', records: 0, loop_replayed: 0, diffs: [], audit_path: chatAuditPath };
|
|
}
|
|
try {
|
|
const stdout = execFileSync('python3', [CHAT_REPLAY_CLI, 'replay'], {
|
|
cwd: APP_ROOT,
|
|
encoding: 'utf8',
|
|
stdio: ['ignore', 'pipe', 'pipe'],
|
|
env: { ...process.env, CASAN_CHAT_AUDIT_LOG: chatAuditPath },
|
|
timeout: 15000,
|
|
});
|
|
const parsed = parseJson<Record<string, any>>(stdout.trim());
|
|
return { ok: true, decision: String(parsed?.decision ?? 'UNKNOWN'), records: num(parsed?.records), ...parsed };
|
|
} catch (err: any) {
|
|
const stdout = String(err?.stdout ?? '').trim();
|
|
const stderr = String(err?.stderr ?? '').trim();
|
|
const parsed = parseJson<Record<string, any>>(stdout);
|
|
if (parsed) {
|
|
return { ok: false, decision: String(parsed.decision ?? 'ERROR'), records: num(parsed.records), ...parsed };
|
|
}
|
|
return { ok: false, decision: 'ERROR', records: 0, loop_replayed: 0, diffs: [], audit_path: chatAuditPath, error: stderr || stdout || 'CHAT_REPLAY_FAILED' };
|
|
}
|
|
}
|
|
|
|
function isChatMetric(row: Row) {
|
|
const agent = String(row.agent ?? '');
|
|
const step = String(row.step ?? '');
|
|
return agent.startsWith('chat.') || step === 'ask-casan-readonly' || step.startsWith('operator:');
|
|
}
|
|
|
|
export function buildChatLoopWidget(
|
|
chatAudit: Row[],
|
|
metrics: Row[],
|
|
chatAuditPath = PATHS.chatAudit,
|
|
replay: ChatReplayResult = runChatReplay(chatAuditPath),
|
|
) {
|
|
const latest = chatAudit[chatAudit.length - 1] ?? null;
|
|
const chatMetrics = metrics.filter(isChatMetric);
|
|
const tokenBudget = Math.max(Number(process.env.CASAN_CHAT_TOKEN_BUDGET ?? 100000) || 100000, 1);
|
|
const totalTokens = sum(chatMetrics, 'total_tokens');
|
|
const status = replay.decision === 'BREAK'
|
|
? 'fail'
|
|
: replay.decision === 'DRIFT' || replay.decision === 'ERROR'
|
|
? 'warn'
|
|
: chatAudit.length > 0
|
|
? 'ok'
|
|
: 'no_data';
|
|
const loopRuns = count(chatAudit, (r) => Boolean(r.loop_run?.run_id));
|
|
const verifiedReplay = replay.decision === 'MATCH' || replay.decision === 'NO_DATA';
|
|
|
|
return widget(
|
|
'chat_loop',
|
|
'Chat / Loop console',
|
|
status,
|
|
`${chatAudit.length} turn(s), ${loopRuns} loop-run(s), replay ${replay.decision}`,
|
|
{
|
|
turns: chatAudit.length,
|
|
answered: count(chatAudit, (r) => r.decision === 'ANSWERED'),
|
|
action_completed: count(chatAudit, (r) => r.decision === 'ACTION_COMPLETED'),
|
|
loop_runs: loopRuns,
|
|
replay_records: replay.records,
|
|
replay_loop_replayed: num(replay.loop_replayed),
|
|
budget_used_pct: pct(totalTokens, tokenBudget),
|
|
},
|
|
artifact(chatAuditPath, 'chat-turns.jsonl + chat-replay.py', verifiedReplay),
|
|
{
|
|
replay,
|
|
budget: { total_tokens: totalTokens, token_budget: tokenBudget, used_pct: pct(totalTokens, tokenBudget) },
|
|
latest_turn: latest ? {
|
|
seq: latest.seq ?? null,
|
|
timestamp: latest.timestamp ?? null,
|
|
chat_id: latest.chat_id ?? null,
|
|
turn_id: latest.turn_id ?? null,
|
|
tenant_id: latest.tenant_id ?? null,
|
|
mode: latest.mode ?? null,
|
|
decision: latest.decision ?? null,
|
|
loop_run: latest.loop_run ?? null,
|
|
record_hash: latest.record_hash ?? null,
|
|
} : null,
|
|
recent_turns: recent(chatAudit, 5).map((r) => ({
|
|
seq: r.seq ?? null,
|
|
timestamp: r.timestamp ?? null,
|
|
chat_id: r.chat_id ?? null,
|
|
turn_id: r.turn_id ?? null,
|
|
decision: r.decision ?? null,
|
|
mode: r.mode ?? null,
|
|
loop_run_id: r.loop_run?.run_id ?? null,
|
|
})),
|
|
},
|
|
);
|
|
}
|
|
|
|
@Injectable()
|
|
export class TelemetryService {
|
|
private freshness() {
|
|
const age = metricsAgeSeconds();
|
|
return { stale: isStale(), age_s: age, stale_after_s: STALE_AFTER_S, sources: telemetryFreshness() };
|
|
}
|
|
|
|
overview() {
|
|
const metrics = readJsonl(PATHS.metrics);
|
|
const provider = readJsonl(PATHS.providerUsage);
|
|
const audit = readJsonl(PATHS.audit);
|
|
const security = readJsonl(PATHS.security);
|
|
const fallback = readJsonl(PATHS.fallback);
|
|
const tools = readJsonl(PATHS.toolRegistry);
|
|
const actions = readJsonl(PATHS.actionGate);
|
|
const incidents = readJsonl(PATHS.incidents);
|
|
|
|
const runs = metrics.length;
|
|
const latencies = metrics.map((m) => num(m.latency_ms)).filter((x) => x > 0);
|
|
return {
|
|
...this.freshness(),
|
|
totals: {
|
|
runs,
|
|
total_cost: sum(metrics, 'cost_estimate'),
|
|
avg_latency_ms: latencies.length ? Math.round(latencies.reduce((a, b) => a + b, 0) / latencies.length) : 0,
|
|
failures: count(metrics, (m) => m.status === 'failed'),
|
|
hallucination_signals: sum(metrics, 'hallucination_signals'),
|
|
provider_tokens: sum(provider, 'total_tokens'),
|
|
provider_cost: sum(provider, 'cost_usd'),
|
|
fallback_routes: count(fallback, (f) => f.route === 'fallback'),
|
|
tool_denies: count(tools, (t) => t.decision === 'denied'),
|
|
action_blocks: count(actions, (a) => a.outcome === 'BLOCK'),
|
|
},
|
|
// Real per-harness signals (counts), not a hardcoded rubric — truthful by construction.
|
|
harness_signals: {
|
|
'H4-security': { verdicts: security.length, blocked: count(security, (s) => s.status === 'blocked') },
|
|
'H5-governance': { decisions: audit.length, denied: count(audit, (a) => a.decision === 'denied') },
|
|
'H6-agentops': { runs, failures: count(metrics, (m) => m.status === 'failed') },
|
|
'H7-drift': { report: readJson(PATHS.drift) ? 'present' : 'absent' },
|
|
tools: { decisions: tools.length, denied: count(tools, (t) => t.decision === 'denied') },
|
|
incidents: { total: incidents.length, critical: count(incidents, (i) => i.severity === 'CRIT') },
|
|
},
|
|
audit_chain: {
|
|
records: audit.length,
|
|
head: readHead(PATHS.auditHead),
|
|
last_decision: audit.length ? audit[audit.length - 1].decision ?? null : null,
|
|
},
|
|
};
|
|
}
|
|
|
|
// Prometheus exposition is deliberately aggregate-only. Do not add trace IDs,
|
|
// actor/tenant identifiers, prompts, filenames or evidence content as labels:
|
|
// those would turn an operations endpoint into a data-exfiltration path.
|
|
prometheusMetrics() {
|
|
const overview = this.overview();
|
|
const totals = overview.totals;
|
|
const lines = [
|
|
'# HELP casan_telemetry_stale Whether the primary telemetry feed is stale (1=true).',
|
|
'# TYPE casan_telemetry_stale gauge',
|
|
`casan_telemetry_stale ${overview.stale ? 1 : 0}`,
|
|
'# HELP casan_telemetry_age_seconds Age of the primary telemetry feed in seconds.',
|
|
'# TYPE casan_telemetry_age_seconds gauge',
|
|
`casan_telemetry_age_seconds ${overview.age_s ?? -1}`,
|
|
'# HELP casan_runs_total Number of telemetry records retained by CASAN.',
|
|
'# TYPE casan_runs_total gauge',
|
|
`casan_runs_total ${totals.runs}`,
|
|
'# HELP casan_failures_total Number of retained failed telemetry records.',
|
|
'# TYPE casan_failures_total gauge',
|
|
`casan_failures_total ${totals.failures}`,
|
|
'# HELP casan_cost_usd_total Aggregate estimated CASAN execution cost in USD.',
|
|
'# TYPE casan_cost_usd_total gauge',
|
|
`casan_cost_usd_total ${totals.total_cost}`,
|
|
'# HELP casan_provider_tokens_total Aggregate provider token usage.',
|
|
'# TYPE casan_provider_tokens_total gauge',
|
|
`casan_provider_tokens_total ${totals.provider_tokens}`,
|
|
'# HELP casan_security_blocks_total H4 security verdicts blocked.',
|
|
'# TYPE casan_security_blocks_total gauge',
|
|
`casan_security_blocks_total ${overview.harness_signals['H4-security'].blocked}`,
|
|
'# HELP casan_governance_denials_total H5 governance decisions denied.',
|
|
'# TYPE casan_governance_denials_total gauge',
|
|
`casan_governance_denials_total ${overview.harness_signals['H5-governance'].denied}`,
|
|
'# HELP casan_tool_denials_total Tool registry decisions denied.',
|
|
'# TYPE casan_tool_denials_total gauge',
|
|
`casan_tool_denials_total ${totals.tool_denies}`,
|
|
'# HELP casan_action_blocks_total Governed action gate block outcomes.',
|
|
'# TYPE casan_action_blocks_total gauge',
|
|
`casan_action_blocks_total ${totals.action_blocks}`,
|
|
'# HELP casan_critical_incidents_total Retained critical incident records.',
|
|
'# TYPE casan_critical_incidents_total gauge',
|
|
`casan_critical_incidents_total ${overview.harness_signals.incidents.critical}`,
|
|
'',
|
|
];
|
|
return lines.join('\n');
|
|
}
|
|
|
|
runs(limit = 50) {
|
|
const metrics = readJsonl(PATHS.metrics);
|
|
return { ...this.freshness(), count: metrics.length, runs: recent(metrics, limit) };
|
|
}
|
|
|
|
run(traceId: string) {
|
|
const trace = readTrace(PATHS.traceDir, traceId);
|
|
return trace ? { found: true, trace } : { found: false, trace: null };
|
|
}
|
|
|
|
traceGraph(traceId: string) {
|
|
const safeId = safeTraceId(traceId);
|
|
const events = safeId
|
|
? readJsonl<HarnessTraceEvent>(join(PATHS.traceEventDir, `${safeId}.jsonl`))
|
|
.filter((event) => event.trace_id === safeId && HARNESS_GATES.some((gate) => gate.id === event.gate_id))
|
|
.map((event) => ({
|
|
timestamp: String(event.timestamp ?? ''),
|
|
trace_id: safeId,
|
|
gate_id: String(event.gate_id ?? ''),
|
|
status: gateStatus(event.status),
|
|
reason: String(event.reason ?? ''),
|
|
evidence: event.evidence && typeof event.evidence === 'object' ? event.evidence : {},
|
|
}))
|
|
: [];
|
|
|
|
if (safeId && events.length === 0) {
|
|
const legacyTrace = readTrace(PATHS.traceDir, safeId);
|
|
if (legacyTrace) {
|
|
const gateId = String(legacyTrace.harness ?? '');
|
|
if (HARNESS_GATES.some((gate) => gate.id === gateId)) {
|
|
events.push({
|
|
timestamp: String(legacyTrace.timestamp ?? ''),
|
|
trace_id: safeId,
|
|
gate_id: gateId,
|
|
status: gateStatus(legacyTrace.status ?? legacyTrace.action),
|
|
reason: `Legacy ${gateId} trace`,
|
|
evidence: {
|
|
mode: legacyTrace.mode ?? null,
|
|
action: legacyTrace.action ?? null,
|
|
risk_level: legacyTrace.risk_level ?? null,
|
|
},
|
|
});
|
|
}
|
|
}
|
|
for (const metric of readJsonl<Row>(PATHS.metrics).filter((row) => row.trace_id === safeId)) {
|
|
events.push({
|
|
timestamp: String(metric.timestamp ?? ''),
|
|
trace_id: safeId,
|
|
gate_id: 'H6-agentops',
|
|
status: gateStatus(metric.status),
|
|
reason: String(metric.step ?? 'Legacy runtime metric'),
|
|
evidence: {
|
|
latency_ms: metric.latency_ms ?? null,
|
|
total_tokens: metric.total_tokens ?? null,
|
|
cost_estimate: metric.cost_estimate ?? null,
|
|
synthesis_mode: metric.synthesis_mode ?? null,
|
|
},
|
|
});
|
|
}
|
|
}
|
|
|
|
const nodes: HarnessGateNode[] = HARNESS_GATES.map((gate) => {
|
|
const gateEvents = events.filter((event) => event.gate_id === gate.id);
|
|
const latest = gateEvents[gateEvents.length - 1];
|
|
return {
|
|
...gate,
|
|
status: latest?.status ?? 'queued',
|
|
reason: latest?.reason ?? 'Waiting for evidence',
|
|
updated_at: latest?.timestamp || null,
|
|
evidence: latest?.evidence ?? {},
|
|
events: gateEvents,
|
|
};
|
|
});
|
|
const lastEvent = events[events.length - 1];
|
|
const outcome = nodes.find((node) => node.id === 'H7-orchestration');
|
|
return {
|
|
found: events.length > 0,
|
|
trace_id: safeId || traceId,
|
|
updated_at: lastEvent?.timestamp || null,
|
|
terminal: outcome?.status === 'pass' || outcome?.status === 'blocked' || outcome?.status === 'error',
|
|
progress: nodes.filter((node) => node.status !== 'queued').length,
|
|
nodes,
|
|
events,
|
|
};
|
|
}
|
|
|
|
governance() {
|
|
const audit = readJsonl(PATHS.audit);
|
|
const byDecision: Record<string, number> = {};
|
|
for (const r of audit) byDecision[String(r.decision ?? 'unknown')] = (byDecision[String(r.decision ?? 'unknown')] || 0) + 1;
|
|
return {
|
|
...this.freshness(),
|
|
records: audit.length,
|
|
by_decision: byDecision,
|
|
head: readHead(PATHS.auditHead),
|
|
recent: recent(audit, 30),
|
|
};
|
|
}
|
|
|
|
security() {
|
|
const s = readJsonl(PATHS.security);
|
|
const byStatus: Record<string, number> = {};
|
|
for (const r of s) byStatus[String(r.status ?? 'unknown')] = (byStatus[String(r.status ?? 'unknown')] || 0) + 1;
|
|
return {
|
|
...this.freshness(),
|
|
verdicts: s.length,
|
|
by_status: byStatus,
|
|
benign_fp: readJson(PATHS.benignFp),
|
|
recent: recent(s, 30),
|
|
};
|
|
}
|
|
|
|
incidents() {
|
|
const inc = readJsonl(PATHS.incidents);
|
|
const engaged = inc.filter((i) => i.action === 'kill_switch_engaged').map((i) => i.scope);
|
|
return {
|
|
...this.freshness(),
|
|
total: inc.length,
|
|
kill_switch_scopes: [...new Set(engaged)],
|
|
incidents: recent(inc, 50),
|
|
};
|
|
}
|
|
|
|
tools() {
|
|
return {
|
|
...this.freshness(),
|
|
tool_registry: recent(readJsonl(PATHS.toolRegistry), 50),
|
|
action_gate: recent(readJsonl(PATHS.actionGate), 50),
|
|
};
|
|
}
|
|
|
|
traceability() {
|
|
return { ...this.freshness(), matrix: readJson(PATHS.traceability) };
|
|
}
|
|
|
|
drift() {
|
|
return { ...this.freshness(), report: readJson(PATHS.drift) };
|
|
}
|
|
|
|
cost() {
|
|
const provider = readJsonl(PATHS.providerUsage);
|
|
return {
|
|
...this.freshness(),
|
|
provider_tokens: sum(provider, 'total_tokens'),
|
|
provider_cost: sum(provider, 'cost_usd'),
|
|
by_provider: recent(provider, 50),
|
|
business_kpi: readJson(PATHS.businessKpi),
|
|
};
|
|
}
|
|
|
|
commandCenter() {
|
|
const metrics = readJsonl(PATHS.metrics);
|
|
const provider = readJsonl(PATHS.providerUsage);
|
|
const audit = readJsonl(PATHS.audit);
|
|
const security = readJsonl(PATHS.security);
|
|
const incidents = readJsonl(PATHS.incidents);
|
|
const actionGate = readJsonl(PATHS.actionGate);
|
|
const chatAudit = readJsonl(PATHS.chatAudit);
|
|
const traceability = readJson(PATHS.traceability) as any;
|
|
const kpi = readJson(PATHS.businessKpi) as any;
|
|
const drift = readJson(PATHS.drift) as any;
|
|
const approvalStore = (readJson(PATHS.approvalInbox) as any) ?? { proposals: [], oversight: [] };
|
|
const proposals = arr(approvalStore.proposals);
|
|
const oversight = arr(approvalStore.oversight);
|
|
const head = readHead(PATHS.auditHead);
|
|
|
|
const passedReq = num(traceability?.summary?.passed);
|
|
const failedReq = num(traceability?.summary?.failed);
|
|
const totalReq = num(traceability?.summary?.requirements) || passedReq + failedReq;
|
|
const blockedSecurity = count(security, (s) => ['blocked', 'BLOCK', 'DENY'].includes(String(s.status ?? s.decision)));
|
|
const pendingApprovals = count(proposals, (p) => p.status === 'pending');
|
|
const approvedApprovals = count(proposals, (p) => p.status === 'approved' || p.status === 'auto_allowed');
|
|
const killSwitchScopes = [...new Set(incidents.filter((i) => i.action === 'kill_switch_engaged').map((i) => String(i.scope ?? 'unknown')))];
|
|
const kpis = arr(kpi?.kpis);
|
|
const kpisMet = count(kpis, (k) => k.target_met === true);
|
|
const auditVerified = audit.length > 0 && Boolean(head);
|
|
const selfImproveRelated = proposals.filter((p) => String(p.action ?? p.type ?? '').includes('improve'));
|
|
|
|
const widgets = {
|
|
maturity: widget(
|
|
'maturity',
|
|
'Maturity gauge + H1-H7 radar',
|
|
metrics.length || audit.length || security.length ? 'ok' : 'no_data',
|
|
`${metrics.length} runs, ${audit.length} governance records, ${security.length} security verdicts`,
|
|
{
|
|
runs: metrics.length,
|
|
governance_records: audit.length,
|
|
security_verdicts: security.length,
|
|
action_blocks: count(actionGate, (a) => a.outcome === 'BLOCK'),
|
|
drift_report: drift ? 'present' : 'missing',
|
|
},
|
|
artifact(PATHS.scoringReport, 'phase3-real-run-scoring.md'),
|
|
{ harness_signals: this.overview().harness_signals },
|
|
),
|
|
hitl: widget(
|
|
'hitl',
|
|
'Human-in-the-loop panel',
|
|
pendingApprovals > 0 ? 'warn' : (proposals.length > 0 ? 'ok' : 'no_data'),
|
|
`${pendingApprovals} pending, ${approvedApprovals} approved/auto-allowed`,
|
|
{
|
|
pending: pendingApprovals,
|
|
approved_or_auto: approvedApprovals,
|
|
rejected: count(proposals, (p) => p.status === 'rejected'),
|
|
oversight_events: oversight.length,
|
|
},
|
|
artifact(PATHS.approvalInbox, 'approval-inbox.json', oversight.length === 0 || Boolean(oversight[oversight.length - 1]?.hash)),
|
|
{ recent_oversight: recent(oversight, 5) },
|
|
),
|
|
kill_switch: widget(
|
|
'kill_switch',
|
|
'Kill-switch and guardrails',
|
|
killSwitchScopes.length > 0 ? 'fail' : 'ok',
|
|
killSwitchScopes.length ? `${killSwitchScopes.length} engaged scope(s)` : 'No active kill-switch incident in telemetry',
|
|
{
|
|
engaged_scopes: killSwitchScopes,
|
|
incidents: incidents.length,
|
|
critical: count(incidents, (i) => i.severity === 'CRIT'),
|
|
},
|
|
artifact(PATHS.incidents, 'incidents.jsonl'),
|
|
{ recent_incidents: recent(incidents, 5) },
|
|
),
|
|
traceability: widget(
|
|
'traceability',
|
|
'Traceability Sankey',
|
|
failedReq > 0 ? 'fail' : (totalReq > 0 ? 'ok' : 'no_data'),
|
|
totalReq > 0 ? `${passedReq}/${totalReq} requirements pass` : 'No traceability artifact found',
|
|
{
|
|
requirements: totalReq,
|
|
passed: passedReq,
|
|
failed: failedReq,
|
|
coverage_pct: pct(passedReq, totalReq),
|
|
},
|
|
artifact(PATHS.traceability, 'traceability-matrix.json'),
|
|
{ summary: traceability?.summary ?? null },
|
|
),
|
|
security: widget(
|
|
'security',
|
|
'Security posture',
|
|
blockedSecurity > 0 ? 'ok' : (security.length > 0 ? 'warn' : 'no_data'),
|
|
`${blockedSecurity}/${security.length} security verdicts blocked`,
|
|
{
|
|
verdicts: security.length,
|
|
blocked: blockedSecurity,
|
|
block_rate_pct: pct(blockedSecurity, security.length),
|
|
},
|
|
artifact(PATHS.security, 'security.jsonl'),
|
|
{ recent_security: recent(security, 5) },
|
|
),
|
|
finops: widget(
|
|
'finops',
|
|
'Token economy / FinOps',
|
|
provider.length > 0 || kpis.length > 0 ? 'ok' : 'no_data',
|
|
`$${sum(provider, 'cost_usd').toFixed(4)} provider cost, ${sum(provider, 'total_tokens')} tokens`,
|
|
{
|
|
provider_cost: sum(provider, 'cost_usd'),
|
|
provider_tokens: sum(provider, 'total_tokens'),
|
|
kpis: kpis.length,
|
|
kpis_met: kpisMet,
|
|
},
|
|
artifact(PATHS.providerUsage, 'provider-usage.jsonl'),
|
|
{ business_kpi: { status: kpi?.status ?? null, kpis_met: kpisMet, kpis: kpis.length } },
|
|
),
|
|
certification: widget(
|
|
'certification',
|
|
'Certified-run seal',
|
|
auditVerified ? 'ok' : 'warn',
|
|
auditVerified ? `Audit head present with ${audit.length} record(s)` : 'Audit chain head missing or empty',
|
|
{
|
|
audit_records: audit.length,
|
|
audit_head_present: Boolean(head),
|
|
last_decision: audit.length ? audit[audit.length - 1].decision ?? null : null,
|
|
},
|
|
artifact(PATHS.auditHead, 'audit-head.txt', auditVerified),
|
|
{ audit_head: head, recent_audit: recent(audit, 5) },
|
|
),
|
|
self_improve: widget(
|
|
'self_improve',
|
|
'Self-improve pipeline',
|
|
selfImproveRelated.length > 0 ? 'warn' : (existsSync(PATHS.selfImprove) ? 'ok' : 'no_data'),
|
|
selfImproveRelated.length ? `${selfImproveRelated.length} self-improve proposal(s) in inbox` : 'Core self-improve primitive is present; no proposal artifact queued',
|
|
{
|
|
inbox_related: selfImproveRelated.length,
|
|
primitive_present: existsSync(PATHS.selfImprove),
|
|
},
|
|
artifact(PATHS.selfImprove, 'self-improve.py'),
|
|
{ proposals: recent(selfImproveRelated, 5) },
|
|
),
|
|
chat_loop: buildChatLoopWidget(chatAudit, metrics),
|
|
};
|
|
|
|
const ticker = [
|
|
...recent(oversight, 10).map((e) => ({ at: e.at ?? e.created_at ?? null, kind: 'oversight', text: `${e.event ?? 'oversight'} ${e.proposal_id ?? ''}`.trim(), status: e.status ?? 'event' })),
|
|
...recent(incidents, 10).map((e) => ({ at: e.timestamp ?? null, kind: 'incident', text: `${e.event ?? 'incident'} ${e.scope ?? ''}`.trim(), status: e.severity ?? 'event' })),
|
|
...recent(audit, 10).map((e) => ({ at: e.timestamp ?? null, kind: 'audit', text: `${e.action ?? 'action'} ${e.decision ?? ''}`.trim(), status: e.decision ?? 'event' })),
|
|
...recent(chatAudit, 10).map((e) => ({
|
|
at: e.timestamp ?? null,
|
|
kind: 'chat_loop',
|
|
text: `${e.decision ?? 'turn'} ${e.chat_id ?? ''} ${e.loop_run?.run_id ? `loop:${e.loop_run.run_id}` : ''}`.trim(),
|
|
status: e.decision ?? 'event',
|
|
})),
|
|
].sort((a, b) => String(b.at ?? '').localeCompare(String(a.at ?? ''))).slice(0, 20);
|
|
|
|
return {
|
|
...this.freshness(),
|
|
generated_at: new Date().toISOString(),
|
|
widgets,
|
|
briefing: [
|
|
{ label: 'Maturity', value: widgets.maturity.summary, status: widgets.maturity.status, widget: 'maturity' },
|
|
{ label: 'Control', value: widgets.hitl.summary, status: widgets.hitl.status, widget: 'hitl' },
|
|
{ label: 'Safety', value: widgets.security.summary, status: widgets.security.status, widget: 'security' },
|
|
{ label: 'Economy', value: widgets.finops.summary, status: widgets.finops.status, widget: 'finops' },
|
|
{ label: 'Certification', value: widgets.certification.summary, status: widgets.certification.status, widget: 'certification' },
|
|
{ label: 'Chat/Loop', value: widgets.chat_loop.summary, status: widgets.chat_loop.status, widget: 'chat_loop' },
|
|
],
|
|
ticker,
|
|
};
|
|
}
|
|
}
|