feat: add command center chat loop widget

2026-07-08 22:58:02 +09:00
parent 36af576f15
commit a029a51e72
14 changed files with 225 additions and 38 deletions
@@ -48,6 +48,7 @@ export const PATHS = {
  benignFp: env('CASAN_CP_BENIGN_FP', 'docs/output/casan/benign-fp-report.json'),
  scoringReport: env('CASAN_CP_SCORING_REPORT', 'docs/output/casan/phase3-real-run-scoring.md'),
  approvalInbox: env('CASAN_CP_APPROVAL_INBOX', '.specify/level5/approval-inbox.json'),
+  chatAudit: env('CASAN_CP_CHAT_AUDIT', '.specify/logs/chat/chat-turns.jsonl'),
  delegationPolicy: env('CASAN_CP_DELEGATION_POLICY', 'packages/casan-harness/config/delegation-policy.yaml'),
  selfImprove: env('CASAN_CP_SELF_IMPROVE', 'packages/casan-harness/scripts/bash/self-improve.py'),
 };
@@ -2,12 +2,24 @@
 // packages/casan-harness/tests/generate-agentops-dashboard.py; all numbers come from real
 // on-disk feeds. Missing feeds degrade to zero/empty + a `stale` flag — never fabricated.
 import { existsSync, readFileSync, statSync } from 'node:fs';
+import { execFileSync } from 'node:child_process';
 import { join, relative } from 'node:path';
 import { Injectable } from '@nestjs/common';
 import { APP_ROOT, PATHS, isStale, metricsAgeSeconds, STALE_AFTER_S } from '../common/app-root.js';
 import { readJsonl, readJson, readHead, readTrace } from './telemetry.reader.js';

 type Row = Record<string, any>;
+type ChatReplayResult = {
+  ok: boolean;
+  decision: string;
+  records: number;
+  loop_replayed?: number;
+  diffs?: any[];
+  audit_path?: string;
+  error?: string;
+};
+
+const CHAT_REPLAY_CLI = join(APP_ROOT, 'packages', 'casan-harness', 'scripts', 'bash', 'chat-replay.py');
 const num = (v: any) => (typeof v === 'number' && isFinite(v) ? v : 0);
 const sum = (rows: Row[], k: string) => rows.reduce((a, r) => a + num(r[k]), 0);
 const count = (rows: Row[], pred: (r: Row) => boolean) => rows.reduce((a, r) => a + (pred(r) ? 1 : 0), 0);
@@ -53,6 +65,108 @@ function widget(
  return { id, title, status, summary, metrics, envelope, evidence };
 }

+function parseJson<T>(raw: string): T | null {
+  if (!raw) return null;
+  try {
+    return JSON.parse(raw) as T;
+  } catch {
+    return null;
+  }
+}
+
+function runChatReplay(chatAuditPath: string): ChatReplayResult {
+  if (!existsSync(chatAuditPath)) {
+    return { ok: true, decision: 'NO_DATA', records: 0, loop_replayed: 0, diffs: [], audit_path: chatAuditPath };
+  }
+  try {
+    const stdout = execFileSync('python3', [CHAT_REPLAY_CLI, 'replay'], {
+      cwd: APP_ROOT,
+      encoding: 'utf8',
+      stdio: ['ignore', 'pipe', 'pipe'],
+      env: { ...process.env, CASAN_CHAT_AUDIT_LOG: chatAuditPath },
+      timeout: 15000,
+    });
+    const parsed = parseJson<Record<string, any>>(stdout.trim());
+    return { ok: true, decision: String(parsed?.decision ?? 'UNKNOWN'), records: num(parsed?.records), ...parsed };
+  } catch (err: any) {
+    const stdout = String(err?.stdout ?? '').trim();
+    const stderr = String(err?.stderr ?? '').trim();
+    const parsed = parseJson<Record<string, any>>(stdout);
+    if (parsed) {
+      return { ok: false, decision: String(parsed.decision ?? 'ERROR'), records: num(parsed.records), ...parsed };
+    }
+    return { ok: false, decision: 'ERROR', records: 0, loop_replayed: 0, diffs: [], audit_path: chatAuditPath, error: stderr || stdout || 'CHAT_REPLAY_FAILED' };
+  }
+}
+
+function isChatMetric(row: Row) {
+  const agent = String(row.agent ?? '');
+  const step = String(row.step ?? '');
+  return agent.startsWith('chat.') || step === 'ask-casan-readonly' || step.startsWith('operator:');
+}
+
+export function buildChatLoopWidget(
+  chatAudit: Row[],
+  metrics: Row[],
+  chatAuditPath = PATHS.chatAudit,
+  replay: ChatReplayResult = runChatReplay(chatAuditPath),
+) {
+  const latest = chatAudit[chatAudit.length - 1] ?? null;
+  const chatMetrics = metrics.filter(isChatMetric);
+  const tokenBudget = Math.max(Number(process.env.CASAN_CHAT_TOKEN_BUDGET ?? 100000) || 100000, 1);
+  const totalTokens = sum(chatMetrics, 'total_tokens');
+  const status = replay.decision === 'BREAK'
+    ? 'fail'
+    : replay.decision === 'DRIFT' || replay.decision === 'ERROR'
+      ? 'warn'
+      : chatAudit.length > 0
+        ? 'ok'
+        : 'no_data';
+  const loopRuns = count(chatAudit, (r) => Boolean(r.loop_run?.run_id));
+  const verifiedReplay = replay.decision === 'MATCH' || replay.decision === 'NO_DATA';
+
+  return widget(
+    'chat_loop',
+    'Chat / Loop console',
+    status,
+    `${chatAudit.length} turn(s), ${loopRuns} loop-run(s), replay ${replay.decision}`,
+    {
+      turns: chatAudit.length,
+      answered: count(chatAudit, (r) => r.decision === 'ANSWERED'),
+      action_completed: count(chatAudit, (r) => r.decision === 'ACTION_COMPLETED'),
+      loop_runs: loopRuns,
+      replay_records: replay.records,
+      replay_loop_replayed: num(replay.loop_replayed),
+      budget_used_pct: pct(totalTokens, tokenBudget),
+    },
+    artifact(chatAuditPath, 'chat-turns.jsonl + chat-replay.py', verifiedReplay),
+    {
+      replay,
+      budget: { total_tokens: totalTokens, token_budget: tokenBudget, used_pct: pct(totalTokens, tokenBudget) },
+      latest_turn: latest ? {
+        seq: latest.seq ?? null,
+        timestamp: latest.timestamp ?? null,
+        chat_id: latest.chat_id ?? null,
+        turn_id: latest.turn_id ?? null,
+        tenant_id: latest.tenant_id ?? null,
+        mode: latest.mode ?? null,
+        decision: latest.decision ?? null,
+        loop_run: latest.loop_run ?? null,
+        record_hash: latest.record_hash ?? null,
+      } : null,
+      recent_turns: recent(chatAudit, 5).map((r) => ({
+        seq: r.seq ?? null,
+        timestamp: r.timestamp ?? null,
+        chat_id: r.chat_id ?? null,
+        turn_id: r.turn_id ?? null,
+        decision: r.decision ?? null,
+        mode: r.mode ?? null,
+        loop_run_id: r.loop_run?.run_id ?? null,
+      })),
+    },
+  );
+}
+
@Injectable()
 export class TelemetryService {
  private freshness() {
@@ -184,6 +298,7 @@ export class TelemetryService {
    const security = readJsonl(PATHS.security);
    const incidents = readJsonl(PATHS.incidents);
    const actionGate = readJsonl(PATHS.actionGate);
+    const chatAudit = readJsonl(PATHS.chatAudit);
    const traceability = readJson(PATHS.traceability) as any;
    const kpi = readJson(PATHS.businessKpi) as any;
    const drift = readJson(PATHS.drift) as any;
@@ -313,12 +428,19 @@ export class TelemetryService {
        artifact(PATHS.selfImprove, 'self-improve.py'),
        { proposals: recent(selfImproveRelated, 5) },
      ),
+      chat_loop: buildChatLoopWidget(chatAudit, metrics),
    };

    const ticker = [
      ...recent(oversight, 10).map((e) => ({ at: e.at ?? e.created_at ?? null, kind: 'oversight', text: `${e.event ?? 'oversight'} ${e.proposal_id ?? ''}`.trim(), status: e.status ?? 'event' })),
      ...recent(incidents, 10).map((e) => ({ at: e.timestamp ?? null, kind: 'incident', text: `${e.event ?? 'incident'} ${e.scope ?? ''}`.trim(), status: e.severity ?? 'event' })),
      ...recent(audit, 10).map((e) => ({ at: e.timestamp ?? null, kind: 'audit', text: `${e.action ?? 'action'} ${e.decision ?? ''}`.trim(), status: e.decision ?? 'event' })),
+      ...recent(chatAudit, 10).map((e) => ({
+        at: e.timestamp ?? null,
+        kind: 'chat_loop',
+        text: `${e.decision ?? 'turn'} ${e.chat_id ?? ''} ${e.loop_run?.run_id ? `loop:${e.loop_run.run_id}` : ''}`.trim(),
+        status: e.decision ?? 'event',
+      })),
    ].sort((a, b) => String(b.at ?? '').localeCompare(String(a.at ?? ''))).slice(0, 20);

    return {
@@ -331,6 +453,7 @@ export class TelemetryService {
        { label: 'Safety', value: widgets.security.summary, status: widgets.security.status, widget: 'security' },
        { label: 'Economy', value: widgets.finops.summary, status: widgets.finops.status, widget: 'finops' },
        { label: 'Certification', value: widgets.certification.summary, status: widgets.certification.status, widget: 'certification' },
+        { label: 'Chat/Loop', value: widgets.chat_loop.summary, status: widgets.chat_loop.status, widget: 'chat_loop' },
      ],
      ticker,
    };
@@ -5,7 +5,7 @@ import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { readJsonl, readJson, readHead } from '../src/telemetry/telemetry.reader.js';
 import { APP_ROOT, PATHS, isStale, metricsAgeSeconds } from '../src/common/app-root.js';
-import { TelemetryService } from '../src/telemetry/telemetry.service.js';
+import { TelemetryService, buildChatLoopWidget } from '../src/telemetry/telemetry.service.js';

 test('readJsonl parses valid lines and skips malformed', () => {
  const d = mkdtempSync(join(tmpdir(), 'cp-'));
@@ -53,6 +53,7 @@ test('commandCenter() returns evidence-backed widget contract', () => {
  const ids = Object.keys(command.widgets);
  assert.deepEqual(ids.sort(), [
    'certification',
+    'chat_loop',
    'finops',
    'hitl',
    'kill_switch',
@@ -71,6 +72,53 @@ test('commandCenter() returns evidence-backed widget contract', () => {
  assert.ok(Array.isArray(command.ticker));
 });

+test('buildChatLoopWidget summarizes chat replay fixture and evidence', () => {
+  const d = mkdtempSync(join(tmpdir(), 'cp-chat-'));
+  const auditPath = join(d, 'chat-turns.jsonl');
+  writeFileSync(auditPath, '{"seq":1}\n{"seq":2}\n');
+  const prevBudget = process.env.CASAN_CHAT_TOKEN_BUDGET;
+  process.env.CASAN_CHAT_TOKEN_BUDGET = '200';
+  try {
+    const widget = buildChatLoopWidget(
+      [
+        { seq: 1, timestamp: '2026-07-08T00:00:00Z', chat_id: 'chat1', turn_id: 't1', mode: 'READ_ONLY', decision: 'ANSWERED' },
+        {
+          seq: 2,
+          timestamp: '2026-07-08T00:01:00Z',
+          chat_id: 'chat1',
+          turn_id: 't2',
+          mode: 'OPERATOR',
+          decision: 'ACTION_COMPLETED',
+          loop_run: { run_id: 'loop1', decision: 'CERTIFIED' },
+          record_hash: 'abc123',
+        },
+      ],
+      [
+        { agent: 'chat.ask-casan', step: 'ask-casan-readonly', total_tokens: 40 },
+        { agent: 'chat.operator', step: 'operator:generate_report', total_tokens: 60 },
+        { agent: 'planner', step: 'unrelated', total_tokens: 900 },
+      ],
+      auditPath,
+      { ok: true, decision: 'MATCH', records: 2, loop_replayed: 1, diffs: [], audit_path: auditPath },
+    ) as any;
+
+    assert.equal(widget.id, 'chat_loop');
+    assert.equal(widget.status, 'ok');
+    assert.equal(widget.metrics.turns, 2);
+    assert.equal(widget.metrics.answered, 1);
+    assert.equal(widget.metrics.action_completed, 1);
+    assert.equal(widget.metrics.loop_runs, 1);
+    assert.equal(widget.metrics.replay_loop_replayed, 1);
+    assert.equal(widget.metrics.budget_used_pct, 50);
+    assert.equal(widget.envelope.verified, true);
+    assert.equal(widget.evidence.latest_turn.loop_run.run_id, 'loop1');
+    assert.equal(widget.evidence.replay.decision, 'MATCH');
+  } finally {
+    if (prevBudget === undefined) delete process.env.CASAN_CHAT_TOKEN_BUDGET;
+    else process.env.CASAN_CHAT_TOKEN_BUDGET = prevBudget;
+  }
+});
+
 test('freshness helpers behave (age is number|null, isStale is boolean)', () => {
  const age = metricsAgeSeconds();
  assert.ok(age === null || typeof age === 'number');