update report h6

This commit is contained in:
thanhnv
2026-07-20 23:47:09 +07:00
parent f462079435
commit 4b6819f578
38 changed files with 1244 additions and 62 deletions
+16 -1
View File
@@ -23,6 +23,17 @@ Open http://127.0.0.1:5174 — panels show REAL metrics from `.specify/logs/**`.
`incidents` · `tools` · `traceability` · `drift` · `cost` · `GET /healthz` (200 fresh /
503 stale — fail-loud, mirrors `dashboard-server.py`).
Harness reports:
- `GET /api/v1/reports` — versioned H1–H7 report catalog and availability.
- `GET /api/v1/reports/h6?project=<id>&from=<ISO>&to=<ISO>&run=<id>` — H6 AgentOps
report from the same runtime metrics, provider usage and alert files as the Control Panel.
- `GET /api/v1/reports/h6/export?format=html|json&...` — standalone, print-ready HTML
or machine-readable JSON export. Both formats serialize the same filtered report object;
no maturity score is hard-coded.
- `/reports/h6` — UI report view with project, time-range and run filters plus source-level
freshness and data-quality warnings.
Metrics export: `GET /api/v1/metrics` provides Prometheus text exposition for
aggregate freshness, run/failure/cost/token and H4/H5/action/incident counters.
It intentionally contains no tenant, actor, trace, prompt or Evidence Pack
@@ -121,7 +132,11 @@ FinOps/SLO:
Data sources + aggregation mirror `packages/casan-harness/tests/generate-agentops-dashboard.py`.
App root + telemetry paths resolve via the same marker walk-up as `casan-paths.sh`
(`.specify` or `packages/casan-harness`) and honor `CASAN_DASHBOARD_*` env overrides.
(`.specify` or `packages/casan-harness`). The canonical source variables are
`CASAN_TELEMETRY_METRICS_LOG`, `CASAN_TELEMETRY_PROVIDER_LOG`, and
`CASAN_TELEMETRY_ALERTS_LOG`; legacy dashboard/control-panel names remain accepted as
compatibility aliases. Freshness is calculated independently from each file's mtime using
`CASAN_DASHBOARD_STALE_S` (default `3600`).
## Security posture (MVP)
Binds `127.0.0.1` by default. Refuses a non-loopback bind under `CASAN_PROFILE=prod` /
@@ -9,9 +9,10 @@ import { ProviderAuthModule } from './provider-auth/provider-auth.module.js';
import { GoalsModule } from './goals/goals.module.js';
import { EvidenceModule } from './evidence/evidence.module.js';
import { SessionController } from './session/session.controller.js';
import { ReportsModule } from './reports/reports.module.js';
@Module({
imports: [TelemetryModule, SettingsModule, KillSwitchModule, ApprovalsModule, ChatModule, ProviderAuthModule, GoalsModule, EvidenceModule],
imports: [TelemetryModule, SettingsModule, KillSwitchModule, ApprovalsModule, ChatModule, ProviderAuthModule, GoalsModule, EvidenceModule, ReportsModule],
controllers: [HealthController, SessionController],
})
export class AppModule {}
@@ -3,8 +3,8 @@
// Mirrors packages/casan-harness/scripts/bash/casan-paths.sh: walk UP from a start dir
// for a marker (`.specify` state dir OR `packages/casan-harness`), so the API reads the
// real harness telemetry whether launched from the repo, a workspace subdir, or a bundle.
// Every path is overridable by the same CASAN_DASHBOARD_* env vars the legacy
// dashboard-server.py honors, so console + dashboard read identical sources.
// Canonical CASAN_TELEMETRY_* variables are shared with harness writers; legacy
// dashboard/control-panel names remain accepted so upgrades do not split sources.
import { existsSync, statSync } from 'node:fs';
import { dirname, join, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
@@ -27,35 +27,74 @@ export const APP_ROOT = process.env.CASAN_APP_ROOT
? resolve(process.env.CASAN_APP_ROOT)
: findAppRoot(HERE);
const env = (k: string, fallback: string) => (process.env[k] ? resolve(process.env[k]!) : join(APP_ROOT, fallback));
const env = (keys: string[], fallback: string) => {
const configured = keys.map((key) => process.env[key]).find((value): value is string => Boolean(value));
return configured ? resolve(configured) : join(APP_ROOT, fallback);
};
// Runtime telemetry (regenerated) + generated reports. Names match the harness layout.
export const PATHS = {
metrics: env('CASAN_DASHBOARD_METRICS', '.specify/logs/cost/metrics.jsonl'),
providerUsage: env('CASAN_CP_PROVIDER_USAGE', '.specify/logs/level5/provider-usage.jsonl'),
audit: env('CASAN_CP_AUDIT', '.specify/logs/audit/audit.jsonl'),
auditHead: env('CASAN_CP_AUDIT_HEAD', '.specify/logs/audit/audit-head.txt'),
security: env('CASAN_CP_SECURITY', '.specify/logs/audit/security.jsonl'),
toolRegistry: env('CASAN_CP_TOOL_REGISTRY', '.specify/logs/level5/tool-registry.jsonl'),
actionGate: env('CASAN_CP_ACTION_GATE', '.specify/logs/level5/action-gate.jsonl'),
fallback: env('CASAN_CP_FALLBACK', '.specify/logs/level5/fallback.jsonl'),
incidents: env('CASAN_CP_INCIDENTS', '.specify/logs/level5/incidents.jsonl'),
alerts: env('CASAN_DASHBOARD_ALERTS', '.specify/agentops/alerts.log'),
traceDir: env('CASAN_CP_TRACE_DIR', '.specify/logs/trace'),
traceEventDir: env('CASAN_CP_TRACE_EVENT_DIR', '.specify/logs/trace-events'),
traceability: env('CASAN_CP_TRACEABILITY', 'docs/output/casan/traceability-matrix.json'),
drift: env('CASAN_CP_DRIFT', 'docs/output/casan/level5-evidence/09-drift-report.json'),
businessKpi: env('CASAN_CP_KPI', 'docs/output/casan/level5-evidence/14-business-kpi-report.json'),
benignFp: env('CASAN_CP_BENIGN_FP', 'docs/output/casan/benign-fp-report.json'),
scoringReport: env('CASAN_CP_SCORING_REPORT', 'docs/output/casan/phase3-real-run-scoring.md'),
approvalInbox: env('CASAN_CP_APPROVAL_INBOX', '.specify/level5/approval-inbox.json'),
chatAudit: env('CASAN_CP_CHAT_AUDIT', '.specify/logs/chat/chat-turns.jsonl'),
delegationPolicy: env('CASAN_CP_DELEGATION_POLICY', 'packages/casan-harness/config/delegation-policy.yaml'),
selfImprove: env('CASAN_CP_SELF_IMPROVE', 'packages/casan-harness/scripts/bash/self-improve.py'),
evidencePacks: env('CASAN_CP_EVIDENCE_PACKS', 'docs/output/casan/evidence-packs'),
metrics: env(['CASAN_TELEMETRY_METRICS_LOG', 'CASAN_DASHBOARD_METRICS'], '.specify/logs/cost/metrics.jsonl'),
providerUsage: env(['CASAN_TELEMETRY_PROVIDER_LOG', 'CASAN_CP_PROVIDER_USAGE'], '.specify/logs/level5/provider-usage.jsonl'),
alerts: env(['CASAN_TELEMETRY_ALERTS_LOG', 'CASAN_DASHBOARD_ALERTS'], '.specify/agentops/alerts.log'),
audit: env(['CASAN_CP_AUDIT'], '.specify/logs/audit/audit.jsonl'),
auditHead: env(['CASAN_CP_AUDIT_HEAD'], '.specify/logs/audit/audit-head.txt'),
security: env(['CASAN_CP_SECURITY'], '.specify/logs/audit/security.jsonl'),
toolRegistry: env(['CASAN_CP_TOOL_REGISTRY'], '.specify/logs/level5/tool-registry.jsonl'),
actionGate: env(['CASAN_CP_ACTION_GATE'], '.specify/logs/level5/action-gate.jsonl'),
fallback: env(['CASAN_CP_FALLBACK'], '.specify/logs/level5/fallback.jsonl'),
incidents: env(['CASAN_CP_INCIDENTS'], '.specify/logs/level5/incidents.jsonl'),
traceDir: env(['CASAN_CP_TRACE_DIR'], '.specify/logs/trace'),
traceEventDir: env(['CASAN_CP_TRACE_EVENT_DIR'], '.specify/logs/trace-events'),
traceability: env(['CASAN_CP_TRACEABILITY'], 'docs/output/casan/traceability-matrix.json'),
drift: env(['CASAN_CP_DRIFT'], 'docs/output/casan/level5-evidence/09-drift-report.json'),
businessKpi: env(['CASAN_CP_KPI'], 'docs/output/casan/level5-evidence/14-business-kpi-report.json'),
benignFp: env(['CASAN_CP_BENIGN_FP'], 'docs/output/casan/benign-fp-report.json'),
scoringReport: env(['CASAN_CP_SCORING_REPORT'], 'docs/output/casan/phase3-real-run-scoring.md'),
approvalInbox: env(['CASAN_CP_APPROVAL_INBOX'], '.specify/level5/approval-inbox.json'),
chatAudit: env(['CASAN_CP_CHAT_AUDIT'], '.specify/logs/chat/chat-turns.jsonl'),
delegationPolicy: env(['CASAN_CP_DELEGATION_POLICY'], 'packages/casan-harness/config/delegation-policy.yaml'),
selfImprove: env(['CASAN_CP_SELF_IMPROVE'], 'packages/casan-harness/scripts/bash/self-improve.py'),
evidencePacks: env(['CASAN_CP_EVIDENCE_PACKS'], 'docs/output/casan/evidence-packs'),
};
export const STALE_AFTER_S = Number(process.env.CASAN_DASHBOARD_STALE_S ?? 3600);
const configuredStaleAfter = Number(process.env.CASAN_DASHBOARD_STALE_S ?? 3600);
export const STALE_AFTER_S = Number.isFinite(configuredStaleAfter) && configuredStaleAfter >= 0
? configuredStaleAfter
: 3600;
export interface SourceFreshness {
source: 'metrics' | 'provider' | 'alerts';
path: string;
present: boolean;
updated_at: string | null;
age_s: number | null;
stale: boolean;
required: boolean;
}
export function sourceFreshness(
source: SourceFreshness['source'],
path: string,
required: boolean,
nowMs = Date.now(),
): SourceFreshness {
try {
const mtime = statSync(path).mtime;
const age = Math.max(0, Math.floor((nowMs - mtime.getTime()) / 1000));
return { source, path, present: true, updated_at: mtime.toISOString(), age_s: age, stale: age > STALE_AFTER_S, required };
} catch {
return { source, path, present: false, updated_at: null, age_s: null, stale: required, required };
}
}
export function telemetryFreshness(nowMs = Date.now()): SourceFreshness[] {
return [
sourceFreshness('metrics', PATHS.metrics, true, nowMs),
sourceFreshness('provider', PATHS.providerUsage, false, nowMs),
sourceFreshness('alerts', PATHS.alerts, false, nowMs),
];
}
// Freshness of the primary metrics feed (mtime), used by /healthz + `stale` flags.
export function metricsAgeSeconds(): number | null {
@@ -0,0 +1,61 @@
import type { H6Breakdown, H6Report } from './h6-report.js';
const escapeHtml = (value: unknown): string => String(value ?? '')
.replaceAll('&', '&amp;')
.replaceAll('<', '&lt;')
.replaceAll('>', '&gt;')
.replaceAll('"', '&quot;')
.replaceAll("'", '&#039;');
const number = (value: number): string => new Intl.NumberFormat('en-US').format(value);
const money = (value: number): string => `$${value.toFixed(6)}`;
function breakdownRows(rows: H6Breakdown[]): string {
if (rows.length === 0) return '<tr><td colspan="6" class="empty">No records in the selected scope.</td></tr>';
return rows.map((row) => `<tr>
<td>${escapeHtml(row.key)}</td><td>${number(row.runs)}</td><td>${number(row.failures)}</td>
<td>${number(row.latency_avg_ms)} ms</td><td>${number(row.tokens)}</td><td>${money(row.cost_usd)}</td>
</tr>`).join('');
}
export function renderH6ReportHtml(report: H6Report): string {
const verdictClass = report.verdict === 'pass' ? 'pass' : report.verdict === 'fail' ? 'fail' : report.verdict === 'attention' ? 'attention' : 'neutral';
const scope = [
report.scope.project ? `Project: ${report.scope.project}` : 'All projects',
report.scope.run ? `Run: ${report.scope.run}` : 'All runs',
report.scope.from ? `From: ${report.scope.from}` : null,
report.scope.to ? `To: ${report.scope.to}` : null,
].filter((value): value is string => Boolean(value));
const findings = report.findings.length
? report.findings.map((finding) => `<li class="${finding.severity}"><strong>${escapeHtml(finding.code)}</strong><span>${escapeHtml(finding.message)}</span></li>`).join('')
: '<li class="info"><strong>NO_FINDINGS</strong><span>No threshold breach was detected in this scope.</span></li>';
const sourceRows = report.evidence_sources.map((source) => `<tr>
<td>${escapeHtml(source.source)}</td><td>${source.present ? 'present' : 'missing'}</td><td>${source.stale ? 'stale' : 'fresh'}</td>
<td>${source.age_s === null ? '—' : `${number(source.age_s)} s`}</td><td>${number(source.records)}</td><td><code>${escapeHtml(source.path)}</code></td>
</tr>`).join('');
const warnings = report.data_quality.warnings.length
? `<ul>${report.data_quality.warnings.map((warning) => `<li>${escapeHtml(warning)}</li>`).join('')}</ul>`
: '<p>No data-quality warning.</p>';
return `<!doctype html>
<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
<title>${escapeHtml(report.title)} · ${escapeHtml(report.report_id)}</title>
<style>
:root{color-scheme:light;--ink:#172033;--muted:#64748b;--line:#dbe2ec;--paper:#fff;--wash:#f4f6fa;--indigo:#4f46e5;--teal:#0f766e}
*{box-sizing:border-box}body{margin:0;background:var(--wash);color:var(--ink);font-family:Inter,ui-sans-serif,system-ui,-apple-system,"Segoe UI",sans-serif;line-height:1.45}
main{max-width:1180px;margin:0 auto;padding:42px 28px 72px}.hero{position:relative;overflow:hidden;border:1px solid #cbd5e1;border-radius:20px;background:#111827;color:#f8fafc;padding:30px;box-shadow:0 24px 50px rgba(15,23,42,.14)}
.hero:after{content:"";position:absolute;right:-90px;top:-110px;width:300px;height:300px;border-radius:999px;background:radial-gradient(circle,#6366f1 0,transparent 68%);opacity:.5}.eyebrow{font-size:11px;text-transform:uppercase;letter-spacing:.18em;color:#a5b4fc;font-weight:800}h1{font-size:34px;line-height:1.1;margin:8px 0}.scope{display:flex;gap:8px;flex-wrap:wrap;color:#cbd5e1;font-size:12px}.scope span{border:1px solid #334155;border-radius:999px;padding:6px 10px;background:#0f172a}.verdict{position:relative;z-index:1;display:inline-flex;margin-top:22px;border-radius:999px;padding:9px 14px;font-size:12px;text-transform:uppercase;letter-spacing:.12em;font-weight:900}.verdict.pass{background:#d1fae5;color:#065f46}.verdict.fail{background:#fee2e2;color:#991b1b}.verdict.attention{background:#fef3c7;color:#92400e}.verdict.neutral{background:#e2e8f0;color:#334155}
.meta{margin-top:10px;color:#94a3b8;font-size:12px}.grid{display:grid;grid-template-columns:repeat(4,minmax(0,1fr));gap:12px;margin-top:18px}.card,.section{background:var(--paper);border:1px solid var(--line);border-radius:16px;box-shadow:0 5px 14px rgba(15,23,42,.04)}.card{padding:17px}.card span{display:block;color:var(--muted);font-size:11px;text-transform:uppercase;letter-spacing:.1em;font-weight:700}.card strong{display:block;margin-top:6px;font-size:26px}.section{padding:22px;margin-top:18px}h2{font-size:16px;margin:0 0 14px}table{border-collapse:collapse;width:100%;font-size:12px}th{text-align:left;color:var(--muted);font-size:10px;text-transform:uppercase;letter-spacing:.1em;padding:9px;border-bottom:1px solid var(--line)}td{padding:10px 9px;border-bottom:1px solid #edf1f5;vertical-align:top}.empty{color:var(--muted);text-align:center;padding:24px}code{font-size:11px;word-break:break-all}.findings{list-style:none;padding:0;margin:0;display:grid;gap:9px}.findings li{display:grid;grid-template-columns:190px 1fr;gap:12px;border-left:4px solid #94a3b8;background:#f8fafc;padding:11px 13px;border-radius:8px}.findings li.warning{border-color:#f59e0b}.findings li.critical{border-color:#ef4444}.findings span{color:#475569}.quality{border-left:4px solid var(--indigo);background:#eef2ff;padding:14px 16px;border-radius:10px;font-size:13px}.quality p,.quality ul{margin:0}.quality ul{padding-left:18px}.footer{margin-top:24px;color:var(--muted);font-size:11px;text-align:center}
@media(max-width:800px){main{padding:20px 14px 50px}.grid{grid-template-columns:repeat(2,minmax(0,1fr))}.hero{padding:22px}h1{font-size:28px}.table-wrap{overflow:auto}.findings li{grid-template-columns:1fr}}
@media print{body{background:#fff}main{max-width:none;padding:0}.hero,.card,.section{box-shadow:none}.hero{background:#111827!important;-webkit-print-color-adjust:exact;print-color-adjust:exact}.section{break-inside:avoid}.footer{margin-top:10px}}
</style></head><body><main>
<section class="hero"><div class="eyebrow">CASAN assurance dossier · contract v${report.schema_version}</div><h1>${escapeHtml(report.title)}</h1><div class="scope">${scope.map((item) => `<span>${escapeHtml(item)}</span>`).join('')}</div><div class="verdict ${verdictClass}">${escapeHtml(report.verdict)}</div><div class="meta">Generated ${escapeHtml(report.generated_at)} · ${escapeHtml(report.report_id)} · Freshness ${escapeHtml(report.freshness.status)}</div></section>
<section class="grid"><div class="card"><span>Governed runs</span><strong>${number(report.summary.runs)}</strong></div><div class="card"><span>Failure rate</span><strong>${report.summary.failure_rate_pct}%</strong></div><div class="card"><span>P95 latency</span><strong>${number(report.summary.latency_ms.p95)} ms</strong></div><div class="card"><span>Provider tokens</span><strong>${number(report.summary.tokens.provider_total)}</strong></div><div class="card"><span>Actual provider cost</span><strong>${money(report.summary.cost_usd.provider_actual)}</strong></div><div class="card"><span>Estimated cost</span><strong>${money(report.summary.cost_usd.estimated)}</strong></div><div class="card"><span>Alerts</span><strong>${number(report.summary.alerts)}</strong></div><div class="card"><span>Retries</span><strong>${number(report.summary.retries)}</strong></div></section>
<section class="section"><h2>Verdict findings</h2><ul class="findings">${findings}</ul></section>
<section class="section"><h2>Evidence freshness</h2><div class="table-wrap"><table><thead><tr><th>Source</th><th>Presence</th><th>State</th><th>Age</th><th>Records</th><th>Path</th></tr></thead><tbody>${sourceRows}</tbody></table></div></section>
<section class="section"><h2>Step breakdown</h2><div class="table-wrap"><table><thead><tr><th>Step</th><th>Runs</th><th>Failures</th><th>Avg latency</th><th>Tokens</th><th>Cost</th></tr></thead><tbody>${breakdownRows(report.details.by_step)}</tbody></table></div></section>
<section class="section"><h2>Provider/model breakdown</h2><div class="table-wrap"><table><thead><tr><th>Provider · model</th><th>Calls</th><th>Failures</th><th>Avg latency</th><th>Tokens</th><th>Cost</th></tr></thead><tbody>${breakdownRows(report.details.by_provider)}</tbody></table></div></section>
<section class="section"><h2>Data quality · ${escapeHtml(report.data_quality.status)}</h2><div class="quality">${warnings}</div></section>
<div class="footer">Generated from CASAN runtime evidence. No maturity score or telemetry value is hard-coded in this document.</div>
</main></body></html>`;
}
@@ -0,0 +1,307 @@
import { BadRequestException } from '@nestjs/common';
import { basename, relative } from 'node:path';
import { APP_ROOT, PATHS, STALE_AFTER_S, telemetryFreshness, type SourceFreshness } from '../common/app-root.js';
import type { HarnessReport, HarnessReportEvidenceSource, HarnessReportFinding, HarnessReportScope } from './report.contract.js';
type TelemetryRow = Record<string, unknown>;
export interface H6ReportQuery {
project: string | null;
from: string | null;
to: string | null;
run: string | null;
limit: number;
}
export interface H6ReportQueryParams {
project?: string;
from?: string;
to?: string;
run?: string;
limit?: string;
}
export interface H6ReportInput {
metrics: TelemetryRow[];
provider: TelemetryRow[];
alerts: TelemetryRow[];
sourceFreshness: SourceFreshness[];
now: Date;
}
export interface H6ReportSummary {
runs: number;
success: number;
failed: number;
degraded: number;
failure_rate_pct: number;
retries: number;
latency_ms: { average: number; p50: number; p95: number; p99: number; max: number };
tokens: { input: number; output: number; total: number; provider_total: number };
cost_usd: { provider_actual: number; estimated: number };
provider_calls: number;
alerts: number;
}
export interface H6Breakdown {
key: string;
runs: number;
failures: number;
latency_avg_ms: number;
tokens: number;
cost_usd: number;
}
export interface H6ReportDetails {
by_status: Array<{ status: string; count: number }>;
by_step: H6Breakdown[];
by_provider: H6Breakdown[];
by_cost_source: Array<{ source: string; records: number; cost_usd: number }>;
by_alert: Array<{ alert: string; count: number }>;
recent_runs: TelemetryRow[];
recent_alerts: TelemetryRow[];
}
export type H6Report = HarnessReport<H6ReportSummary, H6ReportDetails>;
const safeFilter = /^[a-zA-Z0-9][a-zA-Z0-9._:-]{0,127}$/;
const numberValue = (value: unknown): number => typeof value === 'number' && Number.isFinite(value) ? value : 0;
const stringValue = (value: unknown, fallback: string): string => typeof value === 'string' && value.trim() ? value.trim() : fallback;
const round = (value: number, digits = 2): number => Number(value.toFixed(digits));
const configuredThreshold = (name: string, fallback: number): number => {
const parsed = Number(process.env[name] ?? fallback);
return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback;
};
const rowProject = (row: TelemetryRow): string => stringValue(row.project ?? row.project_id, 'default');
const rowRun = (row: TelemetryRow): string => stringValue(row.run_id ?? row.trace_id, 'unattributed');
const rowTime = (row: TelemetryRow): number | null => {
const timestamp = typeof row.timestamp === 'string' ? Date.parse(row.timestamp) : Number.NaN;
return Number.isFinite(timestamp) ? timestamp : null;
};
function parseDate(value: string | undefined, label: 'from' | 'to'): string | null {
if (!value) return null;
const dateOnly = /^\d{4}-\d{2}-\d{2}$/.test(value);
const normalized = dateOnly ? `${value}T${label === 'from' ? '00:00:00.000' : '23:59:59.999'}Z` : value;
const parsed = Date.parse(normalized);
if (!Number.isFinite(parsed)) throw new BadRequestException(`H6_REPORT_INVALID_${label.toUpperCase()}`);
return new Date(parsed).toISOString();
}
function parseFilter(value: string | undefined, label: 'project' | 'run'): string | null {
if (!value) return null;
if (!safeFilter.test(value)) throw new BadRequestException(`H6_REPORT_INVALID_${label.toUpperCase()}`);
return value;
}
export function parseH6ReportQuery(raw: H6ReportQueryParams): H6ReportQuery {
const from = parseDate(raw.from, 'from');
const to = parseDate(raw.to, 'to');
if (from && to && Date.parse(from) > Date.parse(to)) throw new BadRequestException('H6_REPORT_INVALID_TIME_RANGE');
const parsedLimit = Number(raw.limit ?? 50);
if (!Number.isInteger(parsedLimit) || parsedLimit < 1 || parsedLimit > 200) {
throw new BadRequestException('H6_REPORT_INVALID_LIMIT');
}
return {
project: parseFilter(raw.project, 'project'),
from,
to,
run: parseFilter(raw.run, 'run'),
limit: parsedLimit,
};
}
function filtered(rows: TelemetryRow[], query: H6ReportQuery): TelemetryRow[] {
const fromMs = query.from ? Date.parse(query.from) : null;
const toMs = query.to ? Date.parse(query.to) : null;
return rows.filter((row) => {
if (query.project && rowProject(row) !== query.project) return false;
if (query.run && rowRun(row) !== query.run) return false;
const timestamp = rowTime(row);
if (fromMs !== null && (timestamp === null || timestamp < fromMs)) return false;
if (toMs !== null && (timestamp === null || timestamp > toMs)) return false;
return true;
});
}
function percentile(values: number[], percentileValue: number): number {
if (values.length === 0) return 0;
const sorted = [...values].sort((left, right) => left - right);
const index = Math.max(0, Math.ceil((percentileValue / 100) * sorted.length) - 1);
return sorted[index];
}
function sum(rows: TelemetryRow[], key: string): number {
return rows.reduce((total, row) => total + numberValue(row[key]), 0);
}
function grouped(rows: TelemetryRow[], keyOf: (row: TelemetryRow) => string): H6Breakdown[] {
const groups = new Map<string, TelemetryRow[]>();
for (const row of rows) {
const key = keyOf(row);
groups.set(key, [...(groups.get(key) ?? []), row]);
}
return [...groups.entries()].map(([key, records]) => {
const latencies = records.map((row) => numberValue(row.latency_ms)).filter((value) => value > 0);
return {
key,
runs: records.length,
failures: records.filter((row) => stringValue(row.status, 'unknown') === 'failed').length,
latency_avg_ms: latencies.length ? Math.round(latencies.reduce((total, value) => total + value, 0) / latencies.length) : 0,
tokens: sum(records, 'total_tokens'),
cost_usd: round(sum(records, records.some((row) => row.cost_usd !== undefined) ? 'cost_usd' : 'cost_estimate'), 6),
};
}).sort((left, right) => right.runs - left.runs || left.key.localeCompare(right.key));
}
function counted(rows: TelemetryRow[], keyOf: (row: TelemetryRow) => string): Array<{ key: string; count: number }> {
const counts = new Map<string, number>();
for (const row of rows) {
const key = keyOf(row);
counts.set(key, (counts.get(key) ?? 0) + 1);
}
return [...counts.entries()].map(([key, countValue]) => ({ key, count: countValue }))
.sort((left, right) => right.count - left.count || left.key.localeCompare(right.key));
}
function alertTypes(row: TelemetryRow): string[] {
const structured = row.body && typeof row.body === 'object' ? (row.body as Record<string, unknown>)['alert.type'] : undefined;
if (typeof structured === 'string') return [structured];
if (Array.isArray(row.alerts)) return row.alerts.filter((value): value is string => typeof value === 'string');
return [];
}
function evidenceSources(input: H6ReportInput): HarnessReportEvidenceSource[] {
const counts: Record<SourceFreshness['source'], number> = {
metrics: input.metrics.length,
provider: input.provider.length,
alerts: input.alerts.length,
};
return input.sourceFreshness.map((source) => {
const appRelativePath = relative(APP_ROOT, source.path) || '.';
return {
...source,
path: appRelativePath.startsWith('..') ? `[external]/${basename(source.path)}` : appRelativePath,
records: counts[source.source],
};
});
}
export function buildH6Report(input: H6ReportInput, query: H6ReportQuery): H6Report {
const metrics = filtered(input.metrics, query);
const provider = filtered(input.provider, query);
const alerts = filtered(input.alerts, query);
const latencies = metrics.map((row) => numberValue(row.latency_ms)).filter((value) => value > 0);
const failed = metrics.filter((row) => stringValue(row.status, 'unknown') === 'failed').length;
const degraded = metrics.filter((row) => stringValue(row.status, 'unknown') === 'degraded').length;
const success = metrics.filter((row) => ['success', 'pass', 'passed'].includes(stringValue(row.status, 'unknown'))).length;
const sourceEvidence = evidenceSources(input);
const primary = sourceEvidence.find((source) => source.source === 'metrics');
const failureRate = metrics.length ? round((failed / metrics.length) * 100, 1) : 0;
const p95 = percentile(latencies, 95);
const failureThreshold = configuredThreshold('CASAN_H6_FAILURE_RATE_THRESHOLD_PCT', 5);
const p95Threshold = configuredThreshold('CASAN_H6_P95_LATENCY_THRESHOLD_MS', 5000);
const findings: HarnessReportFinding[] = [];
const warnings: string[] = [];
if (!primary?.present) findings.push({ severity: 'critical', code: 'METRICS_MISSING', message: 'The required runtime metrics source is missing.' });
else if (primary.stale) findings.push({ severity: 'warning', code: 'METRICS_STALE', message: 'The primary runtime metrics source is stale.', metric: 'age_s', value: primary.age_s ?? 'unknown', threshold: STALE_AFTER_S });
if (failureRate > failureThreshold) findings.push({ severity: 'critical', code: 'FAILURE_RATE_BREACH', message: 'Failure rate exceeds the configured H6 threshold.', metric: 'failure_rate_pct', value: failureRate, threshold: failureThreshold });
else if (failed > 0) findings.push({ severity: 'warning', code: 'FAILURES_PRESENT', message: `${failed} failed run(s) are present in the selected scope.`, metric: 'failed', value: failed });
if (p95 > p95Threshold) findings.push({ severity: 'warning', code: 'P95_LATENCY_BREACH', message: 'P95 latency exceeds the configured H6 threshold.', metric: 'p95_latency_ms', value: p95, threshold: p95Threshold });
const alertFingerprints = new Map<string, Set<string>>();
for (const row of [...metrics, ...alerts]) {
for (const alert of alertTypes(row)) {
const run = rowRun(row);
const fingerprint = `${run === 'unattributed' ? stringValue(row.timestamp, 'unknown-time') : run}:${alert}`;
const known = alertFingerprints.get(alert) ?? new Set<string>();
known.add(fingerprint);
alertFingerprints.set(alert, known);
}
}
const alertCounts = [...alertFingerprints.entries()].map(([alert, fingerprints]) => ({ alert, count: fingerprints.size }));
const alertCount = alertCounts.reduce((total, entry) => total + entry.count, 0);
if (alertCount > 0) findings.push({ severity: 'warning', code: 'ALERTS_PRESENT', message: `${alertCount} alert signal(s) require review.`, metric: 'alerts', value: alertCount });
if (degraded > 0) findings.push({ severity: 'warning', code: 'DEGRADED_RUNS_PRESENT', message: `${degraded} degraded run(s) are present in the selected scope.`, metric: 'degraded', value: degraded });
if (provider.length === 0) warnings.push('No provider usage records matched the selected scope; token and actual-cost breakdown may be incomplete.');
for (const source of sourceEvidence) {
if (source.source === 'metrics') continue;
if (!source.present) warnings.push(`Optional ${source.source} telemetry source is missing; its breakdown is unavailable.`);
else if (source.stale) warnings.push(`Optional ${source.source} telemetry source is stale; its breakdown may not reflect recent activity.`);
}
const estimatedRecords = metrics.filter((row) => stringValue(row.cost_source, 'unknown').includes('estimate')).length;
if (estimatedRecords > 0) warnings.push(`${estimatedRecords} metric record(s) use estimated rather than provider-reported cost.`);
const unattributed = [...input.metrics, ...input.provider].filter((row) => row.project === undefined && row.project_id === undefined).length;
if (unattributed > 0) warnings.push(`${unattributed} source record(s) have no project attribute and are classified as project "default".`);
const critical = findings.some((finding) => finding.severity === 'critical');
const warning = findings.some((finding) => finding.severity === 'warning');
const verdict = metrics.length === 0 && provider.length === 0 ? 'no_data' : critical ? 'fail' : warning ? 'attention' : 'pass';
const freshnessStatus = !primary?.present ? 'missing' : primary.stale ? 'stale' : 'live';
const availableProjects = [...new Set([...input.metrics, ...input.provider, ...input.alerts].map(rowProject))].sort();
const availableRuns = [...new Set([...input.metrics, ...input.provider, ...input.alerts].map(rowRun).filter((run) => run !== 'unattributed'))].sort();
const costSources = grouped(metrics, (row) => stringValue(row.cost_source, 'unknown')).map((entry) => ({
source: entry.key,
records: entry.runs,
cost_usd: entry.cost_usd,
}));
const scope: HarnessReportScope = { project: query.project, from: query.from, to: query.to, run: query.run };
return {
schema_version: 1,
report_id: `H6-${input.now.toISOString().replace(/[-:.TZ]/g, '').slice(0, 14)}`,
harness: 'H6',
title: 'H6 · AgentOps Report',
description: 'Evidence-backed runtime, token, cost, failure, retry and alert telemetry.',
generated_at: input.now.toISOString(),
scope,
verdict,
verdict_reasons: findings.map((finding) => finding.code),
freshness: { status: freshnessStatus, stale_after_s: STALE_AFTER_S, primary_age_s: primary?.age_s ?? null, sources: sourceEvidence },
summary: {
runs: metrics.length,
success,
failed,
degraded,
failure_rate_pct: failureRate,
retries: sum(metrics, 'retry_count'),
latency_ms: {
average: latencies.length ? Math.round(latencies.reduce((total, value) => total + value, 0) / latencies.length) : 0,
p50: percentile(latencies, 50),
p95,
p99: percentile(latencies, 99),
max: latencies.length ? Math.max(...latencies) : 0,
},
tokens: { input: sum(metrics, 'input_tokens'), output: sum(metrics, 'output_tokens'), total: sum(metrics, 'total_tokens'), provider_total: sum(provider, 'total_tokens') },
cost_usd: { provider_actual: round(sum(provider, 'cost_usd'), 6), estimated: round(sum(metrics, 'cost_estimate'), 6) },
provider_calls: provider.length,
alerts: alertCount,
},
thresholds: { failure_rate_pct: failureThreshold, p95_latency_ms: p95Threshold, freshness_age_s: STALE_AFTER_S },
findings,
evidence_sources: sourceEvidence,
data_quality: { status: !primary?.present ? 'insufficient' : warnings.length ? 'partial' : 'complete', warnings },
available_filters: { projects: availableProjects, runs: availableRuns },
details: {
by_status: counted(metrics, (row) => stringValue(row.status, 'unknown')).map(({ key, count: countValue }) => ({ status: key, count: countValue })),
by_step: grouped(metrics, (row) => stringValue(row.step, 'unknown-step')),
by_provider: grouped(provider, (row) => `${stringValue(row.provider, 'unknown-provider')} · ${stringValue(row.model, 'unknown-model')}`),
by_cost_source: costSources,
by_alert: alertCounts.sort((left, right) => right.count - left.count || left.alert.localeCompare(right.alert)),
recent_runs: [...metrics].slice(-query.limit).reverse(),
recent_alerts: [...alerts].slice(-query.limit).reverse(),
},
};
}
export function defaultH6ReportInput(now = new Date()): H6ReportInput {
return {
metrics: [],
provider: [],
alerts: [],
sourceFreshness: telemetryFreshness(now.getTime()),
now,
};
}
export const H6_REPORT_PATHS = PATHS;
@@ -0,0 +1,83 @@
import type { SourceFreshness } from '../common/app-root.js';
export const HARNESS_REPORT_DEFINITIONS = [
{ id: 'H1', slug: 'context', title: 'H1 · Context', description: 'Context selection, prompt contract and risk classification' },
{ id: 'H2', slug: 'tool', title: 'H2 · Tool', description: 'Allowlisted tools, actions and execution boundaries' },
{ id: 'H3', slug: 'evaluation', title: 'H3 · Evaluation', description: 'Quality evaluation, traceability and acceptance evidence' },
{ id: 'H4', slug: 'security', title: 'H4 · Security', description: 'Input, output and artifact security controls' },
{ id: 'H5', slug: 'governance', title: 'H5 · Governance', description: 'Policy decisions, approvals and audit integrity' },
{ id: 'H6', slug: 'agentops', title: 'H6 · AgentOps', description: 'Runtime, token, cost, failure and alert telemetry' },
{ id: 'H7', slug: 'orchestration', title: 'H7 · Orchestration', description: 'Final outcome, certification and rollback evidence' },
] as const;
export type HarnessReportId = typeof HARNESS_REPORT_DEFINITIONS[number]['id'];
export type HarnessReportVerdict = 'pass' | 'attention' | 'fail' | 'no_data';
export type ReportExportFormat = 'json' | 'html';
export interface HarnessReportScope {
project: string | null;
from: string | null;
to: string | null;
run: string | null;
}
export interface HarnessReportFinding {
severity: 'info' | 'warning' | 'critical';
code: string;
message: string;
metric?: string;
value?: number | string;
threshold?: number | string;
}
export interface HarnessReportEvidenceSource extends SourceFreshness {
records: number;
}
export interface HarnessReport<TSummary, TDetails> {
schema_version: 1;
report_id: string;
harness: HarnessReportId;
title: string;
description: string;
generated_at: string;
scope: HarnessReportScope;
verdict: HarnessReportVerdict;
verdict_reasons: string[];
freshness: {
status: 'live' | 'stale' | 'missing';
stale_after_s: number;
primary_age_s: number | null;
sources: HarnessReportEvidenceSource[];
};
summary: TSummary;
thresholds: Record<string, number | string | boolean>;
findings: HarnessReportFinding[];
evidence_sources: HarnessReportEvidenceSource[];
data_quality: {
status: 'complete' | 'partial' | 'insufficient';
warnings: string[];
};
available_filters: {
projects: string[];
runs: string[];
};
details: TDetails;
}
export interface HarnessReportCatalogEntry {
id: HarnessReportId;
slug: string;
title: string;
description: string;
contract_version: 1;
endpoint: string;
availability: 'implemented' | 'contract_ready';
}
export const HARNESS_REPORT_CATALOG: HarnessReportCatalogEntry[] = HARNESS_REPORT_DEFINITIONS.map((definition) => ({
...definition,
contract_version: 1,
endpoint: `/api/v1/reports/${definition.id.toLowerCase()}`,
availability: definition.id === 'H6' ? 'implemented' : 'contract_ready',
}));
@@ -0,0 +1,37 @@
import { BadRequestException, Controller, Get, Header, Inject, Query, Res } from '@nestjs/common';
import type { Response } from 'express';
import { ok } from '../common/api-response.js';
import { parseH6ReportQuery, type H6ReportQueryParams } from './h6-report.js';
import { ReportsService } from './reports.service.js';
interface H6QueryParams extends H6ReportQueryParams {
format?: string;
}
@Controller('api/v1/reports')
export class ReportsController {
constructor(@Inject(ReportsService) private readonly reports: ReportsService) {}
@Get()
catalog() {
return ok(this.reports.catalog());
}
@Get('h6')
h6(@Query() raw: H6QueryParams) {
return ok(this.reports.h6(parseH6ReportQuery(raw)));
}
@Get('h6/export')
@Header('Cache-Control', 'no-store')
exportH6(@Query() raw: H6QueryParams, @Res() response: Response) {
const format = raw.format ?? 'json';
if (format !== 'json' && format !== 'html') throw new BadRequestException('H6_REPORT_INVALID_FORMAT');
const report = this.reports.h6(parseH6ReportQuery(raw));
const body = this.reports.serializeH6(report, format);
const stamp = report.generated_at.slice(0, 10);
response.type(format === 'html' ? 'text/html; charset=utf-8' : 'application/json; charset=utf-8');
response.setHeader('Content-Disposition', `attachment; filename="casan-h6-report-${stamp}.${format}"`);
response.send(body);
}
}
@@ -0,0 +1,10 @@
import { Module } from '@nestjs/common';
import { ReportsController } from './reports.controller.js';
import { ReportsService } from './reports.service.js';
@Module({
controllers: [ReportsController],
providers: [ReportsService],
exports: [ReportsService],
})
export class ReportsModule {}
@@ -0,0 +1,28 @@
import { Injectable } from '@nestjs/common';
import { PATHS, telemetryFreshness } from '../common/app-root.js';
import { readJsonl } from '../telemetry/telemetry.reader.js';
import { buildH6Report, type H6Report, type H6ReportQuery } from './h6-report.js';
import { renderH6ReportHtml } from './h6-report.html.js';
import { HARNESS_REPORT_CATALOG } from './report.contract.js';
@Injectable()
export class ReportsService {
catalog() {
return { schema_version: 1, reports: HARNESS_REPORT_CATALOG };
}
h6(query: H6ReportQuery): H6Report {
const now = new Date();
return buildH6Report({
metrics: readJsonl(PATHS.metrics),
provider: readJsonl(PATHS.providerUsage),
alerts: readJsonl(PATHS.alerts),
sourceFreshness: telemetryFreshness(now.getTime()),
now,
}, query);
}
serializeH6(report: H6Report, format: 'json' | 'html'): string {
return format === 'html' ? renderH6ReportHtml(report) : `${JSON.stringify(report, null, 2)}\n`;
}
}
@@ -5,7 +5,7 @@ import { existsSync, readFileSync, statSync } from 'node:fs';
import { execFileSync } from 'node:child_process';
import { join, relative } from 'node:path';
import { Injectable } from '@nestjs/common';
import { APP_ROOT, PATHS, isStale, metricsAgeSeconds, STALE_AFTER_S } from '../common/app-root.js';
import { APP_ROOT, PATHS, isStale, metricsAgeSeconds, STALE_AFTER_S, telemetryFreshness } from '../common/app-root.js';
import { readJsonl, readJson, readHead, readTrace } from './telemetry.reader.js';
type Row = Record<string, any>;
@@ -217,7 +217,7 @@ export function buildChatLoopWidget(
export class TelemetryService {
private freshness() {
const age = metricsAgeSeconds();
return { stale: isStale(), age_s: age, stale_after_s: STALE_AFTER_S };
return { stale: isStale(), age_s: age, stale_after_s: STALE_AFTER_S, sources: telemetryFreshness() };
}
overview() {
@@ -0,0 +1,119 @@
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { BadRequestException } from '@nestjs/common';
import { mkdtempSync, utimesSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { sourceFreshness } from '../src/common/app-root.js';
import { buildH6Report, parseH6ReportQuery, type H6ReportInput } from '../src/reports/h6-report.js';
import { renderH6ReportHtml } from '../src/reports/h6-report.html.js';
import { HARNESS_REPORT_CATALOG } from '../src/reports/report.contract.js';
const NOW = new Date('2026-07-20T12:00:00.000Z');
function fixture(): H6ReportInput {
return {
now: NOW,
metrics: [
{ timestamp: '2026-07-19T08:00:00Z', trace_id: 'run-a', project: 'okr', status: 'success', step: 'generate', latency_ms: 100, input_tokens: 10, output_tokens: 20, total_tokens: 30, cost_estimate: 0.001, cost_source: 'word_count_estimate', retry_count: 0, alerts: [] },
{ timestamp: '2026-07-19T09:00:00Z', trace_id: 'run-b', project: 'okr', status: 'failed', step: 'test', latency_ms: 9000, input_tokens: 20, output_tokens: 5, total_tokens: 25, cost_estimate: 0.002, cost_source: 'provider_telemetry', retry_count: 2, alerts: ['high-latency', 'execution-failed'] },
{ timestamp: '2026-07-18T09:00:00Z', trace_id: 'run-c', project: 'desk', status: 'success', step: 'test', latency_ms: 250, total_tokens: 10, cost_estimate: 0.0001, cost_source: 'word_count_estimate' },
],
provider: [
{ timestamp: '2026-07-19T08:00:00Z', run_id: 'run-a', project: 'okr', provider: 'ollama', model: 'ornith:9b', status: 'success', step: 'generate', latency_ms: 80, total_tokens: 28, cost_usd: 0 },
{ timestamp: '2026-07-18T09:00:00Z', run_id: 'run-c', project: 'desk', provider: 'openai', model: 'gpt-test', status: 'success', step: 'test', latency_ms: 200, total_tokens: 10, cost_usd: 0.01 },
],
alerts: [
{ timestamp: '2026-07-19T09:00:00Z', trace_id: 'run-b', project: 'okr', body: { 'alert.type': 'execution-failed' } },
],
sourceFreshness: [
{ source: 'metrics', path: '/workspace/.specify/logs/cost/metrics.jsonl', present: true, updated_at: '2026-07-20T11:59:00Z', age_s: 60, stale: false, required: true },
{ source: 'provider', path: '/workspace/.specify/logs/level5/provider-usage.jsonl', present: true, updated_at: '2026-07-20T11:58:00Z', age_s: 120, stale: false, required: false },
{ source: 'alerts', path: '/workspace/.specify/agentops/alerts.log', present: true, updated_at: '2026-07-20T11:57:00Z', age_s: 180, stale: false, required: false },
],
};
}
test('H1-H7 share one versioned report catalog contract', () => {
assert.deepEqual(HARNESS_REPORT_CATALOG.map((entry) => entry.id), ['H1', 'H2', 'H3', 'H4', 'H5', 'H6', 'H7']);
assert.ok(HARNESS_REPORT_CATALOG.every((entry) => entry.contract_version === 1));
assert.equal(HARNESS_REPORT_CATALOG.find((entry) => entry.id === 'H6')?.availability, 'implemented');
});
test('telemetry freshness is source-specific and missing required data fails loud', () => {
const directory = mkdtempSync(join(tmpdir(), 'casan-h6-'));
const metricsPath = join(directory, 'metrics.jsonl');
writeFileSync(metricsPath, '{}\n');
const oldTime = new Date(NOW.getTime() - 7200 * 1000);
utimesSync(metricsPath, oldTime, oldTime);
const staleMetrics = sourceFreshness('metrics', metricsPath, true, NOW.getTime());
const missingMetrics = sourceFreshness('metrics', join(directory, 'missing.jsonl'), true, NOW.getTime());
assert.equal(staleMetrics.present, true);
assert.equal(staleMetrics.age_s, 7200);
assert.equal(staleMetrics.stale, true);
assert.deepEqual(
{ present: missingMetrics.present, stale: missingMetrics.stale, required: missingMetrics.required },
{ present: false, stale: true, required: true },
);
});
test('H6 report filters project/time/run and aggregates measured evidence', () => {
const query = parseH6ReportQuery({ project: 'okr', from: '2026-07-19', to: '2026-07-19' });
const report = buildH6Report(fixture(), query);
assert.equal(report.schema_version, 1);
assert.equal(report.harness, 'H6');
assert.equal(report.summary.runs, 2);
assert.equal(report.summary.failed, 1);
assert.equal(report.summary.retries, 2);
assert.equal(report.summary.latency_ms.p50, 100);
assert.equal(report.summary.latency_ms.p95, 9000);
assert.equal(report.summary.tokens.provider_total, 28);
assert.equal(report.summary.alerts, 2);
assert.deepEqual(report.details.by_alert, [
{ alert: 'execution-failed', count: 1 },
{ alert: 'high-latency', count: 1 },
]);
assert.equal(report.available_filters.projects.length, 2);
assert.equal(report.freshness.status, 'live');
assert.equal(report.verdict, 'fail');
assert.ok(report.findings.some((finding) => finding.code === 'FAILURE_RATE_BREACH'));
});
test('H6 report supports exact run filtering and truthful no-data verdict', () => {
const selected = buildH6Report(fixture(), parseH6ReportQuery({ run: 'run-a' }));
assert.equal(selected.summary.runs, 1);
assert.equal(selected.summary.provider_calls, 1);
assert.equal(selected.verdict, 'pass');
const empty = buildH6Report(fixture(), parseH6ReportQuery({ run: 'not-found' }));
assert.equal(empty.verdict, 'no_data');
assert.equal(empty.summary.runs, 0);
});
test('H6 query validation rejects traversal, inverted dates and invalid limits', () => {
assert.throws(() => parseH6ReportQuery({ project: '../secret' }), BadRequestException);
assert.throws(() => parseH6ReportQuery({ from: '2026-07-20', to: '2026-07-19' }), BadRequestException);
assert.throws(() => parseH6ReportQuery({ limit: '5000' }), BadRequestException);
});
test('H6 report makes stale optional sources explicit in data quality', () => {
const input = fixture();
input.sourceFreshness[1] = { ...input.sourceFreshness[1], stale: true, age_s: 7200 };
input.sourceFreshness[2] = { ...input.sourceFreshness[2], present: false, updated_at: null, age_s: null, stale: false };
const report = buildH6Report(input, parseH6ReportQuery({ run: 'run-a' }));
assert.equal(report.data_quality.status, 'partial');
assert.ok(report.data_quality.warnings.some((warning) => warning.includes('provider telemetry source is stale')));
assert.ok(report.data_quality.warnings.some((warning) => warning.includes('alerts telemetry source is missing')));
});
test('HTML export is standalone, escaped and contains no hard-coded maturity score', () => {
const input = fixture();
input.metrics[0].step = '<script>alert(1)</script>';
const report = buildH6Report(input, parseH6ReportQuery({ project: 'okr' }));
const html = renderH6ReportHtml(report);
assert.match(html, /^<!doctype html>/);
assert.match(html, /H6 · AgentOps Report/);
assert.match(html, /&lt;script&gt;alert\(1\)&lt;\/script&gt;/);
assert.doesNotMatch(html, /<script>alert\(1\)<\/script>/);
assert.doesNotMatch(html, /Average\s+\d|\/100|218 core tests/i);
assert.match(html, /No maturity score or telemetry value is hard-coded/);
});
@@ -13,6 +13,7 @@ import { CommandCenter } from './pages/CommandCenter';
import { Chat } from './pages/Chat';
import { Goals } from './pages/Goals';
import { EvidencePacks } from './pages/EvidencePacks';
import { H6ReportPage } from './pages/H6Report';
export default function App() {
return (
@@ -31,6 +32,7 @@ export default function App() {
<Route path="/chat" element={<Chat />} />
<Route path="/goals" element={<Goals />} />
<Route path="/evidence-packs" element={<EvidencePacks />} />
<Route path="/reports/h6" element={<H6ReportPage />} />
<Route path="*" element={<Navigate to="/" replace />} />
</Routes>
</AppLayout>
@@ -13,6 +13,7 @@ const PAGE_COPY: Record<string, { title: string; eyebrow: string }> = {
'/incidents': { title: 'Incident response', eyebrow: 'Containment' },
'/traceability': { title: 'Traceability', eyebrow: 'Evidence graph' },
'/finops': { title: 'FinOps & SLO', eyebrow: 'Model economics' },
'/reports/h6': { title: 'H6 AgentOps report', eyebrow: 'Assurance dossier' },
'/approvals': { title: 'Approval inbox', eyebrow: 'Human-in-the-loop' },
'/settings': { title: 'Governed settings', eyebrow: 'Control plane' },
};
@@ -1,7 +1,7 @@
import type { ReactNode } from 'react';
import { NavLink } from 'react-router-dom';
type IconName = 'grid' | 'command' | 'chat' | 'goal' | 'runs' | 'shield' | 'governance' | 'incident' | 'trace' | 'evidence' | 'coins' | 'approval' | 'settings';
type IconName = 'grid' | 'command' | 'chat' | 'goal' | 'runs' | 'shield' | 'governance' | 'incident' | 'trace' | 'evidence' | 'report' | 'coins' | 'approval' | 'settings';
interface NavItem { to: string; label: string; icon: IconName; }
@@ -19,6 +19,7 @@ const NAVIGATION: Array<{ label: string; items: NavItem[] }> = [
{ to: '/incidents', label: 'Incidents', icon: 'incident' },
{ to: '/traceability', label: 'Traceability', icon: 'trace' },
{ to: '/evidence-packs', label: 'Evidence packs', icon: 'evidence' },
{ to: '/reports/h6', label: 'H6 AgentOps report', icon: 'report' },
] },
{ label: 'Control', items: [
{ to: '/finops', label: 'FinOps & SLO', icon: 'coins' },
@@ -41,6 +42,7 @@ function Icon({ name }: { name: IconName }) {
incident: <><path d="M10.3 3.3 2.7 17a2 2 0 0 0 1.7 3h15.2a2 2 0 0 0 1.7-3L13.7 3.3a2 2 0 0 0-3.4 0Z" /><path d="M12 9v4M12 17h.01" /></>,
trace: <><circle cx="6" cy="6" r="3" /><circle cx="18" cy="18" r="3" /><circle cx="18" cy="6" r="3" /><path d="m8.6 7.5 6.8 3M9 6h6" /></>,
evidence: <><path d="M7 3h7l3 3v15H7a2 2 0 0 1-2-2V5a2 2 0 0 1 2-2Z" /><path d="M14 3v4h4M8.5 12h7M8.5 16h5" /><path d="m9 8.5 1 1 2-2" /></>,
report: <><path d="M4 20V10M10 20V4M16 20v-7M22 20H2" /><path d="M3 10h2M9 4h2M15 13h2" /><path d="m16 7 2 2 4-5" /></>,
coins: <><ellipse cx="12" cy="5" rx="7" ry="3" /><path d="M5 5v7c0 1.7 3.1 3 7 3s7-1.3 7-3V5M5 12v7c0 1.7 3.1 3 7 3s7-1.3 7-3v-7" /></>,
approval: <><path d="M9 11 11 13l4-4" /><path d="M12 22c5-2.1 8-5.3 8-10V5l-8-3-8 3v7c0 4.7 3 7.9 8 10Z" /></>,
settings: <><circle cx="12" cy="12" r="3" /><path d="M19.4 15a1.7 1.7 0 0 0 .3 1.9l.1.1-2 2-.1-.1a1.7 1.7 0 0 0-1.9-.3 1.7 1.7 0 0 0-1 1.5v.2h-2.8v-.2a1.7 1.7 0 0 0-1-1.5 1.7 1.7 0 0 0-1.9.3l-.1.1-2-2 .1-.1A1.7 1.7 0 0 0 7.4 15a1.7 1.7 0 0 0-1.5-1H5.7v-2.8h.2a1.7 1.7 0 0 0 1.5-1 1.7 1.7 0 0 0-.3-1.9L7 8.2l2-2 .1.1a1.7 1.7 0 0 0 1.9.3 1.7 1.7 0 0 0 1-1.5v-.2h2.8v.2a1.7 1.7 0 0 0 1 1.5 1.7 1.7 0 0 0 1.9-.3l.1-.1 2 2-.1.1a1.7 1.7 0 0 0-.3 1.9 1.7 1.7 0 0 0 1.5 1h.2V14h-.2a1.7 1.7 0 0 0-1.5 1Z" /></>,
@@ -21,7 +21,92 @@ async function post<T>(path: string, body: unknown, headers: Record<string, stri
return res.data.data as T;
}
export interface Freshness { stale: boolean; age_s: number | null; stale_after_s: number }
export interface TelemetrySourceFreshness {
source: 'metrics' | 'provider' | 'alerts';
path: string;
present: boolean;
updated_at: string | null;
age_s: number | null;
stale: boolean;
required: boolean;
records?: number;
}
export interface Freshness { stale: boolean; age_s: number | null; stale_after_s: number; sources?: TelemetrySourceFreshness[] }
export type HarnessReportId = 'H1' | 'H2' | 'H3' | 'H4' | 'H5' | 'H6' | 'H7';
export type HarnessReportVerdict = 'pass' | 'attention' | 'fail' | 'no_data';
export interface H6ReportQuery {
project?: string;
from?: string;
to?: string;
run?: string;
limit?: number;
}
export interface H6ReportFinding {
severity: 'info' | 'warning' | 'critical';
code: string;
message: string;
metric?: string;
value?: number | string;
threshold?: number | string;
}
export interface H6Breakdown {
key: string;
runs: number;
failures: number;
latency_avg_ms: number;
tokens: number;
cost_usd: number;
}
export interface H6Report {
schema_version: 1;
report_id: string;
harness: 'H6';
title: string;
description: string;
generated_at: string;
scope: { project: string | null; from: string | null; to: string | null; run: string | null };
verdict: HarnessReportVerdict;
verdict_reasons: string[];
freshness: {
status: 'live' | 'stale' | 'missing';
stale_after_s: number;
primary_age_s: number | null;
sources: Array<TelemetrySourceFreshness & { records: number }>;
};
summary: {
runs: number;
success: number;
failed: number;
degraded: number;
failure_rate_pct: number;
retries: number;
latency_ms: { average: number; p50: number; p95: number; p99: number; max: number };
tokens: { input: number; output: number; total: number; provider_total: number };
cost_usd: { provider_actual: number; estimated: number };
provider_calls: number;
alerts: number;
};
thresholds: Record<string, number | string | boolean>;
findings: H6ReportFinding[];
evidence_sources: Array<TelemetrySourceFreshness & { records: number }>;
data_quality: { status: 'complete' | 'partial' | 'insufficient'; warnings: string[] };
available_filters: { projects: string[]; runs: string[] };
details: {
by_status: Array<{ status: string; count: number }>;
by_step: H6Breakdown[];
by_provider: H6Breakdown[];
by_cost_source: Array<{ source: string; records: number; cost_usd: number }>;
by_alert: Array<{ alert: string; count: number }>;
recent_runs: HarnessRunRecord[];
recent_alerts: Array<Record<string, unknown>>;
};
}
export interface HarnessRunRecord {
timestamp?: string;
@@ -478,6 +563,24 @@ function actorHeaders(actor: SettingsActor): Record<string, string> {
};
}
function reportQuery(query: H6ReportQuery): string {
const params = new URLSearchParams();
if (query.project) params.set('project', query.project);
if (query.from) params.set('from', query.from);
if (query.to) params.set('to', query.to);
if (query.run) params.set('run', query.run);
if (query.limit) params.set('limit', String(query.limit));
const serialized = params.toString();
return serialized ? `?${serialized}` : '';
}
export function h6ReportExportUrl(query: H6ReportQuery, format: 'json' | 'html'): string {
const base = import.meta.env.VITE_API_BASE_URL ?? '/api/v1';
const params = new URLSearchParams(reportQuery(query).replace(/^\?/, ''));
params.set('format', format);
return `${base}/reports/h6/export?${params.toString()}`;
}
export const api = {
session: () => get<SettingsActor>('session'),
overview: () => get<Overview>('overview'),
@@ -502,6 +605,7 @@ export const api = {
incidents: () => get<Freshness & { total: number; kill_switch_scopes: string[]; incidents: any[] }>('incidents'),
traceability: () => get<Freshness & { matrix: any }>('traceability'),
cost: () => get<Freshness & { provider_tokens: number; provider_cost: number; by_provider: any[]; business_kpi: any }>('cost'),
h6Report: (query: H6ReportQuery = {}) => get<H6Report>(`reports/h6${reportQuery(query)}`),
command: () => get<CommandCenterState>('command'),
evidencePacks: () => get<{ root: string; packs: EvidencePackSummary[] }>('evidence-packs'),
evidencePack: (id: string) => get<EvidencePackDetail>(`evidence-packs/${encodeURIComponent(id)}`),
@@ -0,0 +1,139 @@
import { useQuery } from '@tanstack/react-query';
import { useMemo, useState, type FormEvent, type ReactNode } from 'react';
import { useSearchParams } from 'react-router-dom';
import { api, h6ReportExportUrl, type H6Breakdown, type H6ReportQuery, type HarnessReportVerdict } from '../lib/api';
import { Card, StatusBadge } from '../components/ui/Card';
const money = (value: number): string => `$${value.toFixed(6)}`;
const integer = (value: number): string => new Intl.NumberFormat('en-US').format(value);
const verdictTone: Record<HarnessReportVerdict, string> = {
pass: 'border-emerald-300 bg-emerald-50 text-emerald-900',
attention: 'border-amber-300 bg-amber-50 text-amber-950',
fail: 'border-rose-300 bg-rose-50 text-rose-950',
no_data: 'border-slate-300 bg-slate-50 text-slate-800',
};
function queryFromSearch(search: URLSearchParams): H6ReportQuery {
return {
project: search.get('project') || undefined,
from: search.get('from') || undefined,
to: search.get('to') || undefined,
run: search.get('run') || undefined,
limit: 50,
};
}
function FilterField({ label, children }: { label: string; children: ReactNode }) {
return <label className="block"><span className="mb-1.5 block text-[10px] font-bold uppercase tracking-[0.13em] text-slate-500">{label}</span>{children}</label>;
}
function MetricCard({ label, value, sub, accent = false }: { label: string; value: string; sub?: string; accent?: boolean }) {
return <div className={`relative overflow-hidden rounded-2xl border p-4 ${accent ? 'border-indigo-200 bg-indigo-50/70' : 'border-slate-200 bg-white'}`}>
{accent && <span className="absolute -right-5 -top-5 h-16 w-16 rounded-full bg-indigo-200/50 blur-xl" />}
<div className="relative text-[10px] font-bold uppercase tracking-[0.12em] text-slate-500">{label}</div>
<div className="relative mt-2 text-2xl font-semibold tracking-tight text-slate-900">{value}</div>
{sub && <div className="relative mt-1 text-xs text-slate-500">{sub}</div>}
</div>;
}
function BreakdownTable({ rows, subject }: { rows: H6Breakdown[]; subject: string }) {
return <div className="overflow-x-auto"><table className="w-full min-w-[680px] text-sm">
<thead><tr className="border-b border-slate-200 text-left text-[10px] font-bold uppercase tracking-[0.1em] text-slate-400">
<th className="py-2.5 pr-3">{subject}</th><th>runs</th><th>failures</th><th>avg latency</th><th>tokens</th><th>cost</th>
</tr></thead>
<tbody>{rows.map((row) => <tr key={row.key} className="border-b border-slate-100 transition-colors hover:bg-slate-50/80">
<td className="max-w-[300px] py-3 pr-3 font-medium text-slate-800">{row.key}</td><td>{integer(row.runs)}</td><td>{integer(row.failures)}</td><td>{integer(row.latency_avg_ms)} ms</td><td>{integer(row.tokens)}</td><td>{money(row.cost_usd)}</td>
</tr>)}{rows.length === 0 && <tr><td colSpan={6} className="py-8 text-center text-slate-500">No matching {subject.toLowerCase()} records.</td></tr>}</tbody>
</table></div>;
}
function LoadingReport() {
return <div className="space-y-4" aria-label="Loading H6 report"><div className="h-44 animate-pulse rounded-3xl bg-slate-200" /><div className="grid grid-cols-2 gap-3 lg:grid-cols-4">{Array.from({ length: 8 }, (_, index) => <div key={index} className="h-28 animate-pulse rounded-2xl bg-slate-200" />)}</div></div>;
}
export function H6ReportPage() {
const [searchParams, setSearchParams] = useSearchParams();
const query = useMemo(() => queryFromSearch(searchParams), [searchParams]);
const [draft, setDraft] = useState(() => ({
project: searchParams.get('project') ?? '',
from: searchParams.get('from') ?? '',
to: searchParams.get('to') ?? '',
run: searchParams.get('run') ?? '',
}));
const report = useQuery({ queryKey: ['reports', 'h6', query], queryFn: () => api.h6Report(query), retry: false });
const applyFilters = (event: FormEvent<HTMLFormElement>) => {
event.preventDefault();
const next = new URLSearchParams();
if (draft.project) next.set('project', draft.project);
if (draft.from) next.set('from', draft.from);
if (draft.to) next.set('to', draft.to);
if (draft.run) next.set('run', draft.run);
setSearchParams(next);
};
const clearFilters = () => {
setDraft({ project: '', from: '', to: '', run: '' });
setSearchParams({});
};
const inputClass = 'w-full rounded-xl border border-slate-300 bg-white px-3 py-2.5 text-sm text-slate-800 outline-none transition focus:border-indigo-400 focus:ring-2 focus:ring-indigo-100';
return <div className="space-y-5">
<section className="relative overflow-hidden rounded-3xl border border-slate-800 bg-[#111827] px-5 py-6 text-white shadow-[0_22px_55px_rgba(15,23,42,0.16)] sm:px-7">
<div className="absolute right-[-60px] top-[-90px] h-64 w-64 rounded-full bg-indigo-500/25 blur-3xl" />
<div className="absolute bottom-[-120px] left-1/3 h-56 w-56 rounded-full bg-teal-500/10 blur-3xl" />
<div className="relative flex flex-col justify-between gap-5 lg:flex-row lg:items-end">
<div><div className="text-[10px] font-bold uppercase tracking-[0.2em] text-indigo-300">H6 · evidence-backed operations</div><h2 className="mt-2 text-2xl font-semibold tracking-tight sm:text-3xl">AgentOps assurance dossier</h2><p className="mt-2 max-w-2xl text-sm leading-6 text-slate-300">One report surface for runtime health, provider usage, cost provenance, failures, retries, alerts and source freshness.</p></div>
{report.data && <div className="flex flex-wrap items-center gap-2"><span className={`rounded-full border px-3 py-2 text-xs font-black uppercase tracking-[0.12em] ${verdictTone[report.data.verdict]}`}>{report.data.verdict}</span><span className="rounded-full border border-slate-700 bg-slate-900/70 px-3 py-2 text-xs text-slate-300">{report.data.freshness.status} · {report.data.report_id}</span></div>}
</div>
</section>
<Card title="Report scope" right={<span className="text-xs font-normal normal-case tracking-normal text-slate-400">Filters apply to API and both exports</span>}>
<form onSubmit={applyFilters} className="grid gap-3 md:grid-cols-2 xl:grid-cols-[1fr_1fr_1fr_1.4fr_auto] xl:items-end">
<FilterField label="Project"><select value={draft.project} onChange={(event) => setDraft((current) => ({ ...current, project: event.target.value }))} className={inputClass}><option value="">All projects</option>{report.data?.available_filters.projects.map((project) => <option key={project} value={project}>{project}</option>)}</select></FilterField>
<FilterField label="From"><input type="date" value={draft.from} onChange={(event) => setDraft((current) => ({ ...current, from: event.target.value }))} className={inputClass} /></FilterField>
<FilterField label="To"><input type="date" value={draft.to} onChange={(event) => setDraft((current) => ({ ...current, to: event.target.value }))} className={inputClass} /></FilterField>
<FilterField label="Run / trace"><input list="h6-run-options" value={draft.run} onChange={(event) => setDraft((current) => ({ ...current, run: event.target.value }))} placeholder="All runs" className={inputClass} /><datalist id="h6-run-options">{report.data?.available_filters.runs.map((run) => <option key={run} value={run} />)}</datalist></FilterField>
<div className="flex gap-2"><button type="submit" className="rounded-xl bg-indigo-600 px-4 py-2.5 text-sm font-semibold text-white transition hover:bg-indigo-700 focus:outline-none focus:ring-2 focus:ring-indigo-300">Apply</button><button type="button" onClick={clearFilters} className="rounded-xl border border-slate-300 bg-white px-3 py-2.5 text-sm font-semibold text-slate-600 transition hover:bg-slate-50">Clear</button></div>
</form>
</Card>
{report.isLoading && <LoadingReport />}
{report.isError && <div role="alert" className="rounded-2xl border border-rose-200 bg-rose-50 p-5 text-sm text-rose-800"><div className="font-semibold">The H6 report could not be generated.</div><p className="mt-1 text-rose-700">Check the selected time range and confirm the Control Panel API can read the canonical telemetry sources.</p></div>}
{report.data && <>
<section className="grid grid-cols-2 gap-3 lg:grid-cols-4 xl:grid-cols-8">
<MetricCard label="Runs" value={integer(report.data.summary.runs)} sub={`${report.data.summary.success} successful`} accent />
<MetricCard label="Failures" value={integer(report.data.summary.failed)} sub={`${report.data.summary.failure_rate_pct}% rate`} />
<MetricCard label="Degraded" value={integer(report.data.summary.degraded)} />
<MetricCard label="P95 latency" value={`${integer(report.data.summary.latency_ms.p95)}ms`} sub={`P50 ${integer(report.data.summary.latency_ms.p50)}ms`} />
<MetricCard label="Provider tokens" value={integer(report.data.summary.tokens.provider_total)} sub={`${report.data.summary.provider_calls} calls`} accent />
<MetricCard label="Actual cost" value={money(report.data.summary.cost_usd.provider_actual)} sub="provider reported" />
<MetricCard label="Estimated cost" value={money(report.data.summary.cost_usd.estimated)} sub="runtime fallback" />
<MetricCard label="Alerts" value={integer(report.data.summary.alerts)} sub={`${report.data.summary.retries} retries`} />
</section>
<div className="grid gap-5 xl:grid-cols-[1.15fr_0.85fr]">
<Card title="Verdict findings" right={<StatusBadge value={report.data.verdict} />}>
<div className="space-y-2.5">{report.data.findings.map((finding) => <div key={finding.code} className={`rounded-xl border-l-4 p-3.5 ${finding.severity === 'critical' ? 'border-rose-500 bg-rose-50' : finding.severity === 'warning' ? 'border-amber-500 bg-amber-50' : 'border-indigo-500 bg-indigo-50'}`}><div className="flex flex-wrap items-center justify-between gap-2"><span className="font-mono text-xs font-bold text-slate-800">{finding.code}</span>{finding.metric && <span className="text-[11px] text-slate-500">{finding.metric}: {String(finding.value)}{finding.threshold !== undefined ? ` · threshold ${String(finding.threshold)}` : ''}</span>}</div><p className="mt-1 text-sm leading-5 text-slate-600">{finding.message}</p></div>)}{report.data.findings.length === 0 && <div className="rounded-xl border border-emerald-200 bg-emerald-50 p-4 text-sm text-emerald-800">No threshold breach was detected in this scope.</div>}</div>
</Card>
<Card title="Evidence freshness" right={<StatusBadge value={report.data.freshness.status} />}>
<div className="space-y-3">{report.data.evidence_sources.map((source) => <div key={source.source} className="rounded-xl border border-slate-200 bg-slate-50/70 p-3"><div className="flex items-center justify-between gap-3"><span className="text-sm font-semibold capitalize text-slate-800">{source.source}</span><StatusBadge value={!source.present ? 'missing' : source.stale ? 'stale' : 'fresh'} /></div><div className="mt-2 flex justify-between text-xs text-slate-500"><span>{source.records} records</span><span>{source.age_s === null ? 'no timestamp' : `${integer(source.age_s)}s old`}</span></div><div className="mt-2 break-all font-mono text-[10px] leading-4 text-slate-400">{source.path}</div></div>)}</div>
</Card>
</div>
<div className="grid gap-5 xl:grid-cols-2"><Card title="Runtime by step"><BreakdownTable rows={report.data.details.by_step} subject="Step" /></Card><Card title="Provider usage"><BreakdownTable rows={report.data.details.by_provider} subject="Provider · model" /></Card></div>
<div className="grid gap-5 xl:grid-cols-[0.8fr_1.2fr]">
<Card title={`Data quality · ${report.data.data_quality.status}`}>
{report.data.data_quality.warnings.length > 0 ? <ul className="space-y-2 text-sm leading-5 text-slate-600">{report.data.data_quality.warnings.map((warning) => <li key={warning} className="flex gap-2"><span className="mt-1 h-2 w-2 shrink-0 rounded-full bg-amber-400" />{warning}</li>)}</ul> : <div className="text-sm text-emerald-700">All required sources are present and no estimation warning was detected.</div>}
</Card>
<Card title="Independent export" right={<span className="text-xs font-normal normal-case tracking-normal text-slate-400">Generated on demand · no hard-coded score</span>}>
<p className="text-sm leading-6 text-slate-600">Both exports use the same contract, filters, verdict rules and canonical sources as this screen. JSON is machine-auditable; HTML is print-ready for team review.</p>
<div className="mt-4 flex flex-wrap gap-3"><a href={h6ReportExportUrl(query, 'json')} className="rounded-xl bg-slate-900 px-4 py-2.5 text-sm font-semibold text-white transition hover:bg-slate-700">Download JSON</a><a href={h6ReportExportUrl(query, 'html')} className="rounded-xl border border-indigo-300 bg-indigo-50 px-4 py-2.5 text-sm font-semibold text-indigo-700 transition hover:bg-indigo-100">Download HTML</a></div>
</Card>
</div>
</>}
</div>;
}
@@ -27,9 +27,9 @@ LOG_DIR="$CASAN_STATE_ROOT/logs"
TRACE_DIR="$LOG_DIR/trace"
# SEC-23 (MT-01): telemetry dir is tenant-scoped when CASAN_METRICS_DIR is set
# (tenant-paths.sh exports it per tenant); default is the shared path.
METRICS_DIR="${CASAN_METRICS_DIR:-$LOG_DIR/cost}"
ALERT_LOG="$CASAN_HARNESS_ROOT/agentops/alerts.log"
METRICS_LOG="$METRICS_DIR/metrics.jsonl"
METRICS_DIR="${CASAN_METRICS_DIR:-$(dirname "$CASAN_TELEMETRY_METRICS_LOG")}"
ALERT_LOG="$CASAN_TELEMETRY_ALERTS_LOG"
METRICS_LOG="${CASAN_METRICS_LOG:-$CASAN_TELEMETRY_METRICS_LOG}"
mkdir -p "$TRACE_DIR" "$METRICS_DIR" "$(dirname "$OUTPUT_FILE")" "$(dirname "$ALERT_LOG")"
# shellcheck source=tool-audit-lib.sh
@@ -127,7 +127,7 @@ COST_SOURCE="word_count_estimate"
# Prefer real provider usage when telemetry has been imported; the word-count
# figure above is an explicit fallback, not presented as a real billed cost.
PROVIDER_LOG="$CASAN_STATE_ROOT/logs/level5/provider-usage.jsonl"
PROVIDER_LOG="$CASAN_TELEMETRY_PROVIDER_LOG"
if [[ -f "$PROVIDER_LOG" ]] && command -v python >/dev/null 2>&1; then
# Use real provider telemetry ONLY when a record genuinely matches this step.
# Do NOT fall back to an arbitrary record (that would reuse one sample's cost
@@ -61,6 +61,20 @@ if [[ -z "${CASAN_STATE_ROOT:-}" ]]; then
CASAN_STATE_ROOT="$CASAN_APP_ROOT/.specify"
fi
# Canonical H6 telemetry files. Writers and readers must use these three names so
# runtime, provider usage and alerts cannot silently drift into different trees.
# The older dashboard/control-panel variables remain accepted as compatibility
# aliases while downstream deployments move to the CASAN_TELEMETRY_* contract.
if [[ -z "${CASAN_TELEMETRY_METRICS_LOG:-}" ]]; then
CASAN_TELEMETRY_METRICS_LOG="${CASAN_DASHBOARD_METRICS:-$CASAN_STATE_ROOT/logs/cost/metrics.jsonl}"
fi
if [[ -z "${CASAN_TELEMETRY_PROVIDER_LOG:-}" ]]; then
CASAN_TELEMETRY_PROVIDER_LOG="${CASAN_CP_PROVIDER_USAGE:-${CASAN_PROVIDER_LOG:-$CASAN_STATE_ROOT/logs/level5/provider-usage.jsonl}}"
fi
if [[ -z "${CASAN_TELEMETRY_ALERTS_LOG:-}" ]]; then
CASAN_TELEMETRY_ALERTS_LOG="${CASAN_DASHBOARD_ALERTS:-$CASAN_STATE_ROOT/agentops/alerts.log}"
fi
# Governance root: central-governance mixes harness pub-keys/registries, runtime
# policy-manifest state, and a private key. Rooted under STATE (not HARNESS) so the
# runtime-regenerated policy-manifest.{json,sig} and the private key never land inside
@@ -75,7 +75,8 @@ def head_path() -> str:
def metrics_path() -> str:
return guarded_override(os.environ["CASAN_CHAT_METRICS_LOG"]) if os.environ.get("CASAN_CHAT_METRICS_LOG") else tenant_path("telemetry/cost/metrics.jsonl", "logs/cost/metrics.jsonl")
override = os.environ.get("CASAN_TELEMETRY_METRICS_LOG") or os.environ.get("CASAN_CHAT_METRICS_LOG")
return guarded_override(override) if override else tenant_path("telemetry/cost/metrics.jsonl", "logs/cost/metrics.jsonl")
def artifact_dir() -> str:
@@ -90,7 +90,8 @@ def head_path() -> str:
def metrics_path() -> str:
return guarded_override(os.environ["CASAN_CHAT_METRICS_LOG"]) if os.environ.get("CASAN_CHAT_METRICS_LOG") else tenant_path("telemetry/cost/metrics.jsonl", "logs/cost/metrics.jsonl")
override = os.environ.get("CASAN_TELEMETRY_METRICS_LOG") or os.environ.get("CASAN_CHAT_METRICS_LOG")
return guarded_override(override) if override else tenant_path("telemetry/cost/metrics.jsonl", "logs/cost/metrics.jsonl")
def trace_events_path(trace_id: str) -> str:
@@ -24,7 +24,7 @@ set -uo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/casan-paths.sh"
ROOT="$CASAN_APP_ROOT"
PROVIDER_LOG="${CASAN_PROVIDER_LOG:-$CASAN_STATE_ROOT/logs/level5/provider-usage.jsonl}"
PROVIDER_LOG="$CASAN_TELEMETRY_PROVIDER_LOG"
CIRCUIT_BREAKER_THRESHOLD="${CIRCUIT_BREAKER_THRESHOLD:-5}"
CIRCUIT_WINDOW="${CIRCUIT_WINDOW:-10}"
CIRCUIT_WINDOW_FAIL_PCT="${CIRCUIT_WINDOW_FAIL_PCT:-50}"
@@ -30,7 +30,7 @@ elif [[ -n "${CASAN_TENANT_ID:-}" ]]; then
LOG="$(bash "$SCRIPT_DIR/tenant-store.sh" resolve telemetry/provider-usage.jsonl 2>/dev/null)" \
|| { echo "COST_SPIKE_TENANT_DENIED" >&2; exit 3; }
else
LOG="$CASAN_STATE_ROOT/logs/level5/provider-usage.jsonl"
LOG="$CASAN_TELEMETRY_PROVIDER_LOG"
fi
MULT="${2:-3.0}"
@@ -35,9 +35,11 @@ PORT = int(sys.argv[1]) if len(sys.argv) > 1 else 8787
DASH = pathlib.Path(os.environ.get(
"CASAN_DASHBOARD_HTML", ROOT / "docs" / "output" / "casan" / "central-agentops-dashboard.html"))
METRICS = pathlib.Path(os.environ.get(
"CASAN_DASHBOARD_METRICS", ROOT / ".specify" / "logs" / "cost" / "metrics.jsonl"))
"CASAN_TELEMETRY_METRICS_LOG", os.environ.get(
"CASAN_DASHBOARD_METRICS", ROOT / ".specify" / "logs" / "cost" / "metrics.jsonl")))
ALERTS = pathlib.Path(os.environ.get(
"CASAN_DASHBOARD_ALERTS", ROOT / ".specify" / "agentops" / "alerts.log"))
"CASAN_TELEMETRY_ALERTS_LOG", os.environ.get(
"CASAN_DASHBOARD_ALERTS", ROOT / ".specify" / "agentops" / "alerts.log")))
STALE_S = int(os.environ.get("CASAN_DASHBOARD_STALE_S", "3600"))
@@ -133,9 +133,17 @@ def main():
"telemetry_integrity": tel_text, "telemetry_ok": tel_rc == 0,
}
# H6 cost telemetry
prov = read_jsonl(os.path.join(logs, "level5", "provider-usage.jsonl"))
metrics = read_jsonl(os.path.join(logs, "cost", "metrics.jsonl"))
# H6 telemetry — use the same canonical sources as the Control Panel/report API.
provider_path = os.environ.get(
"CASAN_TELEMETRY_PROVIDER_LOG",
os.environ.get("CASAN_CP_PROVIDER_USAGE", os.path.join(logs, "level5", "provider-usage.jsonl")),
)
metrics_path = os.environ.get(
"CASAN_TELEMETRY_METRICS_LOG",
os.environ.get("CASAN_DASHBOARD_METRICS", os.path.join(logs, "cost", "metrics.jsonl")),
)
prov = read_jsonl(provider_path)
metrics = read_jsonl(metrics_path)
cost_rc = int(os.environ.get("CASAN_EP_COST_RC", "3") or "3")
total_tokens = sum(int(r.get("total_tokens", 0)) for r in prov if str(r.get("total_tokens", "")).isdigit())
reports["h6-cost-telemetry.json"] = {
@@ -1063,7 +1063,11 @@ def audit(job: dict, status: str) -> str:
def metric(job: dict, status: str, started: float, local_meta: dict, cloud_meta: dict) -> None:
append_jsonl(os.path.join(STATE_ROOT, "logs", "cost", "metrics.jsonl"), {
metrics_path = os.environ.get(
"CASAN_TELEMETRY_METRICS_LOG",
os.environ.get("CASAN_DASHBOARD_METRICS", os.path.join(STATE_ROOT, "logs", "cost", "metrics.jsonl")),
)
append_jsonl(metrics_path, {
"timestamp": now(),
"trace_id": job["id"],
"harness": "H6-agentops",
@@ -14,8 +14,8 @@ fi
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/casan-paths.sh"
PROJECT_ROOT="$CASAN_APP_ROOT"
LOG_DIR="$CASAN_STATE_ROOT/logs/level5"
OUT="$LOG_DIR/provider-usage.jsonl"
OUT="$CASAN_TELEMETRY_PROVIDER_LOG"
LOG_DIR="$(dirname "$OUT")"
mkdir -p "$LOG_DIR"
python - "$INPUT_JSON" "$OUT" <<'PY'
@@ -22,6 +22,85 @@ AUTH_BRIDGE_AUDIT="$AUTH_BRIDGE_DIR/model-audit.jsonl"
AUTH_BRIDGE="$ROOT/packages/casan-control-panel/scripts/provider-auth-bridge.py"
CMD="${1:-status}"
resolve_python() {
if [[ -n "${CASAN_PYTHON_BIN:-}" ]]; then
if command -v "$CASAN_PYTHON_BIN" >/dev/null 2>&1 && "$CASAN_PYTHON_BIN" --version >/dev/null 2>&1; then
printf '%s\n' "$CASAN_PYTHON_BIN"
return 0
fi
echo "CASAN_LOCAL_PYTHON_INVALID path=$CASAN_PYTHON_BIN" >&2
return 1
fi
# Prefer macOS' universal system Python over stale framework installs that may
# appear first in PATH but are terminated by Gatekeeper/Rosetta on Apple Silicon.
local candidate
for candidate in /usr/bin/python3 python3 python; do
if command -v "$candidate" >/dev/null 2>&1 && "$candidate" --version >/dev/null 2>&1; then
printf '%s\n' "$candidate"
return 0
fi
done
echo "CASAN_LOCAL_PYTHON_MISSING" >&2
return 1
}
prepare_docker_cli() {
local original_config="${DOCKER_CONFIG:-$HOME/.docker}"
local config_file="$original_config/config.json"
[[ -f "$config_file" ]] || return 0
local python_bin credential_store helper context_name docker_host fallback_config
python_bin="$(resolve_python)" || return 1
credential_store="$("$python_bin" - "$config_file" <<'PY'
import json
import sys
try:
with open(sys.argv[1], encoding="utf-8") as handle:
print(json.load(handle).get("credsStore", ""))
except (OSError, ValueError):
print("")
PY
)"
[[ -n "$credential_store" ]] || return 0
helper="$(command -v "docker-credential-$credential_store" 2>/dev/null || true)"
if [[ -n "$helper" ]] && "$python_bin" - "$helper" <<'PY' >/dev/null 2>&1
import subprocess
import sys
raise SystemExit(subprocess.run(
[sys.argv[1], "list"],
stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
).returncode)
PY
then
return 0
fi
# CASAN's local compose files use public images only. If Docker Desktop's
# credential helper is broken, isolate this process from it without changing
# ~/.docker/config.json or touching any stored login credentials.
context_name="$(docker context show)"
docker_host="$(docker context inspect --format '{{(index .Endpoints "docker").Host}}' "$context_name")"
[[ -n "$docker_host" ]] || { echo "CASAN_LOCAL_DOCKER_CONTEXT_INVALID context=$context_name" >&2; return 1; }
fallback_config="${TMPDIR:-/tmp}/casan-docker-public-$UID"
mkdir -p "$fallback_config"
if [[ -d "$original_config/cli-plugins" && ! -e "$fallback_config/cli-plugins" ]]; then
ln -s "$original_config/cli-plugins" "$fallback_config/cli-plugins"
fi
umask 077
printf '%s\n' '{"auths":{"https://index.docker.io/v1/":{},"quay.io":{}}}' > "$fallback_config/config.json"
export DOCKER_CONFIG="$fallback_config"
export DOCKER_HOST="$docker_host"
export DOCKER_BUILDKIT=0
export COMPOSE_DOCKER_CLI_BUILD=0
echo "CASAN_LOCAL_DOCKER_CREDENTIAL_FALLBACK helper=$credential_store context=$context_name builder=classic" >&2
}
cp_compose() {
if [[ -f "$AUTH_BRIDGE_TOKEN_FILE" ]]; then
export CASAN_AUTH_BRIDGE_TOKEN
@@ -35,6 +114,8 @@ cp_compose() {
}
start_auth_bridge() {
local python_bin
python_bin="$(resolve_python)" || return 1
mkdir -p "$AUTH_BRIDGE_DIR"
if [[ ! -s "$AUTH_BRIDGE_TOKEN_FILE" ]]; then
openssl rand -hex 32 > "$AUTH_BRIDGE_TOKEN_FILE"
@@ -52,7 +133,7 @@ start_auth_bridge() {
rm -f "$AUTH_BRIDGE_PID_FILE"
fi
[[ -f "$AUTH_BRIDGE" ]] || { echo "CASAN_AUTH_BRIDGE_MISSING" >&2; return 1; }
nohup python3 "$AUTH_BRIDGE" --bind 0.0.0.0 --port 20130 --token-file "$AUTH_BRIDGE_TOKEN_FILE" --audit-log "$AUTH_BRIDGE_AUDIT" > "$AUTH_BRIDGE_LOG" 2>&1 &
nohup "$python_bin" "$AUTH_BRIDGE" --bind 0.0.0.0 --port 20130 --token-file "$AUTH_BRIDGE_TOKEN_FILE" --audit-log "$AUTH_BRIDGE_AUDIT" > "$AUTH_BRIDGE_LOG" 2>&1 &
echo "$!" > "$AUTH_BRIDGE_PID_FILE"
chmod 600 "$AUTH_BRIDGE_PID_FILE" "$AUTH_BRIDGE_LOG" "$AUTH_BRIDGE_AUDIT" 2>/dev/null || true
if ! wait_url "http://127.0.0.1:20130/healthz"; then
@@ -74,6 +155,7 @@ stop_auth_bridge() {
need_docker() {
command -v docker >/dev/null 2>&1 || { echo "CASAN_LOCAL_DOCKER_MISSING" >&2; exit 1; }
prepare_docker_cli
docker compose version >/dev/null 2>&1 || { echo "CASAN_LOCAL_COMPOSE_MISSING" >&2; exit 1; }
}
@@ -48,7 +48,10 @@ OLLAMA_HOST = "127.0.0.1:11434" # the only allowed ollama endpoint
DOCKER_OLLAMA_HOST = "host.docker.internal:11434"
ALLOWED_CLOUD = {"api.anthropic.com", "api.openai.com"}
REPO_ROOT = _casan_app_root()
PROVIDER_LOG = os.path.join(REPO_ROOT, ".specify/logs/level5/provider-usage.jsonl")
PROVIDER_LOG = os.environ.get(
"CASAN_TELEMETRY_PROVIDER_LOG",
os.environ.get("CASAN_CP_PROVIDER_USAGE", os.path.join(REPO_ROOT, ".specify/logs/level5/provider-usage.jsonl")),
)
# SEC-21 (ARCH-07): a 180s-per-call timeout across many pipeline steps let a hung
# model stall a run for tens of minutes. Use a lower, configurable per-call timeout,
@@ -22,7 +22,7 @@ fi
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/casan-paths.sh"
PROJECT_ROOT="$CASAN_APP_ROOT"
OUT="${2:-$CASAN_STATE_ROOT/logs/level5/provider-usage.jsonl}"
OUT="${2:-$CASAN_TELEMETRY_PROVIDER_LOG}"
mkdir -p "$(dirname "$OUT")"
# SEC-13 (M-09): SSRF guard on the fetch URL. ALWAYS reject non-http(s) schemes
@@ -19,9 +19,9 @@ $scriptDir = Split-Path $MyInvocation.MyCommand.Path -Parent
$projectRoot = (Resolve-Path (Join-Path $scriptDir "../../..")).Path
$logDir = Join-Path $projectRoot ".specify/logs"
$traceDir = Join-Path $logDir "trace"
$metricsDir = Join-Path $logDir "cost"
$alertLog = Join-Path $projectRoot ".specify/agentops/alerts.log"
$metricsLog = Join-Path $metricsDir "metrics.jsonl"
$metricsLog = if ($env:CASAN_TELEMETRY_METRICS_LOG) { $env:CASAN_TELEMETRY_METRICS_LOG } elseif ($env:CASAN_DASHBOARD_METRICS) { $env:CASAN_DASHBOARD_METRICS } else { Join-Path $logDir "cost/metrics.jsonl" }
$metricsDir = Split-Path $metricsLog -Parent
$alertLog = if ($env:CASAN_TELEMETRY_ALERTS_LOG) { $env:CASAN_TELEMETRY_ALERTS_LOG } elseif ($env:CASAN_DASHBOARD_ALERTS) { $env:CASAN_DASHBOARD_ALERTS } else { Join-Path $projectRoot ".specify/agentops/alerts.log" }
$toolAudit = Join-Path $logDir "audit/tool-calls.jsonl"
foreach ($d in @($traceDir, $metricsDir, (Split-Path $OutputFile -Parent), (Split-Path $alertLog -Parent), (Split-Path $toolAudit -Parent))) {
@@ -10,8 +10,8 @@ param(
$scriptDir = Split-Path $MyInvocation.MyCommand.Path -Parent
$projectRoot = (Resolve-Path (Join-Path $scriptDir "../../..")).Path
$metricsLog = Join-Path $projectRoot ".specify/logs/cost/metrics.jsonl"
$alertLog = Join-Path $projectRoot ".specify/agentops/alerts.log"
$metricsLog = if ($env:CASAN_TELEMETRY_METRICS_LOG) { $env:CASAN_TELEMETRY_METRICS_LOG } elseif ($env:CASAN_DASHBOARD_METRICS) { $env:CASAN_DASHBOARD_METRICS } else { Join-Path $projectRoot ".specify/logs/cost/metrics.jsonl" }
$alertLog = if ($env:CASAN_TELEMETRY_ALERTS_LOG) { $env:CASAN_TELEMETRY_ALERTS_LOG } elseif ($env:CASAN_DASHBOARD_ALERTS) { $env:CASAN_DASHBOARD_ALERTS } else { Join-Path $projectRoot ".specify/agentops/alerts.log" }
if (!$OutputHtml) { $OutputHtml = Join-Path $projectRoot "docs/output/casan/agentops-dashboard.html" }
if (!(Test-Path (Split-Path $OutputHtml -Parent))) { New-Item -ItemType Directory -Force -Path (Split-Path $OutputHtml -Parent) | Out-Null }
@@ -2,6 +2,7 @@
from __future__ import annotations
import json
import os
import pathlib
from datetime import datetime, timezone
@@ -16,10 +17,14 @@ def _app_root(start):
ROOT = _app_root(__file__)
METRICS = ROOT / ".specify" / "logs" / "cost" / "metrics.jsonl"
METRICS = pathlib.Path(os.environ.get(
"CASAN_TELEMETRY_METRICS_LOG", os.environ.get(
"CASAN_DASHBOARD_METRICS", ROOT / ".specify" / "logs" / "cost" / "metrics.jsonl")))
FALLBACK = ROOT / ".specify" / "logs" / "level5" / "fallback.jsonl"
TOOL = ROOT / ".specify" / "logs" / "level5" / "tool-registry.jsonl"
PROVIDER = ROOT / ".specify" / "logs" / "level5" / "provider-usage.jsonl"
PROVIDER = pathlib.Path(os.environ.get(
"CASAN_TELEMETRY_PROVIDER_LOG", os.environ.get(
"CASAN_CP_PROVIDER_USAGE", ROOT / ".specify" / "logs" / "level5" / "provider-usage.jsonl")))
PROJECT_REGISTRY = ROOT / ".specify" / "level5" / "project-registry.json"
DASHBOARD = ROOT / "docs" / "output" / "casan" / "central-agentops-dashboard.html"
LEGACY_DASHBOARD = ROOT / "docs" / "output" / "casan" / "agentops-dashboard.html"