update report h6
This commit is contained in:
thanhnv
@@ -9,9 +9,10 @@ import { ProviderAuthModule } from './provider-auth/provider-auth.module.js';
|
||||
import { GoalsModule } from './goals/goals.module.js';
|
||||
import { EvidenceModule } from './evidence/evidence.module.js';
|
||||
import { SessionController } from './session/session.controller.js';
|
||||
import { ReportsModule } from './reports/reports.module.js';
|
||||
|
||||
@Module({
|
||||
imports: [TelemetryModule, SettingsModule, KillSwitchModule, ApprovalsModule, ChatModule, ProviderAuthModule, GoalsModule, EvidenceModule],
|
||||
imports: [TelemetryModule, SettingsModule, KillSwitchModule, ApprovalsModule, ChatModule, ProviderAuthModule, GoalsModule, EvidenceModule, ReportsModule],
|
||||
controllers: [HealthController, SessionController],
|
||||
})
|
||||
export class AppModule {}
|
||||
|
||||
@@ -3,8 +3,8 @@
|
||||
// Mirrors packages/casan-harness/scripts/bash/casan-paths.sh: walk UP from a start dir
|
||||
// for a marker (`.specify` state dir OR `packages/casan-harness`), so the API reads the
|
||||
// real harness telemetry whether launched from the repo, a workspace subdir, or a bundle.
|
||||
// Every path is overridable by the same CASAN_DASHBOARD_* env vars the legacy
|
||||
// dashboard-server.py honors, so console + dashboard read identical sources.
|
||||
// Canonical CASAN_TELEMETRY_* variables are shared with harness writers; legacy
|
||||
// dashboard/control-panel names remain accepted so upgrades do not split sources.
|
||||
import { existsSync, statSync } from 'node:fs';
|
||||
import { dirname, join, resolve } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
@@ -27,35 +27,74 @@ export const APP_ROOT = process.env.CASAN_APP_ROOT
|
||||
? resolve(process.env.CASAN_APP_ROOT)
|
||||
: findAppRoot(HERE);
|
||||
|
||||
const env = (k: string, fallback: string) => (process.env[k] ? resolve(process.env[k]!) : join(APP_ROOT, fallback));
|
||||
const env = (keys: string[], fallback: string) => {
|
||||
const configured = keys.map((key) => process.env[key]).find((value): value is string => Boolean(value));
|
||||
return configured ? resolve(configured) : join(APP_ROOT, fallback);
|
||||
};
|
||||
|
||||
// Runtime telemetry (regenerated) + generated reports. Names match the harness layout.
|
||||
export const PATHS = {
|
||||
metrics: env('CASAN_DASHBOARD_METRICS', '.specify/logs/cost/metrics.jsonl'),
|
||||
providerUsage: env('CASAN_CP_PROVIDER_USAGE', '.specify/logs/level5/provider-usage.jsonl'),
|
||||
audit: env('CASAN_CP_AUDIT', '.specify/logs/audit/audit.jsonl'),
|
||||
auditHead: env('CASAN_CP_AUDIT_HEAD', '.specify/logs/audit/audit-head.txt'),
|
||||
security: env('CASAN_CP_SECURITY', '.specify/logs/audit/security.jsonl'),
|
||||
toolRegistry: env('CASAN_CP_TOOL_REGISTRY', '.specify/logs/level5/tool-registry.jsonl'),
|
||||
actionGate: env('CASAN_CP_ACTION_GATE', '.specify/logs/level5/action-gate.jsonl'),
|
||||
fallback: env('CASAN_CP_FALLBACK', '.specify/logs/level5/fallback.jsonl'),
|
||||
incidents: env('CASAN_CP_INCIDENTS', '.specify/logs/level5/incidents.jsonl'),
|
||||
alerts: env('CASAN_DASHBOARD_ALERTS', '.specify/agentops/alerts.log'),
|
||||
traceDir: env('CASAN_CP_TRACE_DIR', '.specify/logs/trace'),
|
||||
traceEventDir: env('CASAN_CP_TRACE_EVENT_DIR', '.specify/logs/trace-events'),
|
||||
traceability: env('CASAN_CP_TRACEABILITY', 'docs/output/casan/traceability-matrix.json'),
|
||||
drift: env('CASAN_CP_DRIFT', 'docs/output/casan/level5-evidence/09-drift-report.json'),
|
||||
businessKpi: env('CASAN_CP_KPI', 'docs/output/casan/level5-evidence/14-business-kpi-report.json'),
|
||||
benignFp: env('CASAN_CP_BENIGN_FP', 'docs/output/casan/benign-fp-report.json'),
|
||||
scoringReport: env('CASAN_CP_SCORING_REPORT', 'docs/output/casan/phase3-real-run-scoring.md'),
|
||||
approvalInbox: env('CASAN_CP_APPROVAL_INBOX', '.specify/level5/approval-inbox.json'),
|
||||
chatAudit: env('CASAN_CP_CHAT_AUDIT', '.specify/logs/chat/chat-turns.jsonl'),
|
||||
delegationPolicy: env('CASAN_CP_DELEGATION_POLICY', 'packages/casan-harness/config/delegation-policy.yaml'),
|
||||
selfImprove: env('CASAN_CP_SELF_IMPROVE', 'packages/casan-harness/scripts/bash/self-improve.py'),
|
||||
evidencePacks: env('CASAN_CP_EVIDENCE_PACKS', 'docs/output/casan/evidence-packs'),
|
||||
metrics: env(['CASAN_TELEMETRY_METRICS_LOG', 'CASAN_DASHBOARD_METRICS'], '.specify/logs/cost/metrics.jsonl'),
|
||||
providerUsage: env(['CASAN_TELEMETRY_PROVIDER_LOG', 'CASAN_CP_PROVIDER_USAGE'], '.specify/logs/level5/provider-usage.jsonl'),
|
||||
alerts: env(['CASAN_TELEMETRY_ALERTS_LOG', 'CASAN_DASHBOARD_ALERTS'], '.specify/agentops/alerts.log'),
|
||||
audit: env(['CASAN_CP_AUDIT'], '.specify/logs/audit/audit.jsonl'),
|
||||
auditHead: env(['CASAN_CP_AUDIT_HEAD'], '.specify/logs/audit/audit-head.txt'),
|
||||
security: env(['CASAN_CP_SECURITY'], '.specify/logs/audit/security.jsonl'),
|
||||
toolRegistry: env(['CASAN_CP_TOOL_REGISTRY'], '.specify/logs/level5/tool-registry.jsonl'),
|
||||
actionGate: env(['CASAN_CP_ACTION_GATE'], '.specify/logs/level5/action-gate.jsonl'),
|
||||
fallback: env(['CASAN_CP_FALLBACK'], '.specify/logs/level5/fallback.jsonl'),
|
||||
incidents: env(['CASAN_CP_INCIDENTS'], '.specify/logs/level5/incidents.jsonl'),
|
||||
traceDir: env(['CASAN_CP_TRACE_DIR'], '.specify/logs/trace'),
|
||||
traceEventDir: env(['CASAN_CP_TRACE_EVENT_DIR'], '.specify/logs/trace-events'),
|
||||
traceability: env(['CASAN_CP_TRACEABILITY'], 'docs/output/casan/traceability-matrix.json'),
|
||||
drift: env(['CASAN_CP_DRIFT'], 'docs/output/casan/level5-evidence/09-drift-report.json'),
|
||||
businessKpi: env(['CASAN_CP_KPI'], 'docs/output/casan/level5-evidence/14-business-kpi-report.json'),
|
||||
benignFp: env(['CASAN_CP_BENIGN_FP'], 'docs/output/casan/benign-fp-report.json'),
|
||||
scoringReport: env(['CASAN_CP_SCORING_REPORT'], 'docs/output/casan/phase3-real-run-scoring.md'),
|
||||
approvalInbox: env(['CASAN_CP_APPROVAL_INBOX'], '.specify/level5/approval-inbox.json'),
|
||||
chatAudit: env(['CASAN_CP_CHAT_AUDIT'], '.specify/logs/chat/chat-turns.jsonl'),
|
||||
delegationPolicy: env(['CASAN_CP_DELEGATION_POLICY'], 'packages/casan-harness/config/delegation-policy.yaml'),
|
||||
selfImprove: env(['CASAN_CP_SELF_IMPROVE'], 'packages/casan-harness/scripts/bash/self-improve.py'),
|
||||
evidencePacks: env(['CASAN_CP_EVIDENCE_PACKS'], 'docs/output/casan/evidence-packs'),
|
||||
};
|
||||
|
||||
export const STALE_AFTER_S = Number(process.env.CASAN_DASHBOARD_STALE_S ?? 3600);
|
||||
const configuredStaleAfter = Number(process.env.CASAN_DASHBOARD_STALE_S ?? 3600);
|
||||
export const STALE_AFTER_S = Number.isFinite(configuredStaleAfter) && configuredStaleAfter >= 0
|
||||
? configuredStaleAfter
|
||||
: 3600;
|
||||
|
||||
export interface SourceFreshness {
|
||||
source: 'metrics' | 'provider' | 'alerts';
|
||||
path: string;
|
||||
present: boolean;
|
||||
updated_at: string | null;
|
||||
age_s: number | null;
|
||||
stale: boolean;
|
||||
required: boolean;
|
||||
}
|
||||
|
||||
export function sourceFreshness(
|
||||
source: SourceFreshness['source'],
|
||||
path: string,
|
||||
required: boolean,
|
||||
nowMs = Date.now(),
|
||||
): SourceFreshness {
|
||||
try {
|
||||
const mtime = statSync(path).mtime;
|
||||
const age = Math.max(0, Math.floor((nowMs - mtime.getTime()) / 1000));
|
||||
return { source, path, present: true, updated_at: mtime.toISOString(), age_s: age, stale: age > STALE_AFTER_S, required };
|
||||
} catch {
|
||||
return { source, path, present: false, updated_at: null, age_s: null, stale: required, required };
|
||||
}
|
||||
}
|
||||
|
||||
export function telemetryFreshness(nowMs = Date.now()): SourceFreshness[] {
|
||||
return [
|
||||
sourceFreshness('metrics', PATHS.metrics, true, nowMs),
|
||||
sourceFreshness('provider', PATHS.providerUsage, false, nowMs),
|
||||
sourceFreshness('alerts', PATHS.alerts, false, nowMs),
|
||||
];
|
||||
}
|
||||
|
||||
// Freshness of the primary metrics feed (mtime), used by /healthz + `stale` flags.
|
||||
export function metricsAgeSeconds(): number | null {
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
import type { H6Breakdown, H6Report } from './h6-report.js';
|
||||
|
||||
const escapeHtml = (value: unknown): string => String(value ?? '')
|
||||
.replaceAll('&', '&')
|
||||
.replaceAll('<', '<')
|
||||
.replaceAll('>', '>')
|
||||
.replaceAll('"', '"')
|
||||
.replaceAll("'", ''');
|
||||
|
||||
const number = (value: number): string => new Intl.NumberFormat('en-US').format(value);
|
||||
const money = (value: number): string => `$${value.toFixed(6)}`;
|
||||
|
||||
function breakdownRows(rows: H6Breakdown[]): string {
|
||||
if (rows.length === 0) return '<tr><td colspan="6" class="empty">No records in the selected scope.</td></tr>';
|
||||
return rows.map((row) => `<tr>
|
||||
<td>${escapeHtml(row.key)}</td><td>${number(row.runs)}</td><td>${number(row.failures)}</td>
|
||||
<td>${number(row.latency_avg_ms)} ms</td><td>${number(row.tokens)}</td><td>${money(row.cost_usd)}</td>
|
||||
</tr>`).join('');
|
||||
}
|
||||
|
||||
export function renderH6ReportHtml(report: H6Report): string {
|
||||
const verdictClass = report.verdict === 'pass' ? 'pass' : report.verdict === 'fail' ? 'fail' : report.verdict === 'attention' ? 'attention' : 'neutral';
|
||||
const scope = [
|
||||
report.scope.project ? `Project: ${report.scope.project}` : 'All projects',
|
||||
report.scope.run ? `Run: ${report.scope.run}` : 'All runs',
|
||||
report.scope.from ? `From: ${report.scope.from}` : null,
|
||||
report.scope.to ? `To: ${report.scope.to}` : null,
|
||||
].filter((value): value is string => Boolean(value));
|
||||
const findings = report.findings.length
|
||||
? report.findings.map((finding) => `<li class="${finding.severity}"><strong>${escapeHtml(finding.code)}</strong><span>${escapeHtml(finding.message)}</span></li>`).join('')
|
||||
: '<li class="info"><strong>NO_FINDINGS</strong><span>No threshold breach was detected in this scope.</span></li>';
|
||||
const sourceRows = report.evidence_sources.map((source) => `<tr>
|
||||
<td>${escapeHtml(source.source)}</td><td>${source.present ? 'present' : 'missing'}</td><td>${source.stale ? 'stale' : 'fresh'}</td>
|
||||
<td>${source.age_s === null ? '—' : `${number(source.age_s)} s`}</td><td>${number(source.records)}</td><td><code>${escapeHtml(source.path)}</code></td>
|
||||
</tr>`).join('');
|
||||
const warnings = report.data_quality.warnings.length
|
||||
? `<ul>${report.data_quality.warnings.map((warning) => `<li>${escapeHtml(warning)}</li>`).join('')}</ul>`
|
||||
: '<p>No data-quality warning.</p>';
|
||||
|
||||
return `<!doctype html>
|
||||
<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<title>${escapeHtml(report.title)} · ${escapeHtml(report.report_id)}</title>
|
||||
<style>
|
||||
:root{color-scheme:light;--ink:#172033;--muted:#64748b;--line:#dbe2ec;--paper:#fff;--wash:#f4f6fa;--indigo:#4f46e5;--teal:#0f766e}
|
||||
*{box-sizing:border-box}body{margin:0;background:var(--wash);color:var(--ink);font-family:Inter,ui-sans-serif,system-ui,-apple-system,"Segoe UI",sans-serif;line-height:1.45}
|
||||
main{max-width:1180px;margin:0 auto;padding:42px 28px 72px}.hero{position:relative;overflow:hidden;border:1px solid #cbd5e1;border-radius:20px;background:#111827;color:#f8fafc;padding:30px;box-shadow:0 24px 50px rgba(15,23,42,.14)}
|
||||
.hero:after{content:"";position:absolute;right:-90px;top:-110px;width:300px;height:300px;border-radius:999px;background:radial-gradient(circle,#6366f1 0,transparent 68%);opacity:.5}.eyebrow{font-size:11px;text-transform:uppercase;letter-spacing:.18em;color:#a5b4fc;font-weight:800}h1{font-size:34px;line-height:1.1;margin:8px 0}.scope{display:flex;gap:8px;flex-wrap:wrap;color:#cbd5e1;font-size:12px}.scope span{border:1px solid #334155;border-radius:999px;padding:6px 10px;background:#0f172a}.verdict{position:relative;z-index:1;display:inline-flex;margin-top:22px;border-radius:999px;padding:9px 14px;font-size:12px;text-transform:uppercase;letter-spacing:.12em;font-weight:900}.verdict.pass{background:#d1fae5;color:#065f46}.verdict.fail{background:#fee2e2;color:#991b1b}.verdict.attention{background:#fef3c7;color:#92400e}.verdict.neutral{background:#e2e8f0;color:#334155}
|
||||
.meta{margin-top:10px;color:#94a3b8;font-size:12px}.grid{display:grid;grid-template-columns:repeat(4,minmax(0,1fr));gap:12px;margin-top:18px}.card,.section{background:var(--paper);border:1px solid var(--line);border-radius:16px;box-shadow:0 5px 14px rgba(15,23,42,.04)}.card{padding:17px}.card span{display:block;color:var(--muted);font-size:11px;text-transform:uppercase;letter-spacing:.1em;font-weight:700}.card strong{display:block;margin-top:6px;font-size:26px}.section{padding:22px;margin-top:18px}h2{font-size:16px;margin:0 0 14px}table{border-collapse:collapse;width:100%;font-size:12px}th{text-align:left;color:var(--muted);font-size:10px;text-transform:uppercase;letter-spacing:.1em;padding:9px;border-bottom:1px solid var(--line)}td{padding:10px 9px;border-bottom:1px solid #edf1f5;vertical-align:top}.empty{color:var(--muted);text-align:center;padding:24px}code{font-size:11px;word-break:break-all}.findings{list-style:none;padding:0;margin:0;display:grid;gap:9px}.findings li{display:grid;grid-template-columns:190px 1fr;gap:12px;border-left:4px solid #94a3b8;background:#f8fafc;padding:11px 13px;border-radius:8px}.findings li.warning{border-color:#f59e0b}.findings li.critical{border-color:#ef4444}.findings span{color:#475569}.quality{border-left:4px solid var(--indigo);background:#eef2ff;padding:14px 16px;border-radius:10px;font-size:13px}.quality p,.quality ul{margin:0}.quality ul{padding-left:18px}.footer{margin-top:24px;color:var(--muted);font-size:11px;text-align:center}
|
||||
@media(max-width:800px){main{padding:20px 14px 50px}.grid{grid-template-columns:repeat(2,minmax(0,1fr))}.hero{padding:22px}h1{font-size:28px}.table-wrap{overflow:auto}.findings li{grid-template-columns:1fr}}
|
||||
@media print{body{background:#fff}main{max-width:none;padding:0}.hero,.card,.section{box-shadow:none}.hero{background:#111827!important;-webkit-print-color-adjust:exact;print-color-adjust:exact}.section{break-inside:avoid}.footer{margin-top:10px}}
|
||||
</style></head><body><main>
|
||||
<section class="hero"><div class="eyebrow">CASAN assurance dossier · contract v${report.schema_version}</div><h1>${escapeHtml(report.title)}</h1><div class="scope">${scope.map((item) => `<span>${escapeHtml(item)}</span>`).join('')}</div><div class="verdict ${verdictClass}">${escapeHtml(report.verdict)}</div><div class="meta">Generated ${escapeHtml(report.generated_at)} · ${escapeHtml(report.report_id)} · Freshness ${escapeHtml(report.freshness.status)}</div></section>
|
||||
<section class="grid"><div class="card"><span>Governed runs</span><strong>${number(report.summary.runs)}</strong></div><div class="card"><span>Failure rate</span><strong>${report.summary.failure_rate_pct}%</strong></div><div class="card"><span>P95 latency</span><strong>${number(report.summary.latency_ms.p95)} ms</strong></div><div class="card"><span>Provider tokens</span><strong>${number(report.summary.tokens.provider_total)}</strong></div><div class="card"><span>Actual provider cost</span><strong>${money(report.summary.cost_usd.provider_actual)}</strong></div><div class="card"><span>Estimated cost</span><strong>${money(report.summary.cost_usd.estimated)}</strong></div><div class="card"><span>Alerts</span><strong>${number(report.summary.alerts)}</strong></div><div class="card"><span>Retries</span><strong>${number(report.summary.retries)}</strong></div></section>
|
||||
<section class="section"><h2>Verdict findings</h2><ul class="findings">${findings}</ul></section>
|
||||
<section class="section"><h2>Evidence freshness</h2><div class="table-wrap"><table><thead><tr><th>Source</th><th>Presence</th><th>State</th><th>Age</th><th>Records</th><th>Path</th></tr></thead><tbody>${sourceRows}</tbody></table></div></section>
|
||||
<section class="section"><h2>Step breakdown</h2><div class="table-wrap"><table><thead><tr><th>Step</th><th>Runs</th><th>Failures</th><th>Avg latency</th><th>Tokens</th><th>Cost</th></tr></thead><tbody>${breakdownRows(report.details.by_step)}</tbody></table></div></section>
|
||||
<section class="section"><h2>Provider/model breakdown</h2><div class="table-wrap"><table><thead><tr><th>Provider · model</th><th>Calls</th><th>Failures</th><th>Avg latency</th><th>Tokens</th><th>Cost</th></tr></thead><tbody>${breakdownRows(report.details.by_provider)}</tbody></table></div></section>
|
||||
<section class="section"><h2>Data quality · ${escapeHtml(report.data_quality.status)}</h2><div class="quality">${warnings}</div></section>
|
||||
<div class="footer">Generated from CASAN runtime evidence. No maturity score or telemetry value is hard-coded in this document.</div>
|
||||
</main></body></html>`;
|
||||
}
|
||||
@@ -0,0 +1,307 @@
|
||||
import { BadRequestException } from '@nestjs/common';
|
||||
import { basename, relative } from 'node:path';
|
||||
import { APP_ROOT, PATHS, STALE_AFTER_S, telemetryFreshness, type SourceFreshness } from '../common/app-root.js';
|
||||
import type { HarnessReport, HarnessReportEvidenceSource, HarnessReportFinding, HarnessReportScope } from './report.contract.js';
|
||||
|
||||
type TelemetryRow = Record<string, unknown>;
|
||||
|
||||
export interface H6ReportQuery {
|
||||
project: string | null;
|
||||
from: string | null;
|
||||
to: string | null;
|
||||
run: string | null;
|
||||
limit: number;
|
||||
}
|
||||
|
||||
export interface H6ReportQueryParams {
|
||||
project?: string;
|
||||
from?: string;
|
||||
to?: string;
|
||||
run?: string;
|
||||
limit?: string;
|
||||
}
|
||||
|
||||
export interface H6ReportInput {
|
||||
metrics: TelemetryRow[];
|
||||
provider: TelemetryRow[];
|
||||
alerts: TelemetryRow[];
|
||||
sourceFreshness: SourceFreshness[];
|
||||
now: Date;
|
||||
}
|
||||
|
||||
export interface H6ReportSummary {
|
||||
runs: number;
|
||||
success: number;
|
||||
failed: number;
|
||||
degraded: number;
|
||||
failure_rate_pct: number;
|
||||
retries: number;
|
||||
latency_ms: { average: number; p50: number; p95: number; p99: number; max: number };
|
||||
tokens: { input: number; output: number; total: number; provider_total: number };
|
||||
cost_usd: { provider_actual: number; estimated: number };
|
||||
provider_calls: number;
|
||||
alerts: number;
|
||||
}
|
||||
|
||||
export interface H6Breakdown {
|
||||
key: string;
|
||||
runs: number;
|
||||
failures: number;
|
||||
latency_avg_ms: number;
|
||||
tokens: number;
|
||||
cost_usd: number;
|
||||
}
|
||||
|
||||
export interface H6ReportDetails {
|
||||
by_status: Array<{ status: string; count: number }>;
|
||||
by_step: H6Breakdown[];
|
||||
by_provider: H6Breakdown[];
|
||||
by_cost_source: Array<{ source: string; records: number; cost_usd: number }>;
|
||||
by_alert: Array<{ alert: string; count: number }>;
|
||||
recent_runs: TelemetryRow[];
|
||||
recent_alerts: TelemetryRow[];
|
||||
}
|
||||
|
||||
export type H6Report = HarnessReport<H6ReportSummary, H6ReportDetails>;
|
||||
|
||||
const safeFilter = /^[a-zA-Z0-9][a-zA-Z0-9._:-]{0,127}$/;
|
||||
const numberValue = (value: unknown): number => typeof value === 'number' && Number.isFinite(value) ? value : 0;
|
||||
const stringValue = (value: unknown, fallback: string): string => typeof value === 'string' && value.trim() ? value.trim() : fallback;
|
||||
const round = (value: number, digits = 2): number => Number(value.toFixed(digits));
|
||||
const configuredThreshold = (name: string, fallback: number): number => {
|
||||
const parsed = Number(process.env[name] ?? fallback);
|
||||
return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback;
|
||||
};
|
||||
const rowProject = (row: TelemetryRow): string => stringValue(row.project ?? row.project_id, 'default');
|
||||
const rowRun = (row: TelemetryRow): string => stringValue(row.run_id ?? row.trace_id, 'unattributed');
|
||||
const rowTime = (row: TelemetryRow): number | null => {
|
||||
const timestamp = typeof row.timestamp === 'string' ? Date.parse(row.timestamp) : Number.NaN;
|
||||
return Number.isFinite(timestamp) ? timestamp : null;
|
||||
};
|
||||
|
||||
function parseDate(value: string | undefined, label: 'from' | 'to'): string | null {
|
||||
if (!value) return null;
|
||||
const dateOnly = /^\d{4}-\d{2}-\d{2}$/.test(value);
|
||||
const normalized = dateOnly ? `${value}T${label === 'from' ? '00:00:00.000' : '23:59:59.999'}Z` : value;
|
||||
const parsed = Date.parse(normalized);
|
||||
if (!Number.isFinite(parsed)) throw new BadRequestException(`H6_REPORT_INVALID_${label.toUpperCase()}`);
|
||||
return new Date(parsed).toISOString();
|
||||
}
|
||||
|
||||
function parseFilter(value: string | undefined, label: 'project' | 'run'): string | null {
|
||||
if (!value) return null;
|
||||
if (!safeFilter.test(value)) throw new BadRequestException(`H6_REPORT_INVALID_${label.toUpperCase()}`);
|
||||
return value;
|
||||
}
|
||||
|
||||
export function parseH6ReportQuery(raw: H6ReportQueryParams): H6ReportQuery {
|
||||
const from = parseDate(raw.from, 'from');
|
||||
const to = parseDate(raw.to, 'to');
|
||||
if (from && to && Date.parse(from) > Date.parse(to)) throw new BadRequestException('H6_REPORT_INVALID_TIME_RANGE');
|
||||
const parsedLimit = Number(raw.limit ?? 50);
|
||||
if (!Number.isInteger(parsedLimit) || parsedLimit < 1 || parsedLimit > 200) {
|
||||
throw new BadRequestException('H6_REPORT_INVALID_LIMIT');
|
||||
}
|
||||
return {
|
||||
project: parseFilter(raw.project, 'project'),
|
||||
from,
|
||||
to,
|
||||
run: parseFilter(raw.run, 'run'),
|
||||
limit: parsedLimit,
|
||||
};
|
||||
}
|
||||
|
||||
function filtered(rows: TelemetryRow[], query: H6ReportQuery): TelemetryRow[] {
|
||||
const fromMs = query.from ? Date.parse(query.from) : null;
|
||||
const toMs = query.to ? Date.parse(query.to) : null;
|
||||
return rows.filter((row) => {
|
||||
if (query.project && rowProject(row) !== query.project) return false;
|
||||
if (query.run && rowRun(row) !== query.run) return false;
|
||||
const timestamp = rowTime(row);
|
||||
if (fromMs !== null && (timestamp === null || timestamp < fromMs)) return false;
|
||||
if (toMs !== null && (timestamp === null || timestamp > toMs)) return false;
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
function percentile(values: number[], percentileValue: number): number {
|
||||
if (values.length === 0) return 0;
|
||||
const sorted = [...values].sort((left, right) => left - right);
|
||||
const index = Math.max(0, Math.ceil((percentileValue / 100) * sorted.length) - 1);
|
||||
return sorted[index];
|
||||
}
|
||||
|
||||
function sum(rows: TelemetryRow[], key: string): number {
|
||||
return rows.reduce((total, row) => total + numberValue(row[key]), 0);
|
||||
}
|
||||
|
||||
function grouped(rows: TelemetryRow[], keyOf: (row: TelemetryRow) => string): H6Breakdown[] {
|
||||
const groups = new Map<string, TelemetryRow[]>();
|
||||
for (const row of rows) {
|
||||
const key = keyOf(row);
|
||||
groups.set(key, [...(groups.get(key) ?? []), row]);
|
||||
}
|
||||
return [...groups.entries()].map(([key, records]) => {
|
||||
const latencies = records.map((row) => numberValue(row.latency_ms)).filter((value) => value > 0);
|
||||
return {
|
||||
key,
|
||||
runs: records.length,
|
||||
failures: records.filter((row) => stringValue(row.status, 'unknown') === 'failed').length,
|
||||
latency_avg_ms: latencies.length ? Math.round(latencies.reduce((total, value) => total + value, 0) / latencies.length) : 0,
|
||||
tokens: sum(records, 'total_tokens'),
|
||||
cost_usd: round(sum(records, records.some((row) => row.cost_usd !== undefined) ? 'cost_usd' : 'cost_estimate'), 6),
|
||||
};
|
||||
}).sort((left, right) => right.runs - left.runs || left.key.localeCompare(right.key));
|
||||
}
|
||||
|
||||
function counted(rows: TelemetryRow[], keyOf: (row: TelemetryRow) => string): Array<{ key: string; count: number }> {
|
||||
const counts = new Map<string, number>();
|
||||
for (const row of rows) {
|
||||
const key = keyOf(row);
|
||||
counts.set(key, (counts.get(key) ?? 0) + 1);
|
||||
}
|
||||
return [...counts.entries()].map(([key, countValue]) => ({ key, count: countValue }))
|
||||
.sort((left, right) => right.count - left.count || left.key.localeCompare(right.key));
|
||||
}
|
||||
|
||||
function alertTypes(row: TelemetryRow): string[] {
|
||||
const structured = row.body && typeof row.body === 'object' ? (row.body as Record<string, unknown>)['alert.type'] : undefined;
|
||||
if (typeof structured === 'string') return [structured];
|
||||
if (Array.isArray(row.alerts)) return row.alerts.filter((value): value is string => typeof value === 'string');
|
||||
return [];
|
||||
}
|
||||
|
||||
function evidenceSources(input: H6ReportInput): HarnessReportEvidenceSource[] {
|
||||
const counts: Record<SourceFreshness['source'], number> = {
|
||||
metrics: input.metrics.length,
|
||||
provider: input.provider.length,
|
||||
alerts: input.alerts.length,
|
||||
};
|
||||
return input.sourceFreshness.map((source) => {
|
||||
const appRelativePath = relative(APP_ROOT, source.path) || '.';
|
||||
return {
|
||||
...source,
|
||||
path: appRelativePath.startsWith('..') ? `[external]/${basename(source.path)}` : appRelativePath,
|
||||
records: counts[source.source],
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export function buildH6Report(input: H6ReportInput, query: H6ReportQuery): H6Report {
|
||||
const metrics = filtered(input.metrics, query);
|
||||
const provider = filtered(input.provider, query);
|
||||
const alerts = filtered(input.alerts, query);
|
||||
const latencies = metrics.map((row) => numberValue(row.latency_ms)).filter((value) => value > 0);
|
||||
const failed = metrics.filter((row) => stringValue(row.status, 'unknown') === 'failed').length;
|
||||
const degraded = metrics.filter((row) => stringValue(row.status, 'unknown') === 'degraded').length;
|
||||
const success = metrics.filter((row) => ['success', 'pass', 'passed'].includes(stringValue(row.status, 'unknown'))).length;
|
||||
const sourceEvidence = evidenceSources(input);
|
||||
const primary = sourceEvidence.find((source) => source.source === 'metrics');
|
||||
const failureRate = metrics.length ? round((failed / metrics.length) * 100, 1) : 0;
|
||||
const p95 = percentile(latencies, 95);
|
||||
const failureThreshold = configuredThreshold('CASAN_H6_FAILURE_RATE_THRESHOLD_PCT', 5);
|
||||
const p95Threshold = configuredThreshold('CASAN_H6_P95_LATENCY_THRESHOLD_MS', 5000);
|
||||
const findings: HarnessReportFinding[] = [];
|
||||
const warnings: string[] = [];
|
||||
|
||||
if (!primary?.present) findings.push({ severity: 'critical', code: 'METRICS_MISSING', message: 'The required runtime metrics source is missing.' });
|
||||
else if (primary.stale) findings.push({ severity: 'warning', code: 'METRICS_STALE', message: 'The primary runtime metrics source is stale.', metric: 'age_s', value: primary.age_s ?? 'unknown', threshold: STALE_AFTER_S });
|
||||
if (failureRate > failureThreshold) findings.push({ severity: 'critical', code: 'FAILURE_RATE_BREACH', message: 'Failure rate exceeds the configured H6 threshold.', metric: 'failure_rate_pct', value: failureRate, threshold: failureThreshold });
|
||||
else if (failed > 0) findings.push({ severity: 'warning', code: 'FAILURES_PRESENT', message: `${failed} failed run(s) are present in the selected scope.`, metric: 'failed', value: failed });
|
||||
if (p95 > p95Threshold) findings.push({ severity: 'warning', code: 'P95_LATENCY_BREACH', message: 'P95 latency exceeds the configured H6 threshold.', metric: 'p95_latency_ms', value: p95, threshold: p95Threshold });
|
||||
const alertFingerprints = new Map<string, Set<string>>();
|
||||
for (const row of [...metrics, ...alerts]) {
|
||||
for (const alert of alertTypes(row)) {
|
||||
const run = rowRun(row);
|
||||
const fingerprint = `${run === 'unattributed' ? stringValue(row.timestamp, 'unknown-time') : run}:${alert}`;
|
||||
const known = alertFingerprints.get(alert) ?? new Set<string>();
|
||||
known.add(fingerprint);
|
||||
alertFingerprints.set(alert, known);
|
||||
}
|
||||
}
|
||||
const alertCounts = [...alertFingerprints.entries()].map(([alert, fingerprints]) => ({ alert, count: fingerprints.size }));
|
||||
const alertCount = alertCounts.reduce((total, entry) => total + entry.count, 0);
|
||||
if (alertCount > 0) findings.push({ severity: 'warning', code: 'ALERTS_PRESENT', message: `${alertCount} alert signal(s) require review.`, metric: 'alerts', value: alertCount });
|
||||
if (degraded > 0) findings.push({ severity: 'warning', code: 'DEGRADED_RUNS_PRESENT', message: `${degraded} degraded run(s) are present in the selected scope.`, metric: 'degraded', value: degraded });
|
||||
if (provider.length === 0) warnings.push('No provider usage records matched the selected scope; token and actual-cost breakdown may be incomplete.');
|
||||
for (const source of sourceEvidence) {
|
||||
if (source.source === 'metrics') continue;
|
||||
if (!source.present) warnings.push(`Optional ${source.source} telemetry source is missing; its breakdown is unavailable.`);
|
||||
else if (source.stale) warnings.push(`Optional ${source.source} telemetry source is stale; its breakdown may not reflect recent activity.`);
|
||||
}
|
||||
const estimatedRecords = metrics.filter((row) => stringValue(row.cost_source, 'unknown').includes('estimate')).length;
|
||||
if (estimatedRecords > 0) warnings.push(`${estimatedRecords} metric record(s) use estimated rather than provider-reported cost.`);
|
||||
const unattributed = [...input.metrics, ...input.provider].filter((row) => row.project === undefined && row.project_id === undefined).length;
|
||||
if (unattributed > 0) warnings.push(`${unattributed} source record(s) have no project attribute and are classified as project "default".`);
|
||||
|
||||
const critical = findings.some((finding) => finding.severity === 'critical');
|
||||
const warning = findings.some((finding) => finding.severity === 'warning');
|
||||
const verdict = metrics.length === 0 && provider.length === 0 ? 'no_data' : critical ? 'fail' : warning ? 'attention' : 'pass';
|
||||
const freshnessStatus = !primary?.present ? 'missing' : primary.stale ? 'stale' : 'live';
|
||||
const availableProjects = [...new Set([...input.metrics, ...input.provider, ...input.alerts].map(rowProject))].sort();
|
||||
const availableRuns = [...new Set([...input.metrics, ...input.provider, ...input.alerts].map(rowRun).filter((run) => run !== 'unattributed'))].sort();
|
||||
const costSources = grouped(metrics, (row) => stringValue(row.cost_source, 'unknown')).map((entry) => ({
|
||||
source: entry.key,
|
||||
records: entry.runs,
|
||||
cost_usd: entry.cost_usd,
|
||||
}));
|
||||
const scope: HarnessReportScope = { project: query.project, from: query.from, to: query.to, run: query.run };
|
||||
|
||||
return {
|
||||
schema_version: 1,
|
||||
report_id: `H6-${input.now.toISOString().replace(/[-:.TZ]/g, '').slice(0, 14)}`,
|
||||
harness: 'H6',
|
||||
title: 'H6 · AgentOps Report',
|
||||
description: 'Evidence-backed runtime, token, cost, failure, retry and alert telemetry.',
|
||||
generated_at: input.now.toISOString(),
|
||||
scope,
|
||||
verdict,
|
||||
verdict_reasons: findings.map((finding) => finding.code),
|
||||
freshness: { status: freshnessStatus, stale_after_s: STALE_AFTER_S, primary_age_s: primary?.age_s ?? null, sources: sourceEvidence },
|
||||
summary: {
|
||||
runs: metrics.length,
|
||||
success,
|
||||
failed,
|
||||
degraded,
|
||||
failure_rate_pct: failureRate,
|
||||
retries: sum(metrics, 'retry_count'),
|
||||
latency_ms: {
|
||||
average: latencies.length ? Math.round(latencies.reduce((total, value) => total + value, 0) / latencies.length) : 0,
|
||||
p50: percentile(latencies, 50),
|
||||
p95,
|
||||
p99: percentile(latencies, 99),
|
||||
max: latencies.length ? Math.max(...latencies) : 0,
|
||||
},
|
||||
tokens: { input: sum(metrics, 'input_tokens'), output: sum(metrics, 'output_tokens'), total: sum(metrics, 'total_tokens'), provider_total: sum(provider, 'total_tokens') },
|
||||
cost_usd: { provider_actual: round(sum(provider, 'cost_usd'), 6), estimated: round(sum(metrics, 'cost_estimate'), 6) },
|
||||
provider_calls: provider.length,
|
||||
alerts: alertCount,
|
||||
},
|
||||
thresholds: { failure_rate_pct: failureThreshold, p95_latency_ms: p95Threshold, freshness_age_s: STALE_AFTER_S },
|
||||
findings,
|
||||
evidence_sources: sourceEvidence,
|
||||
data_quality: { status: !primary?.present ? 'insufficient' : warnings.length ? 'partial' : 'complete', warnings },
|
||||
available_filters: { projects: availableProjects, runs: availableRuns },
|
||||
details: {
|
||||
by_status: counted(metrics, (row) => stringValue(row.status, 'unknown')).map(({ key, count: countValue }) => ({ status: key, count: countValue })),
|
||||
by_step: grouped(metrics, (row) => stringValue(row.step, 'unknown-step')),
|
||||
by_provider: grouped(provider, (row) => `${stringValue(row.provider, 'unknown-provider')} · ${stringValue(row.model, 'unknown-model')}`),
|
||||
by_cost_source: costSources,
|
||||
by_alert: alertCounts.sort((left, right) => right.count - left.count || left.alert.localeCompare(right.alert)),
|
||||
recent_runs: [...metrics].slice(-query.limit).reverse(),
|
||||
recent_alerts: [...alerts].slice(-query.limit).reverse(),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function defaultH6ReportInput(now = new Date()): H6ReportInput {
|
||||
return {
|
||||
metrics: [],
|
||||
provider: [],
|
||||
alerts: [],
|
||||
sourceFreshness: telemetryFreshness(now.getTime()),
|
||||
now,
|
||||
};
|
||||
}
|
||||
|
||||
export const H6_REPORT_PATHS = PATHS;
|
||||
@@ -0,0 +1,83 @@
|
||||
import type { SourceFreshness } from '../common/app-root.js';
|
||||
|
||||
export const HARNESS_REPORT_DEFINITIONS = [
|
||||
{ id: 'H1', slug: 'context', title: 'H1 · Context', description: 'Context selection, prompt contract and risk classification' },
|
||||
{ id: 'H2', slug: 'tool', title: 'H2 · Tool', description: 'Allowlisted tools, actions and execution boundaries' },
|
||||
{ id: 'H3', slug: 'evaluation', title: 'H3 · Evaluation', description: 'Quality evaluation, traceability and acceptance evidence' },
|
||||
{ id: 'H4', slug: 'security', title: 'H4 · Security', description: 'Input, output and artifact security controls' },
|
||||
{ id: 'H5', slug: 'governance', title: 'H5 · Governance', description: 'Policy decisions, approvals and audit integrity' },
|
||||
{ id: 'H6', slug: 'agentops', title: 'H6 · AgentOps', description: 'Runtime, token, cost, failure and alert telemetry' },
|
||||
{ id: 'H7', slug: 'orchestration', title: 'H7 · Orchestration', description: 'Final outcome, certification and rollback evidence' },
|
||||
] as const;
|
||||
|
||||
export type HarnessReportId = typeof HARNESS_REPORT_DEFINITIONS[number]['id'];
|
||||
export type HarnessReportVerdict = 'pass' | 'attention' | 'fail' | 'no_data';
|
||||
export type ReportExportFormat = 'json' | 'html';
|
||||
|
||||
export interface HarnessReportScope {
|
||||
project: string | null;
|
||||
from: string | null;
|
||||
to: string | null;
|
||||
run: string | null;
|
||||
}
|
||||
|
||||
export interface HarnessReportFinding {
|
||||
severity: 'info' | 'warning' | 'critical';
|
||||
code: string;
|
||||
message: string;
|
||||
metric?: string;
|
||||
value?: number | string;
|
||||
threshold?: number | string;
|
||||
}
|
||||
|
||||
export interface HarnessReportEvidenceSource extends SourceFreshness {
|
||||
records: number;
|
||||
}
|
||||
|
||||
export interface HarnessReport<TSummary, TDetails> {
|
||||
schema_version: 1;
|
||||
report_id: string;
|
||||
harness: HarnessReportId;
|
||||
title: string;
|
||||
description: string;
|
||||
generated_at: string;
|
||||
scope: HarnessReportScope;
|
||||
verdict: HarnessReportVerdict;
|
||||
verdict_reasons: string[];
|
||||
freshness: {
|
||||
status: 'live' | 'stale' | 'missing';
|
||||
stale_after_s: number;
|
||||
primary_age_s: number | null;
|
||||
sources: HarnessReportEvidenceSource[];
|
||||
};
|
||||
summary: TSummary;
|
||||
thresholds: Record<string, number | string | boolean>;
|
||||
findings: HarnessReportFinding[];
|
||||
evidence_sources: HarnessReportEvidenceSource[];
|
||||
data_quality: {
|
||||
status: 'complete' | 'partial' | 'insufficient';
|
||||
warnings: string[];
|
||||
};
|
||||
available_filters: {
|
||||
projects: string[];
|
||||
runs: string[];
|
||||
};
|
||||
details: TDetails;
|
||||
}
|
||||
|
||||
export interface HarnessReportCatalogEntry {
|
||||
id: HarnessReportId;
|
||||
slug: string;
|
||||
title: string;
|
||||
description: string;
|
||||
contract_version: 1;
|
||||
endpoint: string;
|
||||
availability: 'implemented' | 'contract_ready';
|
||||
}
|
||||
|
||||
export const HARNESS_REPORT_CATALOG: HarnessReportCatalogEntry[] = HARNESS_REPORT_DEFINITIONS.map((definition) => ({
|
||||
...definition,
|
||||
contract_version: 1,
|
||||
endpoint: `/api/v1/reports/${definition.id.toLowerCase()}`,
|
||||
availability: definition.id === 'H6' ? 'implemented' : 'contract_ready',
|
||||
}));
|
||||
@@ -0,0 +1,37 @@
|
||||
import { BadRequestException, Controller, Get, Header, Inject, Query, Res } from '@nestjs/common';
|
||||
import type { Response } from 'express';
|
||||
import { ok } from '../common/api-response.js';
|
||||
import { parseH6ReportQuery, type H6ReportQueryParams } from './h6-report.js';
|
||||
import { ReportsService } from './reports.service.js';
|
||||
|
||||
interface H6QueryParams extends H6ReportQueryParams {
|
||||
format?: string;
|
||||
}
|
||||
|
||||
@Controller('api/v1/reports')
|
||||
export class ReportsController {
|
||||
constructor(@Inject(ReportsService) private readonly reports: ReportsService) {}
|
||||
|
||||
@Get()
|
||||
catalog() {
|
||||
return ok(this.reports.catalog());
|
||||
}
|
||||
|
||||
@Get('h6')
|
||||
h6(@Query() raw: H6QueryParams) {
|
||||
return ok(this.reports.h6(parseH6ReportQuery(raw)));
|
||||
}
|
||||
|
||||
@Get('h6/export')
|
||||
@Header('Cache-Control', 'no-store')
|
||||
exportH6(@Query() raw: H6QueryParams, @Res() response: Response) {
|
||||
const format = raw.format ?? 'json';
|
||||
if (format !== 'json' && format !== 'html') throw new BadRequestException('H6_REPORT_INVALID_FORMAT');
|
||||
const report = this.reports.h6(parseH6ReportQuery(raw));
|
||||
const body = this.reports.serializeH6(report, format);
|
||||
const stamp = report.generated_at.slice(0, 10);
|
||||
response.type(format === 'html' ? 'text/html; charset=utf-8' : 'application/json; charset=utf-8');
|
||||
response.setHeader('Content-Disposition', `attachment; filename="casan-h6-report-${stamp}.${format}"`);
|
||||
response.send(body);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { ReportsController } from './reports.controller.js';
|
||||
import { ReportsService } from './reports.service.js';
|
||||
|
||||
@Module({
|
||||
controllers: [ReportsController],
|
||||
providers: [ReportsService],
|
||||
exports: [ReportsService],
|
||||
})
|
||||
export class ReportsModule {}
|
||||
@@ -0,0 +1,28 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { PATHS, telemetryFreshness } from '../common/app-root.js';
|
||||
import { readJsonl } from '../telemetry/telemetry.reader.js';
|
||||
import { buildH6Report, type H6Report, type H6ReportQuery } from './h6-report.js';
|
||||
import { renderH6ReportHtml } from './h6-report.html.js';
|
||||
import { HARNESS_REPORT_CATALOG } from './report.contract.js';
|
||||
|
||||
@Injectable()
|
||||
export class ReportsService {
|
||||
catalog() {
|
||||
return { schema_version: 1, reports: HARNESS_REPORT_CATALOG };
|
||||
}
|
||||
|
||||
h6(query: H6ReportQuery): H6Report {
|
||||
const now = new Date();
|
||||
return buildH6Report({
|
||||
metrics: readJsonl(PATHS.metrics),
|
||||
provider: readJsonl(PATHS.providerUsage),
|
||||
alerts: readJsonl(PATHS.alerts),
|
||||
sourceFreshness: telemetryFreshness(now.getTime()),
|
||||
now,
|
||||
}, query);
|
||||
}
|
||||
|
||||
serializeH6(report: H6Report, format: 'json' | 'html'): string {
|
||||
return format === 'html' ? renderH6ReportHtml(report) : `${JSON.stringify(report, null, 2)}\n`;
|
||||
}
|
||||
}
|
||||
@@ -5,7 +5,7 @@ import { existsSync, readFileSync, statSync } from 'node:fs';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { join, relative } from 'node:path';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { APP_ROOT, PATHS, isStale, metricsAgeSeconds, STALE_AFTER_S } from '../common/app-root.js';
|
||||
import { APP_ROOT, PATHS, isStale, metricsAgeSeconds, STALE_AFTER_S, telemetryFreshness } from '../common/app-root.js';
|
||||
import { readJsonl, readJson, readHead, readTrace } from './telemetry.reader.js';
|
||||
|
||||
type Row = Record<string, any>;
|
||||
@@ -217,7 +217,7 @@ export function buildChatLoopWidget(
|
||||
export class TelemetryService {
|
||||
private freshness() {
|
||||
const age = metricsAgeSeconds();
|
||||
return { stale: isStale(), age_s: age, stale_after_s: STALE_AFTER_S };
|
||||
return { stale: isStale(), age_s: age, stale_after_s: STALE_AFTER_S, sources: telemetryFreshness() };
|
||||
}
|
||||
|
||||
overview() {
|
||||
|
||||
Reference in New Issue
Block a user