feat: add production assurance dashboard flow
This commit is contained in:
thanhnv
@@ -1,7 +1,8 @@
|
||||
# CASAN Ops Console (Plan-13 Track 1/2/3/4 + Command Center) — Control Panel
|
||||
|
||||
Real **NestJS API + React UI** that surfaces CASAN harness telemetry and governed settings
|
||||
management. This is the Level-3 `casan-platform` **Control Panel** component.
|
||||
management. This is the `casan-platform` **Control Plane** preview. “Platform”
|
||||
is a product edition, not a CASAN Maturity L3 claim.
|
||||
Monitoring remains read-only ("Đọc ≠ Ghi"). Settings writes go through RBAC and the
|
||||
harness-owned governance CLI; the UI never writes harness files directly or bypasses a gate.
|
||||
|
||||
@@ -11,6 +12,18 @@ frontend/ React + Vite + Tailwind + TanStack Query Ops Console + Settings/Appro
|
||||
```
|
||||
|
||||
## Run (local)
|
||||
|
||||
Preferred golden path from an adopted project:
|
||||
|
||||
```bash
|
||||
casan dashboard start # starts API + UI, enrolls the local deep link
|
||||
casan dashboard status
|
||||
casan view # opens the most recent prompt trace
|
||||
casan dashboard stop
|
||||
```
|
||||
|
||||
Manual developer mode:
|
||||
|
||||
```bash
|
||||
npm install # from repo root (picks up the workspaces)
|
||||
npm run console:api # NestJS API → http://127.0.0.1:3010/api/v1
|
||||
@@ -33,6 +46,24 @@ Harness reports:
|
||||
no maturity score is hard-coded.
|
||||
- `/reports/h6` — UI report view with project, time-range and run filters plus source-level
|
||||
freshness and data-quality warnings.
|
||||
- `GET /api/v1/reports/run/:traceId` and `/export?format=html|json` — complete
|
||||
per-prompt H1–H7 assurance receipt with truthful H6 availability.
|
||||
- `POST /api/v1/ingest/turn` — optional central ingestion. Disabled unless
|
||||
`CASAN_CP_INGEST_TOKEN` is set; requests require a five-minute timestamp and
|
||||
HMAC-SHA256 signature. Raw prompt/tool content keys are rejected.
|
||||
|
||||
Core always writes a sanitized pending envelope locally first. Central delivery
|
||||
is asynchronous and never delays or changes a prompt verdict:
|
||||
|
||||
```bash
|
||||
export CASAN_CONTROL_PLANE_TOKEN='use-a-secret-manager-in-production'
|
||||
casan init \
|
||||
--dashboard-url https://casan.example \
|
||||
--ingest-url https://casan.example/api/v1/ingest/turn
|
||||
```
|
||||
|
||||
The server receives the same secret as `CASAN_CP_INGEST_TOKEN`. The secret name,
|
||||
not the secret value, is stored in project configuration.
|
||||
|
||||
Metrics export: `GET /api/v1/metrics` provides Prometheus text exposition for
|
||||
aggregate freshness, run/failure/cost/token and H4/H5/action/incident counters.
|
||||
|
||||
@@ -10,9 +10,10 @@ import { GoalsModule } from './goals/goals.module.js';
|
||||
import { EvidenceModule } from './evidence/evidence.module.js';
|
||||
import { SessionController } from './session/session.controller.js';
|
||||
import { ReportsModule } from './reports/reports.module.js';
|
||||
import { IngestModule } from './ingest/ingest.module.js';
|
||||
|
||||
@Module({
|
||||
imports: [TelemetryModule, SettingsModule, KillSwitchModule, ApprovalsModule, ChatModule, ProviderAuthModule, GoalsModule, EvidenceModule, ReportsModule],
|
||||
imports: [TelemetryModule, SettingsModule, KillSwitchModule, ApprovalsModule, ChatModule, ProviderAuthModule, GoalsModule, EvidenceModule, ReportsModule, IngestModule],
|
||||
controllers: [HealthController, SessionController],
|
||||
})
|
||||
export class AppModule {}
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
import { Body, Controller, Headers, Inject, Post } from '@nestjs/common';
|
||||
import { ok } from '../common/api-response.js';
|
||||
import { IngestService } from './ingest.service.js';
|
||||
|
||||
@Controller('api/v1/ingest')
|
||||
export class IngestController {
|
||||
constructor(@Inject(IngestService) private readonly ingestService: IngestService) {}
|
||||
|
||||
@Post('turn')
|
||||
turn(
|
||||
@Body() body: unknown,
|
||||
@Headers('x-casan-timestamp') timestamp?: string,
|
||||
@Headers('x-casan-signature') signature?: string,
|
||||
) {
|
||||
return ok(this.ingestService.ingest(body, timestamp, signature));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { IngestController } from './ingest.controller.js';
|
||||
import { IngestService } from './ingest.service.js';
|
||||
|
||||
@Module({
|
||||
controllers: [IngestController],
|
||||
providers: [IngestService],
|
||||
})
|
||||
export class IngestModule {}
|
||||
@@ -0,0 +1,119 @@
|
||||
import { createHmac, timingSafeEqual } from 'node:crypto';
|
||||
import { appendFileSync, existsSync, mkdirSync, renameSync, writeFileSync } from 'node:fs';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { BadRequestException, ForbiddenException, Injectable, ServiceUnavailableException } from '@nestjs/common';
|
||||
import { APP_ROOT, PATHS } from '../common/app-root.js';
|
||||
|
||||
type Row = Record<string, unknown>;
|
||||
|
||||
export interface IngestEnvelope {
|
||||
schema_version: 1;
|
||||
sent_at: string;
|
||||
project_id: string;
|
||||
trace_id: string;
|
||||
receipt: Row;
|
||||
metric: Row;
|
||||
trace: Row;
|
||||
events: Row[];
|
||||
}
|
||||
|
||||
const SAFE_ID = /^[a-zA-Z0-9][a-zA-Z0-9._:-]{0,127}$/;
|
||||
const FORBIDDEN_KEYS = new Set([
|
||||
'prompt', 'raw_prompt', 'assistant_summary', 'tool_input', 'tool_output',
|
||||
'result_content', 'secret', 'password', 'authorization',
|
||||
]);
|
||||
|
||||
export function stableJson(value: unknown): string {
|
||||
if (Array.isArray(value)) return `[${value.map(stableJson).join(',')}]`;
|
||||
if (value && typeof value === 'object') {
|
||||
const record = value as Row;
|
||||
return `{${Object.keys(record).sort().map((key) => `${JSON.stringify(key)}:${stableJson(record[key])}`).join(',')}}`;
|
||||
}
|
||||
return JSON.stringify(value) ?? 'null';
|
||||
}
|
||||
|
||||
export function expectedSignature(token: string, timestamp: string, body: unknown): string {
|
||||
return `sha256=${createHmac('sha256', token).update(`${timestamp}.${stableJson(body)}`).digest('hex')}`;
|
||||
}
|
||||
|
||||
function containsForbiddenKey(value: unknown): boolean {
|
||||
if (Array.isArray(value)) return value.some(containsForbiddenKey);
|
||||
if (!value || typeof value !== 'object') return false;
|
||||
return Object.entries(value as Row).some(([key, nested]) => (
|
||||
FORBIDDEN_KEYS.has(key.toLowerCase()) || containsForbiddenKey(nested)
|
||||
));
|
||||
}
|
||||
|
||||
function safeEqual(left: string, right: string): boolean {
|
||||
const leftBuffer = Buffer.from(left);
|
||||
const rightBuffer = Buffer.from(right);
|
||||
return leftBuffer.length === rightBuffer.length && timingSafeEqual(leftBuffer, rightBuffer);
|
||||
}
|
||||
|
||||
function atomicJson(path: string, value: unknown): void {
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
const temporary = `${path}.${process.pid}.tmp`;
|
||||
writeFileSync(temporary, `${JSON.stringify(value, null, 2)}\n`, { encoding: 'utf8', mode: 0o600 });
|
||||
renameSync(temporary, path);
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class IngestService {
|
||||
ingest(raw: unknown, timestamp: string | undefined, signature: string | undefined) {
|
||||
const token = process.env.CASAN_CP_INGEST_TOKEN;
|
||||
if (!token) throw new ServiceUnavailableException('CASAN_INGEST_DISABLED');
|
||||
const epoch = Number(timestamp);
|
||||
if (!Number.isInteger(epoch) || Math.abs(Math.floor(Date.now() / 1000) - epoch) > 300) {
|
||||
throw new ForbiddenException('CASAN_INGEST_TIMESTAMP_INVALID');
|
||||
}
|
||||
const expected = expectedSignature(token, String(timestamp), raw);
|
||||
if (!signature || !safeEqual(signature, expected)) {
|
||||
throw new ForbiddenException('CASAN_INGEST_SIGNATURE_INVALID');
|
||||
}
|
||||
if (!raw || typeof raw !== 'object' || Array.isArray(raw)) {
|
||||
throw new BadRequestException('CASAN_INGEST_INVALID_ENVELOPE');
|
||||
}
|
||||
const envelope = raw as Partial<IngestEnvelope>;
|
||||
if (
|
||||
envelope.schema_version !== 1
|
||||
|| typeof envelope.project_id !== 'string'
|
||||
|| typeof envelope.trace_id !== 'string'
|
||||
|| !SAFE_ID.test(envelope.project_id)
|
||||
|| !SAFE_ID.test(envelope.trace_id)
|
||||
|| !envelope.metric
|
||||
|| !envelope.trace
|
||||
|| !Array.isArray(envelope.events)
|
||||
|| envelope.events.length > 2000
|
||||
) throw new BadRequestException('CASAN_INGEST_INVALID_ENVELOPE');
|
||||
if (containsForbiddenKey(envelope)) {
|
||||
throw new BadRequestException('CASAN_INGEST_RAW_CONTENT_FORBIDDEN');
|
||||
}
|
||||
if (
|
||||
envelope.metric.trace_id !== envelope.trace_id
|
||||
|| envelope.trace.trace_id !== envelope.trace_id
|
||||
|| envelope.events.some((event) => event.trace_id !== envelope.trace_id)
|
||||
) throw new BadRequestException('CASAN_INGEST_TRACE_MISMATCH');
|
||||
|
||||
const tracePath = join(PATHS.traceDir, `agentic-${envelope.trace_id}.json`);
|
||||
if (existsSync(tracePath)) {
|
||||
return { accepted: true, duplicate: true, trace_id: envelope.trace_id };
|
||||
}
|
||||
|
||||
mkdirSync(dirname(PATHS.metrics), { recursive: true });
|
||||
appendFileSync(PATHS.metrics, `${JSON.stringify(envelope.metric)}\n`, 'utf8');
|
||||
atomicJson(tracePath, envelope.trace);
|
||||
if (envelope.events.length > 0) {
|
||||
const eventPath = join(PATHS.traceEventDir, `${envelope.trace_id}.jsonl`);
|
||||
mkdirSync(dirname(eventPath), { recursive: true });
|
||||
appendFileSync(eventPath, `${envelope.events.map((event) => JSON.stringify(event)).join('\n')}\n`, 'utf8');
|
||||
}
|
||||
const receiptPath = join(APP_ROOT, '.specify', 'state', 'ingested', `${envelope.trace_id}.json`);
|
||||
atomicJson(receiptPath, envelope.receipt ?? {});
|
||||
return {
|
||||
accepted: true,
|
||||
duplicate: false,
|
||||
project_id: envelope.project_id,
|
||||
trace_id: envelope.trace_id,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -14,7 +14,7 @@ function breakdownRows(rows: H6Breakdown[]): string {
|
||||
if (rows.length === 0) return '<tr><td colspan="6" class="empty">No records in the selected scope.</td></tr>';
|
||||
return rows.map((row) => `<tr>
|
||||
<td>${escapeHtml(row.key)}</td><td>${number(row.runs)}</td><td>${number(row.failures)}</td>
|
||||
<td>${number(row.latency_avg_ms)} ms</td><td>${number(row.tokens)}</td><td>${money(row.cost_usd)}</td>
|
||||
<td>${number(row.latency_avg_ms)} ms</td><td>${row.tokens === null ? 'Unavailable' : number(row.tokens)}</td><td>${row.cost_usd === null ? 'Unavailable' : money(row.cost_usd)}</td>
|
||||
</tr>`).join('');
|
||||
}
|
||||
|
||||
@@ -50,7 +50,7 @@ main{max-width:1180px;margin:0 auto;padding:42px 28px 72px}.hero{position:relati
|
||||
@media print{body{background:#fff}main{max-width:none;padding:0}.hero,.card,.section{box-shadow:none}.hero{background:#111827!important;-webkit-print-color-adjust:exact;print-color-adjust:exact}.section{break-inside:avoid}.footer{margin-top:10px}}
|
||||
</style></head><body><main>
|
||||
<section class="hero"><div class="eyebrow">CASAN assurance dossier · contract v${report.schema_version}</div><h1>${escapeHtml(report.title)}</h1><div class="scope">${scope.map((item) => `<span>${escapeHtml(item)}</span>`).join('')}</div><div class="verdict ${verdictClass}">${escapeHtml(report.verdict)}</div><div class="meta">Generated ${escapeHtml(report.generated_at)} · ${escapeHtml(report.report_id)} · Freshness ${escapeHtml(report.freshness.status)}</div></section>
|
||||
<section class="grid"><div class="card"><span>Governed runs</span><strong>${number(report.summary.runs)}</strong></div><div class="card"><span>Failure rate</span><strong>${report.summary.failure_rate_pct}%</strong></div><div class="card"><span>P95 latency</span><strong>${number(report.summary.latency_ms.p95)} ms</strong></div><div class="card"><span>Provider tokens</span><strong>${number(report.summary.tokens.provider_total)}</strong></div><div class="card"><span>Actual provider cost</span><strong>${money(report.summary.cost_usd.provider_actual)}</strong></div><div class="card"><span>Estimated cost</span><strong>${money(report.summary.cost_usd.estimated)}</strong></div><div class="card"><span>Alerts</span><strong>${number(report.summary.alerts)}</strong></div><div class="card"><span>Retries</span><strong>${number(report.summary.retries)}</strong></div></section>
|
||||
<section class="grid"><div class="card"><span>Governed runs</span><strong>${number(report.summary.runs)}</strong></div><div class="card"><span>Failure rate</span><strong>${report.summary.failure_rate_pct}%</strong></div><div class="card"><span>P95 latency</span><strong>${number(report.summary.latency_ms.p95)} ms</strong></div><div class="card"><span>Provider tokens</span><strong>${report.summary.coverage.token_records > 0 ? number(report.summary.tokens.provider_total ?? report.summary.tokens.total ?? 0) : 'Unavailable'}</strong></div><div class="card"><span>Actual provider cost</span><strong>${report.summary.cost_usd.provider_actual !== null ? money(report.summary.cost_usd.provider_actual) : 'Unavailable'}</strong></div><div class="card"><span>Token coverage</span><strong>${report.summary.coverage.token_pct}%</strong></div><div class="card"><span>Cost coverage</span><strong>${report.summary.coverage.cost_pct}%</strong></div><div class="card"><span>Retries</span><strong>${number(report.summary.retries)}</strong></div></section>
|
||||
<section class="section"><h2>Verdict findings</h2><ul class="findings">${findings}</ul></section>
|
||||
<section class="section"><h2>Evidence freshness</h2><div class="table-wrap"><table><thead><tr><th>Source</th><th>Presence</th><th>State</th><th>Age</th><th>Records</th><th>Path</th></tr></thead><tbody>${sourceRows}</tbody></table></div></section>
|
||||
<section class="section"><h2>Step breakdown</h2><div class="table-wrap"><table><thead><tr><th>Step</th><th>Runs</th><th>Failures</th><th>Avg latency</th><th>Tokens</th><th>Cost</th></tr></thead><tbody>${breakdownRows(report.details.by_step)}</tbody></table></div></section>
|
||||
|
||||
@@ -37,10 +37,18 @@ export interface H6ReportSummary {
|
||||
failure_rate_pct: number;
|
||||
retries: number;
|
||||
latency_ms: { average: number; p50: number; p95: number; p99: number; max: number };
|
||||
tokens: { input: number; output: number; total: number; provider_total: number };
|
||||
cost_usd: { provider_actual: number; estimated: number };
|
||||
tokens: { input: number | null; output: number | null; total: number | null; provider_total: number | null };
|
||||
cost_usd: { provider_actual: number | null; estimated: number | null };
|
||||
provider_calls: number;
|
||||
alerts: number;
|
||||
coverage: {
|
||||
runtime_records: number;
|
||||
token_records: number;
|
||||
cost_records: number;
|
||||
token_pct: number;
|
||||
cost_pct: number;
|
||||
quality: { complete: number; partial: number; insufficient: number; unknown: number };
|
||||
};
|
||||
}
|
||||
|
||||
export interface H6Breakdown {
|
||||
@@ -48,8 +56,8 @@ export interface H6Breakdown {
|
||||
runs: number;
|
||||
failures: number;
|
||||
latency_avg_ms: number;
|
||||
tokens: number;
|
||||
cost_usd: number;
|
||||
tokens: number | null;
|
||||
cost_usd: number | null;
|
||||
}
|
||||
|
||||
export interface H6ReportDetails {
|
||||
@@ -66,6 +74,7 @@ export type H6Report = HarnessReport<H6ReportSummary, H6ReportDetails>;
|
||||
|
||||
const safeFilter = /^[a-zA-Z0-9][a-zA-Z0-9._:-]{0,127}$/;
|
||||
const numberValue = (value: unknown): number => typeof value === 'number' && Number.isFinite(value) ? value : 0;
|
||||
const hasNumber = (value: unknown): boolean => typeof value === 'number' && Number.isFinite(value);
|
||||
const stringValue = (value: unknown, fallback: string): string => typeof value === 'string' && value.trim() ? value.trim() : fallback;
|
||||
const round = (value: number, digits = 2): number => Number(value.toFixed(digits));
|
||||
const configuredThreshold = (name: string, fallback: number): number => {
|
||||
@@ -148,8 +157,13 @@ function grouped(rows: TelemetryRow[], keyOf: (row: TelemetryRow) => string): H6
|
||||
runs: records.length,
|
||||
failures: records.filter((row) => stringValue(row.status, 'unknown') === 'failed').length,
|
||||
latency_avg_ms: latencies.length ? Math.round(latencies.reduce((total, value) => total + value, 0) / latencies.length) : 0,
|
||||
tokens: sum(records, 'total_tokens'),
|
||||
cost_usd: round(sum(records, records.some((row) => row.cost_usd !== undefined) ? 'cost_usd' : 'cost_estimate'), 6),
|
||||
tokens: records.some((row) => hasNumber(row.total_tokens))
|
||||
? sum(records, 'total_tokens') : null,
|
||||
cost_usd: records.some((row) => hasNumber(row.cost_usd))
|
||||
? round(sum(records, 'cost_usd'), 6)
|
||||
: records.some((row) => hasNumber(row.cost_estimate))
|
||||
? round(sum(records, 'cost_estimate'), 6)
|
||||
: null,
|
||||
};
|
||||
}).sort((left, right) => right.runs - left.runs || left.key.localeCompare(right.key));
|
||||
}
|
||||
@@ -195,6 +209,29 @@ export function buildH6Report(input: H6ReportInput, query: H6ReportQuery): H6Rep
|
||||
const failed = metrics.filter((row) => stringValue(row.status, 'unknown') === 'failed').length;
|
||||
const degraded = metrics.filter((row) => stringValue(row.status, 'unknown') === 'degraded').length;
|
||||
const success = metrics.filter((row) => ['success', 'pass', 'passed'].includes(stringValue(row.status, 'unknown'))).length;
|
||||
const providerTokenRuns = new Set(provider.filter((row) => hasNumber(row.total_tokens)).map(rowRun));
|
||||
const providerCostRuns = new Set(provider.filter((row) => hasNumber(row.cost_usd)).map(rowRun));
|
||||
const tokenRecords = metrics.filter((row) => (
|
||||
hasNumber(row.total_tokens)
|
||||
|| hasNumber(row.input_tokens)
|
||||
|| hasNumber(row.output_tokens)
|
||||
|| providerTokenRuns.has(rowRun(row))
|
||||
)).length;
|
||||
const costRecords = metrics.filter((row) => (
|
||||
hasNumber(row.cost_estimate)
|
||||
|| providerCostRuns.has(rowRun(row))
|
||||
)).length;
|
||||
const tokenCoverage = metrics.length ? round((tokenRecords / metrics.length) * 100, 1) : 0;
|
||||
const costCoverage = metrics.length ? round((costRecords / metrics.length) * 100, 1) : 0;
|
||||
const qualityCounts = { complete: 0, partial: 0, insufficient: 0, unknown: 0 };
|
||||
for (const row of metrics) {
|
||||
const quality = stringValue(row.telemetry_quality, 'unknown');
|
||||
if (quality === 'complete' || quality === 'partial' || quality === 'insufficient') {
|
||||
qualityCounts[quality] += 1;
|
||||
} else {
|
||||
qualityCounts.unknown += 1;
|
||||
}
|
||||
}
|
||||
const sourceEvidence = evidenceSources(input);
|
||||
const primary = sourceEvidence.find((source) => source.source === 'metrics');
|
||||
const failureRate = metrics.length ? round((failed / metrics.length) * 100, 1) : 0;
|
||||
@@ -223,7 +260,19 @@ export function buildH6Report(input: H6ReportInput, query: H6ReportQuery): H6Rep
|
||||
const alertCount = alertCounts.reduce((total, entry) => total + entry.count, 0);
|
||||
if (alertCount > 0) findings.push({ severity: 'warning', code: 'ALERTS_PRESENT', message: `${alertCount} alert signal(s) require review.`, metric: 'alerts', value: alertCount });
|
||||
if (degraded > 0) findings.push({ severity: 'warning', code: 'DEGRADED_RUNS_PRESENT', message: `${degraded} degraded run(s) are present in the selected scope.`, metric: 'degraded', value: degraded });
|
||||
if (metrics.length > 0 && (tokenCoverage < 100 || costCoverage < 100)) {
|
||||
findings.push({
|
||||
severity: 'warning',
|
||||
code: 'TELEMETRY_COVERAGE_GAP',
|
||||
message: 'Some runs do not have reliable provider token or cost attribution.',
|
||||
metric: 'token_cost_coverage_pct',
|
||||
value: `${tokenCoverage}/${costCoverage}`,
|
||||
threshold: '100/100',
|
||||
});
|
||||
}
|
||||
if (provider.length === 0) warnings.push('No provider usage records matched the selected scope; token and actual-cost breakdown may be incomplete.');
|
||||
if (metrics.length > 0 && tokenCoverage < 100) warnings.push(`${tokenCoverage}% of runtime records have reliable token attribution; unavailable values remain null, never zero.`);
|
||||
if (metrics.length > 0 && costCoverage < 100) warnings.push(`${costCoverage}% of runtime records have reliable cost attribution; unavailable values remain null, never zero.`);
|
||||
for (const source of sourceEvidence) {
|
||||
if (source.source === 'metrics') continue;
|
||||
if (!source.present) warnings.push(`Optional ${source.source} telemetry source is missing; its breakdown is unavailable.`);
|
||||
@@ -243,7 +292,7 @@ export function buildH6Report(input: H6ReportInput, query: H6ReportQuery): H6Rep
|
||||
const costSources = grouped(metrics, (row) => stringValue(row.cost_source, 'unknown')).map((entry) => ({
|
||||
source: entry.key,
|
||||
records: entry.runs,
|
||||
cost_usd: entry.cost_usd,
|
||||
cost_usd: entry.cost_usd ?? 0,
|
||||
}));
|
||||
const scope: HarnessReportScope = { project: query.project, from: query.from, to: query.to, run: query.run };
|
||||
|
||||
@@ -272,15 +321,37 @@ export function buildH6Report(input: H6ReportInput, query: H6ReportQuery): H6Rep
|
||||
p99: percentile(latencies, 99),
|
||||
max: latencies.length ? Math.max(...latencies) : 0,
|
||||
},
|
||||
tokens: { input: sum(metrics, 'input_tokens'), output: sum(metrics, 'output_tokens'), total: sum(metrics, 'total_tokens'), provider_total: sum(provider, 'total_tokens') },
|
||||
cost_usd: { provider_actual: round(sum(provider, 'cost_usd'), 6), estimated: round(sum(metrics, 'cost_estimate'), 6) },
|
||||
tokens: {
|
||||
input: metrics.some((row) => hasNumber(row.input_tokens)) ? sum(metrics, 'input_tokens') : null,
|
||||
output: metrics.some((row) => hasNumber(row.output_tokens)) ? sum(metrics, 'output_tokens') : null,
|
||||
total: metrics.some((row) => hasNumber(row.total_tokens)) ? sum(metrics, 'total_tokens') : null,
|
||||
provider_total: provider.some((row) => hasNumber(row.total_tokens)) ? sum(provider, 'total_tokens') : null,
|
||||
},
|
||||
cost_usd: {
|
||||
provider_actual: provider.some((row) => hasNumber(row.cost_usd)) ? round(sum(provider, 'cost_usd'), 6) : null,
|
||||
estimated: metrics.some((row) => hasNumber(row.cost_estimate)) ? round(sum(metrics, 'cost_estimate'), 6) : null,
|
||||
},
|
||||
provider_calls: provider.length,
|
||||
alerts: alertCount,
|
||||
coverage: {
|
||||
runtime_records: metrics.length,
|
||||
token_records: tokenRecords,
|
||||
cost_records: costRecords,
|
||||
token_pct: tokenCoverage,
|
||||
cost_pct: costCoverage,
|
||||
quality: qualityCounts,
|
||||
},
|
||||
},
|
||||
thresholds: { failure_rate_pct: failureThreshold, p95_latency_ms: p95Threshold, freshness_age_s: STALE_AFTER_S },
|
||||
findings,
|
||||
evidence_sources: sourceEvidence,
|
||||
data_quality: { status: !primary?.present ? 'insufficient' : warnings.length ? 'partial' : 'complete', warnings },
|
||||
data_quality: {
|
||||
status: (
|
||||
!primary?.present
|
||||
|| (metrics.length > 0 && tokenRecords === 0 && costRecords === 0)
|
||||
) ? 'insufficient' : warnings.length ? 'partial' : 'complete',
|
||||
warnings,
|
||||
},
|
||||
available_filters: { projects: availableProjects, runs: availableRuns },
|
||||
details: {
|
||||
by_status: counted(metrics, (row) => stringValue(row.status, 'unknown')).map(({ key, count: countValue }) => ({ status: key, count: countValue })),
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { BadRequestException, Controller, Get, Header, Inject, Query, Res } from '@nestjs/common';
|
||||
import { BadRequestException, Controller, Get, Header, Inject, Param, Query, Res } from '@nestjs/common';
|
||||
import type { Response } from 'express';
|
||||
import { ok } from '../common/api-response.js';
|
||||
import { parseH6ReportQuery, type H6ReportQueryParams } from './h6-report.js';
|
||||
@@ -34,4 +34,24 @@ export class ReportsController {
|
||||
response.setHeader('Content-Disposition', `attachment; filename="casan-h6-report-${stamp}.${format}"`);
|
||||
response.send(body);
|
||||
}
|
||||
|
||||
@Get('run/:traceId')
|
||||
run(@Param('traceId') traceId: string) {
|
||||
return ok(this.reports.run(traceId));
|
||||
}
|
||||
|
||||
@Get('run/:traceId/export')
|
||||
@Header('Cache-Control', 'no-store')
|
||||
exportRun(
|
||||
@Param('traceId') traceId: string,
|
||||
@Query('format') rawFormat: string | undefined,
|
||||
@Res() response: Response,
|
||||
) {
|
||||
const format = rawFormat ?? 'json';
|
||||
if (format !== 'json' && format !== 'html') throw new BadRequestException('RUN_REPORT_INVALID_FORMAT');
|
||||
const report = this.reports.run(traceId);
|
||||
response.type(format === 'html' ? 'text/html; charset=utf-8' : 'application/json; charset=utf-8');
|
||||
response.setHeader('Content-Disposition', `attachment; filename="casan-run-${traceId}.${format}"`);
|
||||
response.send(this.reports.serializeRun(report, format));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { ReportsController } from './reports.controller.js';
|
||||
import { ReportsService } from './reports.service.js';
|
||||
import { TelemetryModule } from '../telemetry/telemetry.module.js';
|
||||
|
||||
@Module({
|
||||
imports: [TelemetryModule],
|
||||
controllers: [ReportsController],
|
||||
providers: [ReportsService],
|
||||
exports: [ReportsService],
|
||||
|
||||
@@ -1,12 +1,20 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { Inject, Injectable, NotFoundException } from '@nestjs/common';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { PATHS, telemetryFreshness } from '../common/app-root.js';
|
||||
import { readJsonl } from '../telemetry/telemetry.reader.js';
|
||||
import { TelemetryService } from '../telemetry/telemetry.service.js';
|
||||
import { buildH6Report, type H6Report, type H6ReportQuery } from './h6-report.js';
|
||||
import { renderH6ReportHtml } from './h6-report.html.js';
|
||||
import { HARNESS_REPORT_CATALOG } from './report.contract.js';
|
||||
import { buildRunAssuranceReport, type RunAssuranceReport } from './run-report.js';
|
||||
import { renderRunAssuranceHtml } from './run-report.html.js';
|
||||
import { APP_ROOT } from '../common/app-root.js';
|
||||
|
||||
@Injectable()
|
||||
export class ReportsService {
|
||||
constructor(@Inject(TelemetryService) private readonly telemetry: TelemetryService) {}
|
||||
|
||||
catalog() {
|
||||
return { schema_version: 1, reports: HARNESS_REPORT_CATALOG };
|
||||
}
|
||||
@@ -25,4 +33,35 @@ export class ReportsService {
|
||||
serializeH6(report: H6Report, format: 'json' | 'html'): string {
|
||||
return format === 'html' ? renderH6ReportHtml(report) : `${JSON.stringify(report, null, 2)}\n`;
|
||||
}
|
||||
|
||||
run(traceId: string): RunAssuranceReport {
|
||||
if (!/^[a-zA-Z0-9][a-zA-Z0-9._:-]{0,127}$/.test(traceId)) {
|
||||
throw new NotFoundException('CASAN_RUN_NOT_FOUND');
|
||||
}
|
||||
const graph = this.telemetry.traceGraph(traceId);
|
||||
const traceResult = this.telemetry.run(traceId);
|
||||
const metrics = readJsonl(PATHS.metrics);
|
||||
const metric = [...metrics].reverse().find((row) => row.trace_id === traceId) ?? null;
|
||||
let config: Record<string, unknown> = {};
|
||||
try {
|
||||
config = JSON.parse(readFileSync(join(APP_ROOT, '.casan', 'config.json'), 'utf8')) as Record<string, unknown>;
|
||||
} catch {
|
||||
config = {};
|
||||
}
|
||||
const report = buildRunAssuranceReport({
|
||||
traceId,
|
||||
graph,
|
||||
trace: traceResult.trace,
|
||||
metric,
|
||||
config,
|
||||
});
|
||||
if (report.verdict === 'not_found') throw new NotFoundException('CASAN_RUN_NOT_FOUND');
|
||||
return report;
|
||||
}
|
||||
|
||||
serializeRun(report: RunAssuranceReport, format: 'json' | 'html'): string {
|
||||
return format === 'html'
|
||||
? renderRunAssuranceHtml(report)
|
||||
: `${JSON.stringify(report, null, 2)}\n`;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
import type { RunAssuranceReport } from './run-report.js';
|
||||
|
||||
const escapeHtml = (value: unknown): string => String(value ?? '')
|
||||
.replaceAll('&', '&')
|
||||
.replaceAll('<', '<')
|
||||
.replaceAll('>', '>')
|
||||
.replaceAll('"', '"')
|
||||
.replaceAll("'", ''');
|
||||
|
||||
const badgeClass = (status: string): string => {
|
||||
if (status === 'pass' || status === 'certified') return 'pass';
|
||||
if (status === 'warning' || status === 'in_progress') return 'warn';
|
||||
if (status === 'queued' || status === 'skipped') return 'muted';
|
||||
return 'fail';
|
||||
};
|
||||
|
||||
export function renderRunAssuranceHtml(report: RunAssuranceReport): string {
|
||||
const gates = report.gates.map((gate) => `
|
||||
<article class="gate">
|
||||
<div class="gate-head"><strong>${escapeHtml(gate.title)}</strong><span class="badge ${badgeClass(gate.status)}">${escapeHtml(gate.status)}</span></div>
|
||||
<p>${escapeHtml(gate.reason)}</p>
|
||||
<small>${escapeHtml(gate.updated_at ?? 'No timestamp')}</small>
|
||||
<details><summary>Evidence fields</summary><pre>${escapeHtml(JSON.stringify(gate.evidence, null, 2))}</pre></details>
|
||||
</article>`).join('');
|
||||
const tokenValue = report.summary.token_usage_available ? 'Available' : 'Unavailable';
|
||||
const costValue = report.summary.cost_available ? 'Available' : 'Unavailable';
|
||||
return `<!doctype html>
|
||||
<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<title>CASAN assurance receipt · ${escapeHtml(report.trace_id)}</title>
|
||||
<style>
|
||||
:root{color-scheme:light;--ink:#0f172a;--muted:#64748b;--line:#e2e8f0;--panel:#fff;--bg:#f8fafc;--indigo:#4f46e5}
|
||||
*{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--ink);font:14px/1.55 Inter,ui-sans-serif,system-ui,-apple-system,sans-serif}
|
||||
main{max-width:1160px;margin:0 auto;padding:40px 24px 64px}.hero{position:relative;overflow:hidden;border-radius:28px;background:#111827;color:#fff;padding:32px;box-shadow:0 24px 60px rgba(15,23,42,.18)}
|
||||
.hero:after{content:"";position:absolute;right:-80px;top:-100px;width:280px;height:280px;border-radius:50%;background:rgba(99,102,241,.3);filter:blur(50px)}.eyebrow{color:#a5b4fc;font-size:11px;font-weight:800;letter-spacing:.18em;text-transform:uppercase}
|
||||
h1{position:relative;margin:8px 0 2px;font-size:30px;letter-spacing:-.035em}.trace{position:relative;color:#94a3b8;font:12px ui-monospace,SFMono-Regular,monospace;word-break:break-all}
|
||||
.summary{display:grid;grid-template-columns:repeat(4,minmax(0,1fr));gap:12px;margin:18px 0}.metric,.section{border:1px solid var(--line);background:var(--panel);border-radius:18px;padding:18px}.metric span{display:block;color:var(--muted);font-size:11px;font-weight:700;text-transform:uppercase;letter-spacing:.08em}.metric strong{display:block;margin-top:8px;font-size:22px}
|
||||
.section{margin-top:16px}.section h2{margin:0 0 14px;font-size:16px}.rail{display:grid;grid-template-columns:repeat(7,minmax(0,1fr));gap:10px}.gate{min-width:0;border:1px solid var(--line);border-radius:15px;padding:14px;background:#fff}.gate-head{display:flex;align-items:flex-start;justify-content:space-between;gap:8px}.gate p{min-height:44px;color:#475569;font-size:12px}.gate small{color:#94a3b8;font-size:10px}.gate details{margin-top:10px;color:#64748b;font-size:10px}.gate summary{cursor:pointer;font-weight:700}.gate pre{max-height:180px;overflow:auto;white-space:pre-wrap;word-break:break-word;border-radius:8px;background:#f8fafc;padding:8px;font-size:9px}.badge{border-radius:999px;padding:3px 7px;font-size:9px;font-weight:800;text-transform:uppercase}.pass{background:#dcfce7;color:#166534}.warn{background:#fef3c7;color:#92400e}.fail{background:#ffe4e6;color:#9f1239}.muted{background:#f1f5f9;color:#64748b}
|
||||
.note{color:var(--muted);font-size:12px}.footer{margin-top:18px;color:#94a3b8;font-size:11px}
|
||||
@media(max-width:900px){.summary{grid-template-columns:repeat(2,1fr)}.rail{grid-template-columns:1fr 1fr}}@media(max-width:520px){main{padding:18px 12px}.hero{padding:22px}.rail,.summary{grid-template-columns:1fr}}
|
||||
@media print{body{background:#fff}main{max-width:none;padding:0}.hero,.metric,.section{box-shadow:none;break-inside:avoid}}
|
||||
</style></head><body><main>
|
||||
<section class="hero"><div class="eyebrow">CASAN · evidence-backed assurance</div><h1>${escapeHtml(report.verdict.replace('_', ' ').toUpperCase())}</h1><div class="trace">${escapeHtml(report.trace_id)}</div></section>
|
||||
<section class="summary">
|
||||
<div class="metric"><span>Harness gates</span><strong>${report.summary.gates_observed}/7</strong></div>
|
||||
<div class="metric"><span>H6 quality</span><strong>${escapeHtml(report.summary.telemetry_quality)}</strong></div>
|
||||
<div class="metric"><span>Token usage</span><strong>${tokenValue}</strong></div>
|
||||
<div class="metric"><span>Cost</span><strong>${costValue}</strong></div>
|
||||
<div class="metric"><span>Duration</span><strong>${report.summary.duration_ms === null ? 'Unavailable' : `${report.summary.duration_ms} ms`}</strong></div>
|
||||
<div class="metric"><span>Tool calls</span><strong>${report.summary.tool_calls}</strong></div>
|
||||
<div class="metric"><span>Failures</span><strong>${report.summary.failures}</strong></div>
|
||||
<div class="metric"><span>Evidence source</span><strong>${report.source.trace_found && report.source.graph_found ? 'Verified' : 'Partial'}</strong></div>
|
||||
</section>
|
||||
<section class="section"><h2>Live assurance rail · H1 → H7</h2><div class="rail">${gates}</div></section>
|
||||
<section class="section"><h2>Certification</h2><p>Strength: <strong>${escapeHtml(report.certification.strength ?? 'unknown')}</strong></p><p class="note">${escapeHtml(report.certification.reasons.join(' · ') || 'No certification reason recorded.')}</p></section>
|
||||
<div class="footer">Generated ${escapeHtml(report.generated_at)} · Project ${escapeHtml(report.project.id)} · Edition ${escapeHtml(report.project.edition ?? 'unknown')} · Maturity ${escapeHtml(report.project.maturity?.status ?? 'not assessed')}. No maturity score or unavailable telemetry value is hard-coded.</div>
|
||||
</main></body></html>`;
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
export interface RunGateSnapshot {
|
||||
id: string;
|
||||
title: string;
|
||||
description: string;
|
||||
status: string;
|
||||
reason: string;
|
||||
updated_at: string | null;
|
||||
evidence: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface RunAssuranceReport {
|
||||
schema_version: 1;
|
||||
report_id: string;
|
||||
generated_at: string;
|
||||
trace_id: string;
|
||||
project: {
|
||||
id: string;
|
||||
edition: string | null;
|
||||
maturity: { level: number | null; status: string } | null;
|
||||
};
|
||||
verdict: 'certified' | 'non_certified' | 'in_progress' | 'not_found';
|
||||
certification: {
|
||||
strength: string | null;
|
||||
reasons: string[];
|
||||
finalized_at: string | null;
|
||||
};
|
||||
summary: {
|
||||
gates_observed: number;
|
||||
gates_total: 7;
|
||||
tool_calls: number;
|
||||
failures: number;
|
||||
duration_ms: number | null;
|
||||
telemetry_quality: string;
|
||||
token_usage_available: boolean;
|
||||
cost_available: boolean;
|
||||
};
|
||||
gates: RunGateSnapshot[];
|
||||
h6: Record<string, unknown> | null;
|
||||
source: {
|
||||
trace_found: boolean;
|
||||
graph_found: boolean;
|
||||
metric_found: boolean;
|
||||
};
|
||||
}
|
||||
|
||||
type Row = Record<string, unknown>;
|
||||
|
||||
const stringValue = (value: unknown, fallback = ''): string => (
|
||||
typeof value === 'string' && value.trim() ? value.trim() : fallback
|
||||
);
|
||||
const numberValue = (value: unknown, fallback = 0): number => (
|
||||
typeof value === 'number' && Number.isFinite(value) ? value : fallback
|
||||
);
|
||||
const hasNumber = (value: unknown): boolean => (
|
||||
typeof value === 'number' && Number.isFinite(value)
|
||||
);
|
||||
|
||||
export function buildRunAssuranceReport(input: {
|
||||
traceId: string;
|
||||
graph: {
|
||||
found: boolean;
|
||||
terminal: boolean;
|
||||
progress: number;
|
||||
nodes: RunGateSnapshot[];
|
||||
};
|
||||
trace: Row | null;
|
||||
metric: Row | null;
|
||||
config: Row;
|
||||
now?: Date;
|
||||
}): RunAssuranceReport {
|
||||
const now = input.now ?? new Date();
|
||||
const trace = input.trace ?? {};
|
||||
const metric = input.metric ?? {};
|
||||
const certified = trace.certified === true;
|
||||
const traceFound = input.trace !== null;
|
||||
const metricFound = input.metric !== null;
|
||||
const verdict = (
|
||||
!traceFound && !input.graph.found && !metricFound ? 'not_found'
|
||||
: !input.graph.terminal && !trace.finalized_at ? 'in_progress'
|
||||
: certified ? 'certified' : 'non_certified'
|
||||
);
|
||||
const maturity = input.config.maturity && typeof input.config.maturity === 'object'
|
||||
? input.config.maturity as Record<string, unknown>
|
||||
: null;
|
||||
const duration = (
|
||||
hasNumber(metric.duration_ms) ? Number(metric.duration_ms)
|
||||
: hasNumber(metric.latency_ms) ? Number(metric.latency_ms)
|
||||
: null
|
||||
);
|
||||
const costObject = metric.cost && typeof metric.cost === 'object'
|
||||
? metric.cost as Record<string, unknown>
|
||||
: {};
|
||||
return {
|
||||
schema_version: 1,
|
||||
report_id: `RUN-${input.traceId}`,
|
||||
generated_at: now.toISOString(),
|
||||
trace_id: input.traceId,
|
||||
project: {
|
||||
id: stringValue(
|
||||
input.config.project_id ?? trace.project_id ?? metric.project_id,
|
||||
'unknown',
|
||||
),
|
||||
edition: stringValue(
|
||||
input.config.edition ?? input.config.target_level_name,
|
||||
) || null,
|
||||
maturity: maturity ? {
|
||||
level: hasNumber(maturity.level) ? Number(maturity.level) : null,
|
||||
status: stringValue(maturity.status, 'not_assessed'),
|
||||
} : null,
|
||||
},
|
||||
verdict,
|
||||
certification: {
|
||||
strength: stringValue(
|
||||
trace.certification_strength ?? metric.certification_strength,
|
||||
) || null,
|
||||
reasons: Array.isArray(trace.certification_reasons)
|
||||
? trace.certification_reasons.filter(
|
||||
(value): value is string => typeof value === 'string')
|
||||
: [],
|
||||
finalized_at: stringValue(trace.finalized_at ?? metric.finished_at) || null,
|
||||
},
|
||||
summary: {
|
||||
gates_observed: input.graph.progress,
|
||||
gates_total: 7,
|
||||
tool_calls: numberValue(trace.tool_calls ?? metric.tool_calls),
|
||||
failures: numberValue(trace.failures ?? metric.failures),
|
||||
duration_ms: duration,
|
||||
telemetry_quality: stringValue(metric.telemetry_quality, 'unknown'),
|
||||
token_usage_available: (
|
||||
hasNumber(metric.total_tokens)
|
||||
|| hasNumber(metric.input_tokens)
|
||||
|| hasNumber(metric.output_tokens)
|
||||
),
|
||||
cost_available: (
|
||||
hasNumber(metric.cost_estimate)
|
||||
|| hasNumber(costObject.amount)
|
||||
),
|
||||
},
|
||||
gates: input.graph.nodes,
|
||||
h6: input.metric,
|
||||
source: {
|
||||
trace_found: traceFound,
|
||||
graph_found: input.graph.found,
|
||||
metric_found: metricFound,
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -16,6 +16,11 @@ export class TelemetryController {
|
||||
return ok(this.svc.overview());
|
||||
}
|
||||
|
||||
@Get('project')
|
||||
project() {
|
||||
return ok(this.svc.projectProfile());
|
||||
}
|
||||
|
||||
// Production reverse proxy policy must restrict this aggregate-only endpoint
|
||||
// to the monitoring network / service account.
|
||||
@Get('metrics')
|
||||
|
||||
@@ -5,5 +5,6 @@ import { TelemetryService } from './telemetry.service.js';
|
||||
@Module({
|
||||
controllers: [TelemetryController],
|
||||
providers: [TelemetryService],
|
||||
exports: [TelemetryService],
|
||||
})
|
||||
export class TelemetryModule {}
|
||||
|
||||
@@ -59,9 +59,9 @@ const pct = (part: number, total: number) => (total > 0 ? Math.round((part / tot
|
||||
|
||||
function gateStatus(value: unknown): HarnessGateStatus {
|
||||
const status = String(value ?? '').toLowerCase();
|
||||
if (['success', 'pass', 'passed', 'allow', 'allowed', 'answered'].includes(status)) return 'pass';
|
||||
if (['warn', 'warning'].includes(status)) return 'warning';
|
||||
if (['block', 'blocked', 'deny', 'denied'].includes(status)) return 'blocked';
|
||||
if (['success', 'pass', 'passed', 'allow', 'allowed', 'answered', 'opened', 'certified'].includes(status)) return 'pass';
|
||||
if (['warn', 'warning', 'degraded', 'partial', 'insufficient'].includes(status)) return 'warning';
|
||||
if (['block', 'blocked', 'deny', 'denied', 'flag', 'non_certified'].includes(status)) return 'blocked';
|
||||
if (['fail', 'failed', 'error'].includes(status)) return 'error';
|
||||
if (status === 'running') return 'running';
|
||||
if (status === 'skipped') return 'skipped';
|
||||
@@ -232,11 +232,23 @@ export class TelemetryService {
|
||||
|
||||
const runs = metrics.length;
|
||||
const latencies = metrics.map((m) => num(m.latency_ms)).filter((x) => x > 0);
|
||||
const latestMetric = metrics.length ? metrics[metrics.length - 1] : null;
|
||||
const latestTraceId = typeof latestMetric?.trace_id === 'string' ? latestMetric.trace_id : null;
|
||||
const latestTrace = latestTraceId ? readTrace(PATHS.traceDir, latestTraceId) : null;
|
||||
const latestGraph = latestTraceId ? this.traceGraph(latestTraceId) : null;
|
||||
return {
|
||||
...this.freshness(),
|
||||
totals: {
|
||||
runs,
|
||||
total_cost: sum(metrics, 'cost_estimate'),
|
||||
token_coverage_pct: pct(
|
||||
count(metrics, (m) => m.total_tokens != null || m.input_tokens != null || m.output_tokens != null),
|
||||
runs,
|
||||
),
|
||||
cost_coverage_pct: pct(
|
||||
count(metrics, (m) => m.cost_estimate != null || (m.cost && m.cost.amount != null)),
|
||||
runs,
|
||||
),
|
||||
avg_latency_ms: latencies.length ? Math.round(latencies.reduce((a, b) => a + b, 0) / latencies.length) : 0,
|
||||
failures: count(metrics, (m) => m.status === 'failed'),
|
||||
hallucination_signals: sum(metrics, 'hallucination_signals'),
|
||||
@@ -260,6 +272,39 @@ export class TelemetryService {
|
||||
head: readHead(PATHS.auditHead),
|
||||
last_decision: audit.length ? audit[audit.length - 1].decision ?? null : null,
|
||||
},
|
||||
latest_assurance: latestTraceId && latestMetric && latestGraph ? {
|
||||
trace_id: latestTraceId,
|
||||
certified: latestTrace?.certified === true,
|
||||
certification_strength: latestTrace?.certification_strength ?? latestMetric.certification_strength ?? null,
|
||||
telemetry_quality: latestMetric.telemetry_quality ?? 'unknown',
|
||||
duration_ms: latestMetric.duration_ms ?? latestMetric.latency_ms ?? null,
|
||||
tool_calls: latestMetric.tool_calls ?? 0,
|
||||
failures: latestMetric.failures ?? 0,
|
||||
graph: latestGraph,
|
||||
} : null,
|
||||
};
|
||||
}
|
||||
|
||||
projectProfile() {
|
||||
const config = readJson(join(APP_ROOT, '.casan', 'config.json')) as Row | null;
|
||||
const maturity = config?.maturity && typeof config.maturity === 'object'
|
||||
? config.maturity as Row
|
||||
: null;
|
||||
return {
|
||||
project_id: String(config?.project_id ?? APP_ROOT.split('/').pop() ?? 'unknown'),
|
||||
app_root: APP_ROOT,
|
||||
edition: String(config?.edition ?? config?.target_level_name ?? 'unknown'),
|
||||
edition_status: String(config?.edition_status ?? 'unknown'),
|
||||
maturity: {
|
||||
level: typeof maturity?.level === 'number' ? maturity.level : null,
|
||||
status: String(maturity?.status ?? 'not_assessed'),
|
||||
evidence: typeof maturity?.evidence === 'string' ? maturity.evidence : null,
|
||||
},
|
||||
enforcement_mode: String(config?.enforcement_mode ?? 'unknown'),
|
||||
integration_mode: String(config?.integration_mode ?? 'unknown'),
|
||||
clients: Array.isArray(config?.clients)
|
||||
? config.clients.filter((value): value is string => typeof value === 'string')
|
||||
: [],
|
||||
};
|
||||
}
|
||||
|
||||
@@ -351,13 +396,45 @@ export class TelemetryService {
|
||||
},
|
||||
});
|
||||
}
|
||||
const gateMap: Record<string, string> = {
|
||||
H1: 'H1-context',
|
||||
H2: 'H2-tool',
|
||||
H3: 'H3-eval',
|
||||
H4: 'H4-security',
|
||||
H5: 'H5-governance',
|
||||
H6: 'H6-agentops',
|
||||
H7: 'H7-orchestration',
|
||||
};
|
||||
for (const evidence of arr(legacyTrace.evidence)) {
|
||||
const harnesses = String(evidence.h ?? '').split('/');
|
||||
for (const harness of harnesses) {
|
||||
const mappedGate = gateMap[harness];
|
||||
if (!mappedGate) continue;
|
||||
events.push({
|
||||
timestamp: String(evidence.at ?? legacyTrace.finalized_at ?? ''),
|
||||
trace_id: safeId,
|
||||
gate_id: mappedGate,
|
||||
status: gateStatus(evidence.decision),
|
||||
reason: `${String(evidence.kind ?? 'legacy-evidence')}: ${String(evidence.detail ?? '')}`,
|
||||
evidence: {
|
||||
kind: evidence.kind ?? null,
|
||||
decision: evidence.decision ?? null,
|
||||
detail: evidence.detail ?? null,
|
||||
certification_strength: legacyTrace.certification_strength ?? null,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const metric of readJsonl<Row>(PATHS.metrics).filter((row) => row.trace_id === safeId)) {
|
||||
const telemetryQuality = String(metric.telemetry_quality ?? '');
|
||||
events.push({
|
||||
timestamp: String(metric.timestamp ?? ''),
|
||||
trace_id: safeId,
|
||||
gate_id: 'H6-agentops',
|
||||
status: gateStatus(metric.status),
|
||||
status: ['partial', 'insufficient'].includes(telemetryQuality)
|
||||
? 'warning'
|
||||
: gateStatus(metric.status),
|
||||
reason: String(metric.step ?? 'Legacy runtime metric'),
|
||||
evidence: {
|
||||
latency_ms: metric.latency_ms ?? null,
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { BadRequestException, ServiceUnavailableException } from '@nestjs/common';
|
||||
import { IngestService, expectedSignature, stableJson } from '../src/ingest/ingest.service.js';
|
||||
|
||||
test('ingest canonical JSON is stable across object key order', () => {
|
||||
assert.equal(stableJson({ z: 1, a: { y: 2, b: 3 } }), '{"a":{"b":3,"y":2},"z":1}');
|
||||
});
|
||||
|
||||
test('ingest is disabled unless an HMAC secret is configured', () => {
|
||||
const previous = process.env.CASAN_CP_INGEST_TOKEN;
|
||||
delete process.env.CASAN_CP_INGEST_TOKEN;
|
||||
try {
|
||||
assert.throws(() => new IngestService().ingest({}, '0', 'none'), ServiceUnavailableException);
|
||||
} finally {
|
||||
if (previous === undefined) delete process.env.CASAN_CP_INGEST_TOKEN;
|
||||
else process.env.CASAN_CP_INGEST_TOKEN = previous;
|
||||
}
|
||||
});
|
||||
|
||||
test('signed ingest rejects raw prompt/content fields before persistence', () => {
|
||||
const previous = process.env.CASAN_CP_INGEST_TOKEN;
|
||||
process.env.CASAN_CP_INGEST_TOKEN = 'test-only-secret';
|
||||
const timestamp = String(Math.floor(Date.now() / 1000));
|
||||
const body = {
|
||||
schema_version: 1,
|
||||
sent_at: new Date().toISOString(),
|
||||
project_id: 'safe-project',
|
||||
trace_id: 'safe-trace',
|
||||
receipt: {},
|
||||
metric: { trace_id: 'safe-trace', prompt: 'must-not-cross-boundary' },
|
||||
trace: { trace_id: 'safe-trace' },
|
||||
events: [],
|
||||
};
|
||||
try {
|
||||
const signature = expectedSignature('test-only-secret', timestamp, body);
|
||||
assert.throws(
|
||||
() => new IngestService().ingest(body, timestamp, signature),
|
||||
BadRequestException,
|
||||
);
|
||||
} finally {
|
||||
if (previous === undefined) delete process.env.CASAN_CP_INGEST_TOKEN;
|
||||
else process.env.CASAN_CP_INGEST_TOKEN = previous;
|
||||
}
|
||||
});
|
||||
@@ -8,6 +8,8 @@ import { sourceFreshness } from '../src/common/app-root.js';
|
||||
import { buildH6Report, parseH6ReportQuery, type H6ReportInput } from '../src/reports/h6-report.js';
|
||||
import { renderH6ReportHtml } from '../src/reports/h6-report.html.js';
|
||||
import { HARNESS_REPORT_CATALOG } from '../src/reports/report.contract.js';
|
||||
import { buildRunAssuranceReport } from '../src/reports/run-report.js';
|
||||
import { renderRunAssuranceHtml } from '../src/reports/run-report.html.js';
|
||||
|
||||
const NOW = new Date('2026-07-20T12:00:00.000Z');
|
||||
|
||||
@@ -68,6 +70,8 @@ test('H6 report filters project/time/run and aggregates measured evidence', () =
|
||||
assert.equal(report.summary.latency_ms.p50, 100);
|
||||
assert.equal(report.summary.latency_ms.p95, 9000);
|
||||
assert.equal(report.summary.tokens.provider_total, 28);
|
||||
assert.equal(report.summary.coverage.token_pct, 100);
|
||||
assert.equal(report.summary.coverage.cost_pct, 100);
|
||||
assert.equal(report.summary.alerts, 2);
|
||||
assert.deepEqual(report.details.by_alert, [
|
||||
{ alert: 'execution-failed', count: 1 },
|
||||
@@ -105,6 +109,30 @@ test('H6 report makes stale optional sources explicit in data quality', () => {
|
||||
assert.ok(report.data_quality.warnings.some((warning) => warning.includes('alerts telemetry source is missing')));
|
||||
});
|
||||
|
||||
test('H6 report never presents missing Codex usage as zero-cost coverage', () => {
|
||||
const input = fixture();
|
||||
input.metrics = [{
|
||||
timestamp: '2026-07-19T10:00:00Z',
|
||||
trace_id: 'codex-null',
|
||||
project_id: 'basic-design',
|
||||
status: 'success',
|
||||
latency_ms: 244090,
|
||||
total_tokens: null,
|
||||
cost_estimate: null,
|
||||
telemetry_quality: 'insufficient',
|
||||
}];
|
||||
input.provider = [];
|
||||
const report = buildH6Report(input, parseH6ReportQuery({ run: 'codex-null' }));
|
||||
assert.equal(report.summary.coverage.token_pct, 0);
|
||||
assert.equal(report.summary.coverage.cost_pct, 0);
|
||||
assert.equal(report.summary.tokens.total, null);
|
||||
assert.equal(report.summary.cost_usd.provider_actual, null);
|
||||
assert.equal(report.summary.cost_usd.estimated, null);
|
||||
assert.equal(report.data_quality.status, 'insufficient');
|
||||
assert.equal(report.verdict, 'attention');
|
||||
assert.ok(report.findings.some((finding) => finding.code === 'TELEMETRY_COVERAGE_GAP'));
|
||||
});
|
||||
|
||||
test('HTML export is standalone, escaped and contains no hard-coded maturity score', () => {
|
||||
const input = fixture();
|
||||
input.metrics[0].step = '<script>alert(1)</script>';
|
||||
@@ -117,3 +145,49 @@ test('HTML export is standalone, escaped and contains no hard-coded maturity sco
|
||||
assert.doesNotMatch(html, /Average\s+\d|\/100|218 core tests/i);
|
||||
assert.match(html, /No maturity score or telemetry value is hard-coded/);
|
||||
});
|
||||
|
||||
test('per-run assurance export carries H1-H7 and truthful H6 availability', () => {
|
||||
const nodes = ['Context', 'Tool', 'Eval', 'Security', 'Governance', 'AgentOps', 'Orchestration']
|
||||
.map((title, index) => ({
|
||||
id: `H${index + 1}`,
|
||||
title: `H${index + 1} · ${title}`,
|
||||
description: title,
|
||||
status: index === 5 ? 'warning' : 'pass',
|
||||
reason: index === 5 ? 'provider usage unavailable' : 'evidence verified',
|
||||
updated_at: NOW.toISOString(),
|
||||
evidence: {},
|
||||
}));
|
||||
const report = buildRunAssuranceReport({
|
||||
traceId: 'trace-safe',
|
||||
graph: { found: true, terminal: true, progress: 7, nodes },
|
||||
trace: {
|
||||
certified: true,
|
||||
certification_strength: 'project_hook',
|
||||
certification_reasons: ['evidence_complete'],
|
||||
finalized_at: NOW.toISOString(),
|
||||
tool_calls: 4,
|
||||
},
|
||||
metric: {
|
||||
trace_id: 'trace-safe',
|
||||
telemetry_quality: 'insufficient',
|
||||
duration_ms: 1200,
|
||||
total_tokens: null,
|
||||
cost_estimate: null,
|
||||
},
|
||||
config: {
|
||||
project_id: 'basic-design',
|
||||
edition: 'core',
|
||||
maturity: { level: null, status: 'not_assessed' },
|
||||
},
|
||||
now: NOW,
|
||||
});
|
||||
assert.equal(report.verdict, 'certified');
|
||||
assert.equal(report.summary.gates_observed, 7);
|
||||
assert.equal(report.summary.token_usage_available, false);
|
||||
assert.equal(report.summary.cost_available, false);
|
||||
const html = renderRunAssuranceHtml(report);
|
||||
assert.match(html, /Live assurance rail · H1 → H7/);
|
||||
assert.match(html, /Evidence fields/);
|
||||
assert.match(html, /Unavailable/);
|
||||
assert.doesNotMatch(html, /\$0(?:\.0+)?/);
|
||||
});
|
||||
|
||||
+4
-4
@@ -100,14 +100,14 @@ function H6Summary({ node, goal, runReport, projectReport, loading, error }: {
|
||||
return <div className="mt-4 rounded-xl border border-amber-300/30 bg-amber-300/10 p-3 text-xs leading-5 text-amber-100">Run telemetry could not be loaded here. <Link to={reportLink} className="font-semibold underline decoration-amber-300/50 underline-offset-4 hover:text-white focus:outline-none focus:ring-2 focus:ring-cyan-300">Open the full H6 report</Link> to inspect the source status.</div>;
|
||||
}
|
||||
|
||||
const tokenTotal = runReport.summary.tokens.provider_total || runReport.summary.tokens.total;
|
||||
const tokenTotal = runReport.summary.tokens.provider_total ?? runReport.summary.tokens.total;
|
||||
const actualCost = runReport.summary.cost_usd.provider_actual;
|
||||
const displayedCost = actualCost || runReport.summary.cost_usd.estimated;
|
||||
const displayedCost = actualCost ?? runReport.summary.cost_usd.estimated;
|
||||
const provider = runReport.details.by_provider.slice(0, 2).map((row) => row.key).join(', ') || 'Not attributed';
|
||||
const metrics = [
|
||||
['Max runtime', formatDuration(runReport.summary.latency_ms.max)],
|
||||
['Tokens', tokenTotal.toLocaleString()],
|
||||
[actualCost ? 'Actual cost' : 'Estimated cost', formatCost(displayedCost)],
|
||||
['Tokens', tokenTotal === null ? 'Unavailable' : tokenTotal.toLocaleString()],
|
||||
[actualCost !== null ? 'Actual cost' : displayedCost !== null ? 'Estimated cost' : 'Cost', displayedCost === null ? 'Unavailable' : formatCost(displayedCost)],
|
||||
['Retries', runReport.summary.retries.toLocaleString()],
|
||||
['Provider calls', runReport.summary.provider_calls.toLocaleString()],
|
||||
['Freshness', runReport.freshness.status],
|
||||
|
||||
@@ -21,6 +21,7 @@ const PAGE_COPY: Record<string, { title: string; eyebrow: string }> = {
|
||||
export function Header() {
|
||||
const { data } = useQuery({ queryKey: ['health'], queryFn: health });
|
||||
const { data: session } = useQuery({ queryKey: ['session'], queryFn: api.session, staleTime: 60_000 });
|
||||
const { data: project } = useQuery({ queryKey: ['project-profile'], queryFn: api.project, staleTime: 60_000 });
|
||||
const { pathname } = useLocation();
|
||||
const stale = data ? !data.ok : true;
|
||||
const page = PAGE_COPY[pathname] ?? PAGE_COPY['/'];
|
||||
@@ -32,6 +33,10 @@ export function Header() {
|
||||
<h1 className="mt-0.5 text-lg font-semibold tracking-tight text-slate-900">{page.title}</h1>
|
||||
</div>
|
||||
<div className="flex items-center gap-2.5 text-xs">
|
||||
<div className="hidden rounded-xl border border-slate-200 bg-white px-3 py-1.5 md:block">
|
||||
<div className="max-w-48 truncate font-semibold text-slate-800">{project?.project_id ?? 'Discovering project…'}</div>
|
||||
<div className="text-[10px] font-bold uppercase tracking-wide text-slate-400">{project?.edition ?? 'unknown'} edition · maturity {project?.maturity.level ? `L${project.maturity.level}` : 'not assessed'}</div>
|
||||
</div>
|
||||
<div className="hidden rounded-xl border border-slate-200 bg-slate-50 px-3 py-2 text-slate-500 sm:block">
|
||||
<span className="font-medium text-slate-700">{data?.runs ?? '—'}</span> governed runs
|
||||
</div>
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
import type { HarnessGateStatus, HarnessTraceGraph } from '../../lib/api';
|
||||
|
||||
const TONE: Record<HarnessGateStatus, string> = {
|
||||
queued: 'border-slate-200 bg-slate-100 text-slate-400',
|
||||
running: 'border-blue-300 bg-blue-50 text-blue-700',
|
||||
pass: 'border-emerald-300 bg-emerald-50 text-emerald-700',
|
||||
warning: 'border-amber-300 bg-amber-50 text-amber-700',
|
||||
blocked: 'border-rose-300 bg-rose-50 text-rose-700',
|
||||
error: 'border-rose-300 bg-rose-50 text-rose-700',
|
||||
skipped: 'border-slate-200 bg-slate-100 text-slate-400',
|
||||
};
|
||||
|
||||
const MARK: Record<HarnessGateStatus, string> = {
|
||||
queued: '○',
|
||||
running: '●',
|
||||
pass: '✓',
|
||||
warning: '!',
|
||||
blocked: '×',
|
||||
error: '×',
|
||||
skipped: '–',
|
||||
};
|
||||
|
||||
export function AssuranceRail({ graph }: { graph: HarnessTraceGraph }) {
|
||||
return (
|
||||
<div className="overflow-x-auto pb-1">
|
||||
<div className="flex min-w-[760px] items-center">
|
||||
{graph.nodes.map((node, index) => (
|
||||
<div key={node.id} className="flex min-w-0 flex-1 items-center">
|
||||
<div className={`min-w-0 flex-1 rounded-xl border p-3 ${TONE[node.status]}`}>
|
||||
<div className="flex items-center justify-between gap-2">
|
||||
<span className="text-[10px] font-black uppercase tracking-[0.12em]">{node.title.split(' · ')[0]}</span>
|
||||
<span className={`flex h-5 w-5 items-center justify-center rounded-full border border-current text-[10px] ${node.status === 'running' ? 'animate-pulse' : ''}`}>{MARK[node.status]}</span>
|
||||
</div>
|
||||
<div className="mt-2 truncate text-xs font-semibold">{node.title.split(' · ')[1]}</div>
|
||||
</div>
|
||||
{index < graph.nodes.length - 1 && <div className={`h-px w-3 shrink-0 ${node.status === 'pass' ? 'bg-emerald-300' : 'bg-slate-200'}`} />}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
import { useEffect, useMemo, useState } from 'react';
|
||||
import { useQuery } from '@tanstack/react-query';
|
||||
import { api, type HarnessGateNode, type HarnessGateStatus, type HarnessTraceGraph } from '../../lib/api';
|
||||
import { api, runReportExportUrl, type HarnessGateNode, type HarnessGateStatus, type HarnessTraceGraph } from '../../lib/api';
|
||||
import { Card, StatusBadge } from '../ui/Card';
|
||||
|
||||
const NODE_TONE: Record<HarnessGateStatus, string> = {
|
||||
@@ -82,7 +82,11 @@ export function TraceExplorer({ traceId, onClose }: { traceId: string; onClose?:
|
||||
return (
|
||||
<Card
|
||||
title="Harness trace explorer"
|
||||
right={onClose && <button type="button" onClick={onClose} className="rounded-lg border border-slate-200 px-3 py-1.5 text-xs font-medium text-slate-600 transition-colors hover:bg-slate-50">Close</button>}
|
||||
right={<div className="flex items-center gap-2">
|
||||
<a href={`/reports/h6?run=${encodeURIComponent(traceId)}`} className="rounded-lg border border-indigo-200 bg-indigo-50 px-3 py-1.5 text-xs font-semibold text-indigo-700 transition hover:bg-indigo-100">Open H6</a>
|
||||
<a href={runReportExportUrl(traceId, 'html')} className="rounded-lg bg-slate-900 px-3 py-1.5 text-xs font-semibold text-white transition hover:bg-slate-700">Export evidence</a>
|
||||
{onClose && <button type="button" onClick={onClose} className="rounded-lg border border-slate-200 px-3 py-1.5 text-xs font-medium text-slate-600 transition-colors hover:bg-slate-50">Close</button>}
|
||||
</div>}
|
||||
className="overflow-hidden"
|
||||
>
|
||||
<div className="mb-5 flex flex-wrap items-end justify-between gap-3">
|
||||
|
||||
@@ -59,8 +59,8 @@ export interface H6Breakdown {
|
||||
runs: number;
|
||||
failures: number;
|
||||
latency_avg_ms: number;
|
||||
tokens: number;
|
||||
cost_usd: number;
|
||||
tokens: number | null;
|
||||
cost_usd: number | null;
|
||||
}
|
||||
|
||||
export interface H6Report {
|
||||
@@ -87,10 +87,18 @@ export interface H6Report {
|
||||
failure_rate_pct: number;
|
||||
retries: number;
|
||||
latency_ms: { average: number; p50: number; p95: number; p99: number; max: number };
|
||||
tokens: { input: number; output: number; total: number; provider_total: number };
|
||||
cost_usd: { provider_actual: number; estimated: number };
|
||||
tokens: { input: number | null; output: number | null; total: number | null; provider_total: number | null };
|
||||
cost_usd: { provider_actual: number | null; estimated: number | null };
|
||||
provider_calls: number;
|
||||
alerts: number;
|
||||
coverage: {
|
||||
runtime_records: number;
|
||||
token_records: number;
|
||||
cost_records: number;
|
||||
token_pct: number;
|
||||
cost_pct: number;
|
||||
quality: { complete: number; partial: number; insufficient: number; unknown: number };
|
||||
};
|
||||
};
|
||||
thresholds: Record<string, number | string | boolean>;
|
||||
findings: H6ReportFinding[];
|
||||
@@ -141,11 +149,22 @@ export interface EvidencePackDetail extends EvidencePackSummary {
|
||||
export interface Overview extends Freshness {
|
||||
totals: {
|
||||
runs: number; total_cost: number; avg_latency_ms: number; failures: number;
|
||||
token_coverage_pct: number; cost_coverage_pct: number;
|
||||
hallucination_signals: number; provider_tokens: number; provider_cost: number;
|
||||
fallback_routes: number; tool_denies: number; action_blocks: number;
|
||||
};
|
||||
harness_signals: Record<string, Record<string, number | string>>;
|
||||
audit_chain: { records: number; head: string | null; last_decision: string | null };
|
||||
latest_assurance: {
|
||||
trace_id: string;
|
||||
certified: boolean;
|
||||
certification_strength: string | null;
|
||||
telemetry_quality: string;
|
||||
duration_ms: number | null;
|
||||
tool_calls: number;
|
||||
failures: number;
|
||||
graph: HarnessTraceGraph;
|
||||
} | null;
|
||||
}
|
||||
|
||||
export interface SettingsActor {
|
||||
@@ -581,8 +600,25 @@ export function h6ReportExportUrl(query: H6ReportQuery, format: 'json' | 'html')
|
||||
return `${base}/reports/h6/export?${params.toString()}`;
|
||||
}
|
||||
|
||||
export function runReportExportUrl(traceId: string, format: 'json' | 'html'): string {
|
||||
const base = import.meta.env.VITE_API_BASE_URL ?? '/api/v1';
|
||||
return `${base}/reports/run/${encodeURIComponent(traceId)}/export?format=${format}`;
|
||||
}
|
||||
|
||||
export interface ProjectProfile {
|
||||
project_id: string;
|
||||
app_root: string;
|
||||
edition: string;
|
||||
edition_status: string;
|
||||
maturity: { level: number | null; status: string; evidence: string | null };
|
||||
enforcement_mode: string;
|
||||
integration_mode: string;
|
||||
clients: string[];
|
||||
}
|
||||
|
||||
export const api = {
|
||||
session: () => get<SettingsActor>('session'),
|
||||
project: () => get<ProjectProfile>('project'),
|
||||
overview: () => get<Overview>('overview'),
|
||||
runs: (limit = 50) => get<Freshness & { count: number; runs: HarnessRunRecord[] }>(`runs?limit=${limit}`),
|
||||
traceGraph: (traceId: string) => get<HarnessTraceGraph>(`runs/${encodeURIComponent(traceId)}/graph`),
|
||||
|
||||
@@ -31,7 +31,7 @@ function MetricCard({ label, value, sub, accent = false }: { label: string; valu
|
||||
return <div className={`relative overflow-hidden rounded-2xl border p-4 ${accent ? 'border-indigo-200 bg-indigo-50/70' : 'border-slate-200 bg-white'}`}>
|
||||
{accent && <span className="absolute -right-5 -top-5 h-16 w-16 rounded-full bg-indigo-200/50 blur-xl" />}
|
||||
<div className="relative text-[10px] font-bold uppercase tracking-[0.12em] text-slate-500">{label}</div>
|
||||
<div className="relative mt-2 text-2xl font-semibold tracking-tight text-slate-900">{value}</div>
|
||||
<div className={`relative mt-2 font-semibold tracking-tight text-slate-900 ${value.length > 9 ? 'text-base' : 'text-2xl'}`}>{value}</div>
|
||||
{sub && <div className="relative mt-1 text-xs text-slate-500">{sub}</div>}
|
||||
</div>;
|
||||
}
|
||||
@@ -42,7 +42,7 @@ function BreakdownTable({ rows, subject }: { rows: H6Breakdown[]; subject: strin
|
||||
<th className="py-2.5 pr-3">{subject}</th><th>runs</th><th>failures</th><th>avg latency</th><th>tokens</th><th>cost</th>
|
||||
</tr></thead>
|
||||
<tbody>{rows.map((row) => <tr key={row.key} className="border-b border-slate-100 transition-colors hover:bg-slate-50/80">
|
||||
<td className="max-w-[300px] py-3 pr-3 font-medium text-slate-800">{row.key}</td><td>{integer(row.runs)}</td><td>{integer(row.failures)}</td><td>{integer(row.latency_avg_ms)} ms</td><td>{integer(row.tokens)}</td><td>{money(row.cost_usd)}</td>
|
||||
<td className="max-w-[300px] py-3 pr-3 font-medium text-slate-800">{row.key}</td><td>{integer(row.runs)}</td><td>{integer(row.failures)}</td><td>{integer(row.latency_avg_ms)} ms</td><td>{row.tokens === null ? 'Unavailable' : integer(row.tokens)}</td><td>{row.cost_usd === null ? 'Unavailable' : money(row.cost_usd)}</td>
|
||||
</tr>)}{rows.length === 0 && <tr><td colSpan={6} className="py-8 text-center text-slate-500">No matching {subject.toLowerCase()} records.</td></tr>}</tbody>
|
||||
</table></div>;
|
||||
}
|
||||
@@ -108,9 +108,22 @@ export function H6ReportPage() {
|
||||
<MetricCard label="Failures" value={integer(report.data.summary.failed)} sub={`${report.data.summary.failure_rate_pct}% rate`} />
|
||||
<MetricCard label="Degraded" value={integer(report.data.summary.degraded)} />
|
||||
<MetricCard label="P95 latency" value={`${integer(report.data.summary.latency_ms.p95)}ms`} sub={`P50 ${integer(report.data.summary.latency_ms.p50)}ms`} />
|
||||
<MetricCard label="Provider tokens" value={integer(report.data.summary.tokens.provider_total)} sub={`${report.data.summary.provider_calls} calls`} accent />
|
||||
<MetricCard label="Actual cost" value={money(report.data.summary.cost_usd.provider_actual)} sub="provider reported" />
|
||||
<MetricCard label="Estimated cost" value={money(report.data.summary.cost_usd.estimated)} sub="runtime fallback" />
|
||||
<MetricCard
|
||||
label="Provider tokens"
|
||||
value={report.data.summary.coverage.token_records > 0 ? integer(report.data.summary.tokens.provider_total ?? report.data.summary.tokens.total ?? 0) : 'Unavailable'}
|
||||
sub={`${report.data.summary.coverage.token_pct}% run coverage`}
|
||||
accent
|
||||
/>
|
||||
<MetricCard
|
||||
label="Actual cost"
|
||||
value={report.data.summary.cost_usd.provider_actual !== null ? money(report.data.summary.cost_usd.provider_actual) : 'Unavailable'}
|
||||
sub={`${report.data.summary.provider_calls} provider calls`}
|
||||
/>
|
||||
<MetricCard
|
||||
label="Cost coverage"
|
||||
value={`${report.data.summary.coverage.cost_pct}%`}
|
||||
sub={`${report.data.summary.coverage.cost_records}/${report.data.summary.coverage.runtime_records} runs`}
|
||||
/>
|
||||
<MetricCard label="Alerts" value={integer(report.data.summary.alerts)} sub={`${report.data.summary.retries} retries`} />
|
||||
</section>
|
||||
|
||||
@@ -127,6 +140,10 @@ export function H6ReportPage() {
|
||||
|
||||
<div className="grid gap-5 xl:grid-cols-[0.8fr_1.2fr]">
|
||||
<Card title={`Data quality · ${report.data.data_quality.status}`}>
|
||||
<div className="mb-4 grid grid-cols-2 gap-3">
|
||||
<div className="rounded-xl border border-slate-200 bg-slate-50 p-3"><div className="text-[10px] font-bold uppercase tracking-wider text-slate-400">Token coverage</div><div className="mt-1 text-xl font-semibold text-slate-900">{report.data.summary.coverage.token_pct}%</div></div>
|
||||
<div className="rounded-xl border border-slate-200 bg-slate-50 p-3"><div className="text-[10px] font-bold uppercase tracking-wider text-slate-400">Cost coverage</div><div className="mt-1 text-xl font-semibold text-slate-900">{report.data.summary.coverage.cost_pct}%</div></div>
|
||||
</div>
|
||||
{report.data.data_quality.warnings.length > 0 ? <ul className="space-y-2 text-sm leading-5 text-slate-600">{report.data.data_quality.warnings.map((warning) => <li key={warning} className="flex gap-2"><span className="mt-1 h-2 w-2 shrink-0 rounded-full bg-amber-400" />{warning}</li>)}</ul> : <div className="text-sm text-emerald-700">All required sources are present and no estimation warning was detected.</div>}
|
||||
</Card>
|
||||
<Card title="Independent export" right={<span className="text-xs font-normal normal-case tracking-normal text-slate-400">Generated on demand · no hard-coded score</span>}>
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import { useQuery } from '@tanstack/react-query';
|
||||
import { api } from '../lib/api';
|
||||
import { Card, StatTile, StatusBadge } from '../components/ui/Card';
|
||||
import { AssuranceRail } from '../components/trace/AssuranceRail';
|
||||
import { Link } from 'react-router-dom';
|
||||
|
||||
function SignalMark({ tone }: { tone: 'indigo' | 'emerald' | 'amber' }) {
|
||||
const color = { indigo: 'bg-indigo-500', emerald: 'bg-emerald-500', amber: 'bg-amber-500' }[tone];
|
||||
@@ -39,10 +41,41 @@ export function Overview() {
|
||||
</div>
|
||||
</section>
|
||||
|
||||
{data.latest_assurance && (
|
||||
<section className="rounded-2xl border border-slate-200 bg-white p-5 shadow-[0_15px_35px_rgba(15,23,42,0.05)] sm:p-6">
|
||||
<div className="mb-4 flex flex-col justify-between gap-3 sm:flex-row sm:items-end">
|
||||
<div>
|
||||
<div className="text-[10px] font-black uppercase tracking-[0.18em] text-indigo-600">Latest assurance receipt</div>
|
||||
<div className="mt-1 flex flex-wrap items-center gap-2">
|
||||
<h3 className="text-lg font-semibold text-slate-900">{data.latest_assurance.certified ? 'Certified governed run' : 'Run requires review'}</h3>
|
||||
<StatusBadge value={data.latest_assurance.certified ? 'certified' : 'attention'} />
|
||||
<StatusBadge value={`H6 ${data.latest_assurance.telemetry_quality}`} />
|
||||
</div>
|
||||
<p className="mt-1 max-w-2xl truncate font-mono text-[11px] text-slate-400">{data.latest_assurance.trace_id}</p>
|
||||
</div>
|
||||
<div className="flex flex-wrap gap-2">
|
||||
<Link to={`/reports/h6?run=${encodeURIComponent(data.latest_assurance.trace_id)}`} className="rounded-xl border border-indigo-200 bg-indigo-50 px-3 py-2 text-xs font-semibold text-indigo-700 transition hover:bg-indigo-100">Inspect H6</Link>
|
||||
<Link to={`/runs?trace=${encodeURIComponent(data.latest_assurance.trace_id)}`} className="rounded-xl bg-slate-900 px-3 py-2 text-xs font-semibold text-white transition hover:bg-slate-700">Open live trace</Link>
|
||||
</div>
|
||||
</div>
|
||||
<AssuranceRail graph={data.latest_assurance.graph} />
|
||||
<div className="mt-4 flex flex-wrap gap-x-5 gap-y-2 border-t border-slate-100 pt-4 text-xs text-slate-500">
|
||||
<span><strong className="text-slate-800">{data.latest_assurance.graph.progress}/7</strong> gates observed</span>
|
||||
<span><strong className="text-slate-800">{data.latest_assurance.tool_calls}</strong> tools</span>
|
||||
<span><strong className="text-slate-800">{data.latest_assurance.failures}</strong> failures</span>
|
||||
<span><strong className="text-slate-800">{data.latest_assurance.duration_ms == null ? '—' : `${Math.round(data.latest_assurance.duration_ms / 1000)}s`}</strong> duration</span>
|
||||
</div>
|
||||
</section>
|
||||
)}
|
||||
|
||||
<div className="grid grid-cols-2 gap-3 md:grid-cols-4 xl:grid-cols-8">
|
||||
<StatTile label="Runs" value={totals.runs} />
|
||||
<StatTile label="Failures" value={totals.failures} />
|
||||
<StatTile label="Model cost" value={`$${totals.total_cost.toFixed(4)}`} sub={`${totals.provider_tokens.toLocaleString()} tokens`} />
|
||||
<StatTile
|
||||
label="Model cost"
|
||||
value={totals.cost_coverage_pct > 0 ? `$${totals.total_cost.toFixed(4)}` : 'Unavailable'}
|
||||
sub={`${totals.cost_coverage_pct}% cost coverage`}
|
||||
/>
|
||||
<StatTile label="Latency" value={`${totals.avg_latency_ms}ms`} sub="average" />
|
||||
<StatTile label="Fallbacks" value={totals.fallback_routes} />
|
||||
<StatTile label="Tool denies" value={totals.tool_denies} />
|
||||
|
||||
@@ -30,7 +30,7 @@ export function Runs() {
|
||||
<td><StatusBadge value={run.status ?? '—'} /></td>
|
||||
<td>{run.latency_ms ?? '—'} ms</td>
|
||||
<td>{run.total_tokens ?? '—'}</td>
|
||||
<td>${Number(run.cost_estimate ?? 0).toFixed(5)}</td>
|
||||
<td>{run.cost_estimate == null ? <span className="font-medium text-amber-600">Unavailable</span> : `$${Number(run.cost_estimate).toFixed(5)}`}</td>
|
||||
<td className="py-2 text-right">
|
||||
{run.trace_id ? (
|
||||
<button
|
||||
|
||||
+179
@@ -0,0 +1,179 @@
|
||||
#!/usr/bin/env bash
|
||||
# One-command local CASAN Control Plane for demos and offline evaluation.
|
||||
# Platform-only: Core stays a non-blocking sensor/enforcer and local spool.
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
SOURCE_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
|
||||
COMMAND="${1:-start}"
|
||||
[[ $# -gt 0 ]] && shift || true
|
||||
|
||||
TARGET="${CASAN_APP_ROOT:-$PWD}"
|
||||
OPEN_BROWSER=1
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--root) TARGET="${2:-}"; shift 2 ;;
|
||||
--no-open) OPEN_BROWSER=0; shift ;;
|
||||
*) echo "casan dashboard: unknown argument: $1" >&2; exit 64 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
TARGET="$(cd "$TARGET" 2>/dev/null && pwd)" || {
|
||||
echo "casan dashboard: project root does not exist: $TARGET" >&2
|
||||
exit 66
|
||||
}
|
||||
[[ -f "$TARGET/.casan/config.json" ]] || {
|
||||
echo "casan dashboard: $TARGET is not an adopted CASAN project" >&2
|
||||
exit 65
|
||||
}
|
||||
[[ -f "$SOURCE_ROOT/package.json" ]] || {
|
||||
echo "casan dashboard: Platform source bundle is incomplete" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
STATE_DIR="${CASAN_CONTROL_PLANE_STATE_DIR:-$TARGET/.specify/state/control-plane}"
|
||||
mkdir -p "$STATE_DIR"
|
||||
API_PID_FILE="$STATE_DIR/api.pid"
|
||||
UI_PID_FILE="$STATE_DIR/ui.pid"
|
||||
API_LOG="$STATE_DIR/api.log"
|
||||
UI_LOG="$STATE_DIR/ui.log"
|
||||
DASHBOARD_URL="${CASAN_DASHBOARD_URL:-http://127.0.0.1:5174}"
|
||||
|
||||
alive() {
|
||||
local file="$1" pid
|
||||
[[ -f "$file" ]] || return 1
|
||||
pid="$(sed -n '1p' "$file" 2>/dev/null || true)"
|
||||
[[ "$pid" =~ ^[0-9]+$ ]] && kill -0 "$pid" 2>/dev/null
|
||||
}
|
||||
|
||||
enroll() {
|
||||
python3 - "$TARGET/.casan/config.json" "$DASHBOARD_URL" <<'PY'
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
path, url = sys.argv[1:3]
|
||||
with open(path, "r", encoding="utf-8") as handle:
|
||||
config = json.load(handle)
|
||||
control_plane = config.get("control_plane")
|
||||
if not isinstance(control_plane, dict):
|
||||
control_plane = {}
|
||||
control_plane.update({
|
||||
"dashboard_url": url.rstrip("/"),
|
||||
"delivery": "local_spool",
|
||||
"receipt_links_enabled": True,
|
||||
})
|
||||
config["control_plane"] = control_plane
|
||||
directory = os.path.dirname(path)
|
||||
fd, temporary = tempfile.mkstemp(prefix=".config.", suffix=".tmp", dir=directory)
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
||||
json.dump(config, handle, ensure_ascii=False, indent=2)
|
||||
handle.write("\n")
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(temporary, path)
|
||||
finally:
|
||||
if os.path.exists(temporary):
|
||||
os.unlink(temporary)
|
||||
PY
|
||||
}
|
||||
|
||||
open_dashboard() {
|
||||
python3 - "$DASHBOARD_URL" <<'PY'
|
||||
import sys
|
||||
import webbrowser
|
||||
raise SystemExit(0 if webbrowser.open(sys.argv[1], new=2) else 1)
|
||||
PY
|
||||
}
|
||||
|
||||
case "$COMMAND" in
|
||||
start)
|
||||
if alive "$API_PID_FILE" && alive "$UI_PID_FILE"; then
|
||||
enroll
|
||||
echo "CASAN_CONTROL_PLANE_READY url=$DASHBOARD_URL project=$TARGET"
|
||||
[[ "$OPEN_BROWSER" == "1" ]] && open_dashboard || true
|
||||
exit 0
|
||||
fi
|
||||
python3 - "$SOURCE_ROOT" "$TARGET" "$API_LOG" "$UI_LOG" "$API_PID_FILE" "$UI_PID_FILE" <<'PY'
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
source, target, api_log, ui_log, api_pid, ui_pid = sys.argv[1:7]
|
||||
environment = os.environ.copy()
|
||||
environment["CASAN_APP_ROOT"] = target
|
||||
with open(api_log, "ab", buffering=0) as api_output:
|
||||
api = subprocess.Popen(
|
||||
[os.path.join(source, "node_modules", ".bin", "tsx"), "src/main.ts"],
|
||||
cwd=os.path.join(source, "packages", "casan-control-panel", "backend"),
|
||||
env=environment,
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=api_output,
|
||||
stderr=subprocess.STDOUT,
|
||||
start_new_session=True,
|
||||
)
|
||||
with open(ui_log, "ab", buffering=0) as ui_output:
|
||||
ui = subprocess.Popen(
|
||||
[
|
||||
os.path.join(source, "node_modules", ".bin", "vite"),
|
||||
"--host", "127.0.0.1", "--port", "5174",
|
||||
],
|
||||
cwd=os.path.join(source, "packages", "casan-control-panel", "frontend"),
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=ui_output,
|
||||
stderr=subprocess.STDOUT,
|
||||
start_new_session=True,
|
||||
)
|
||||
for path, pid in ((api_pid, api.pid), (ui_pid, ui.pid)):
|
||||
with open(path, "w", encoding="utf-8") as handle:
|
||||
handle.write("%s\n" % pid)
|
||||
PY
|
||||
ready=0
|
||||
for _attempt in $(seq 1 60); do
|
||||
if curl -fsS "http://127.0.0.1:3010/api/v1/project" >/dev/null 2>&1 \
|
||||
&& curl -fsS "$DASHBOARD_URL" >/dev/null 2>&1; then
|
||||
ready=1
|
||||
break
|
||||
fi
|
||||
sleep 0.25
|
||||
done
|
||||
if [[ "$ready" != "1" ]]; then
|
||||
echo "casan dashboard: Control Plane did not become ready" >&2
|
||||
echo "API log: $API_LOG" >&2
|
||||
echo "UI log: $UI_LOG" >&2
|
||||
exit 1
|
||||
fi
|
||||
enroll
|
||||
echo "CASAN_CONTROL_PLANE_READY url=$DASHBOARD_URL project=$TARGET"
|
||||
echo "logs=$STATE_DIR"
|
||||
[[ "$OPEN_BROWSER" == "1" ]] && open_dashboard || true
|
||||
;;
|
||||
status)
|
||||
api="stopped"; ui="stopped"
|
||||
alive "$API_PID_FILE" && api="running"
|
||||
alive "$UI_PID_FILE" && ui="running"
|
||||
echo "CASAN_CONTROL_PLANE_STATUS api=$api ui=$ui url=$DASHBOARD_URL project=$TARGET"
|
||||
[[ "$api" == "running" && "$ui" == "running" ]]
|
||||
;;
|
||||
stop)
|
||||
for file in "$API_PID_FILE" "$UI_PID_FILE"; do
|
||||
if alive "$file"; then
|
||||
pid="$(sed -n '1p' "$file")"
|
||||
kill "$pid"
|
||||
fi
|
||||
rm -f "$file"
|
||||
done
|
||||
echo "CASAN_CONTROL_PLANE_STOPPED project=$TARGET"
|
||||
;;
|
||||
open)
|
||||
enroll
|
||||
echo "$DASHBOARD_URL"
|
||||
open_dashboard
|
||||
;;
|
||||
*)
|
||||
echo "casan: usage: casan dashboard <start|status|stop|open> [--root path] [--no-open]" >&2
|
||||
exit 64
|
||||
;;
|
||||
esac
|
||||
Reference in New Issue
Block a user