feat: add production assurance dashboard flow
This commit is contained in:
thanhnv
@@ -10,9 +10,10 @@ import { GoalsModule } from './goals/goals.module.js';
|
||||
import { EvidenceModule } from './evidence/evidence.module.js';
|
||||
import { SessionController } from './session/session.controller.js';
|
||||
import { ReportsModule } from './reports/reports.module.js';
|
||||
import { IngestModule } from './ingest/ingest.module.js';
|
||||
|
||||
@Module({
|
||||
imports: [TelemetryModule, SettingsModule, KillSwitchModule, ApprovalsModule, ChatModule, ProviderAuthModule, GoalsModule, EvidenceModule, ReportsModule],
|
||||
imports: [TelemetryModule, SettingsModule, KillSwitchModule, ApprovalsModule, ChatModule, ProviderAuthModule, GoalsModule, EvidenceModule, ReportsModule, IngestModule],
|
||||
controllers: [HealthController, SessionController],
|
||||
})
|
||||
export class AppModule {}
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
import { Body, Controller, Headers, Inject, Post } from '@nestjs/common';
|
||||
import { ok } from '../common/api-response.js';
|
||||
import { IngestService } from './ingest.service.js';
|
||||
|
||||
@Controller('api/v1/ingest')
|
||||
export class IngestController {
|
||||
constructor(@Inject(IngestService) private readonly ingestService: IngestService) {}
|
||||
|
||||
@Post('turn')
|
||||
turn(
|
||||
@Body() body: unknown,
|
||||
@Headers('x-casan-timestamp') timestamp?: string,
|
||||
@Headers('x-casan-signature') signature?: string,
|
||||
) {
|
||||
return ok(this.ingestService.ingest(body, timestamp, signature));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { IngestController } from './ingest.controller.js';
|
||||
import { IngestService } from './ingest.service.js';
|
||||
|
||||
@Module({
|
||||
controllers: [IngestController],
|
||||
providers: [IngestService],
|
||||
})
|
||||
export class IngestModule {}
|
||||
@@ -0,0 +1,119 @@
|
||||
import { createHmac, timingSafeEqual } from 'node:crypto';
|
||||
import { appendFileSync, existsSync, mkdirSync, renameSync, writeFileSync } from 'node:fs';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { BadRequestException, ForbiddenException, Injectable, ServiceUnavailableException } from '@nestjs/common';
|
||||
import { APP_ROOT, PATHS } from '../common/app-root.js';
|
||||
|
||||
type Row = Record<string, unknown>;
|
||||
|
||||
export interface IngestEnvelope {
|
||||
schema_version: 1;
|
||||
sent_at: string;
|
||||
project_id: string;
|
||||
trace_id: string;
|
||||
receipt: Row;
|
||||
metric: Row;
|
||||
trace: Row;
|
||||
events: Row[];
|
||||
}
|
||||
|
||||
const SAFE_ID = /^[a-zA-Z0-9][a-zA-Z0-9._:-]{0,127}$/;
|
||||
const FORBIDDEN_KEYS = new Set([
|
||||
'prompt', 'raw_prompt', 'assistant_summary', 'tool_input', 'tool_output',
|
||||
'result_content', 'secret', 'password', 'authorization',
|
||||
]);
|
||||
|
||||
export function stableJson(value: unknown): string {
|
||||
if (Array.isArray(value)) return `[${value.map(stableJson).join(',')}]`;
|
||||
if (value && typeof value === 'object') {
|
||||
const record = value as Row;
|
||||
return `{${Object.keys(record).sort().map((key) => `${JSON.stringify(key)}:${stableJson(record[key])}`).join(',')}}`;
|
||||
}
|
||||
return JSON.stringify(value) ?? 'null';
|
||||
}
|
||||
|
||||
export function expectedSignature(token: string, timestamp: string, body: unknown): string {
|
||||
return `sha256=${createHmac('sha256', token).update(`${timestamp}.${stableJson(body)}`).digest('hex')}`;
|
||||
}
|
||||
|
||||
function containsForbiddenKey(value: unknown): boolean {
|
||||
if (Array.isArray(value)) return value.some(containsForbiddenKey);
|
||||
if (!value || typeof value !== 'object') return false;
|
||||
return Object.entries(value as Row).some(([key, nested]) => (
|
||||
FORBIDDEN_KEYS.has(key.toLowerCase()) || containsForbiddenKey(nested)
|
||||
));
|
||||
}
|
||||
|
||||
function safeEqual(left: string, right: string): boolean {
|
||||
const leftBuffer = Buffer.from(left);
|
||||
const rightBuffer = Buffer.from(right);
|
||||
return leftBuffer.length === rightBuffer.length && timingSafeEqual(leftBuffer, rightBuffer);
|
||||
}
|
||||
|
||||
function atomicJson(path: string, value: unknown): void {
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
const temporary = `${path}.${process.pid}.tmp`;
|
||||
writeFileSync(temporary, `${JSON.stringify(value, null, 2)}\n`, { encoding: 'utf8', mode: 0o600 });
|
||||
renameSync(temporary, path);
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class IngestService {
|
||||
ingest(raw: unknown, timestamp: string | undefined, signature: string | undefined) {
|
||||
const token = process.env.CASAN_CP_INGEST_TOKEN;
|
||||
if (!token) throw new ServiceUnavailableException('CASAN_INGEST_DISABLED');
|
||||
const epoch = Number(timestamp);
|
||||
if (!Number.isInteger(epoch) || Math.abs(Math.floor(Date.now() / 1000) - epoch) > 300) {
|
||||
throw new ForbiddenException('CASAN_INGEST_TIMESTAMP_INVALID');
|
||||
}
|
||||
const expected = expectedSignature(token, String(timestamp), raw);
|
||||
if (!signature || !safeEqual(signature, expected)) {
|
||||
throw new ForbiddenException('CASAN_INGEST_SIGNATURE_INVALID');
|
||||
}
|
||||
if (!raw || typeof raw !== 'object' || Array.isArray(raw)) {
|
||||
throw new BadRequestException('CASAN_INGEST_INVALID_ENVELOPE');
|
||||
}
|
||||
const envelope = raw as Partial<IngestEnvelope>;
|
||||
if (
|
||||
envelope.schema_version !== 1
|
||||
|| typeof envelope.project_id !== 'string'
|
||||
|| typeof envelope.trace_id !== 'string'
|
||||
|| !SAFE_ID.test(envelope.project_id)
|
||||
|| !SAFE_ID.test(envelope.trace_id)
|
||||
|| !envelope.metric
|
||||
|| !envelope.trace
|
||||
|| !Array.isArray(envelope.events)
|
||||
|| envelope.events.length > 2000
|
||||
) throw new BadRequestException('CASAN_INGEST_INVALID_ENVELOPE');
|
||||
if (containsForbiddenKey(envelope)) {
|
||||
throw new BadRequestException('CASAN_INGEST_RAW_CONTENT_FORBIDDEN');
|
||||
}
|
||||
if (
|
||||
envelope.metric.trace_id !== envelope.trace_id
|
||||
|| envelope.trace.trace_id !== envelope.trace_id
|
||||
|| envelope.events.some((event) => event.trace_id !== envelope.trace_id)
|
||||
) throw new BadRequestException('CASAN_INGEST_TRACE_MISMATCH');
|
||||
|
||||
const tracePath = join(PATHS.traceDir, `agentic-${envelope.trace_id}.json`);
|
||||
if (existsSync(tracePath)) {
|
||||
return { accepted: true, duplicate: true, trace_id: envelope.trace_id };
|
||||
}
|
||||
|
||||
mkdirSync(dirname(PATHS.metrics), { recursive: true });
|
||||
appendFileSync(PATHS.metrics, `${JSON.stringify(envelope.metric)}\n`, 'utf8');
|
||||
atomicJson(tracePath, envelope.trace);
|
||||
if (envelope.events.length > 0) {
|
||||
const eventPath = join(PATHS.traceEventDir, `${envelope.trace_id}.jsonl`);
|
||||
mkdirSync(dirname(eventPath), { recursive: true });
|
||||
appendFileSync(eventPath, `${envelope.events.map((event) => JSON.stringify(event)).join('\n')}\n`, 'utf8');
|
||||
}
|
||||
const receiptPath = join(APP_ROOT, '.specify', 'state', 'ingested', `${envelope.trace_id}.json`);
|
||||
atomicJson(receiptPath, envelope.receipt ?? {});
|
||||
return {
|
||||
accepted: true,
|
||||
duplicate: false,
|
||||
project_id: envelope.project_id,
|
||||
trace_id: envelope.trace_id,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -14,7 +14,7 @@ function breakdownRows(rows: H6Breakdown[]): string {
|
||||
if (rows.length === 0) return '<tr><td colspan="6" class="empty">No records in the selected scope.</td></tr>';
|
||||
return rows.map((row) => `<tr>
|
||||
<td>${escapeHtml(row.key)}</td><td>${number(row.runs)}</td><td>${number(row.failures)}</td>
|
||||
<td>${number(row.latency_avg_ms)} ms</td><td>${number(row.tokens)}</td><td>${money(row.cost_usd)}</td>
|
||||
<td>${number(row.latency_avg_ms)} ms</td><td>${row.tokens === null ? 'Unavailable' : number(row.tokens)}</td><td>${row.cost_usd === null ? 'Unavailable' : money(row.cost_usd)}</td>
|
||||
</tr>`).join('');
|
||||
}
|
||||
|
||||
@@ -50,7 +50,7 @@ main{max-width:1180px;margin:0 auto;padding:42px 28px 72px}.hero{position:relati
|
||||
@media print{body{background:#fff}main{max-width:none;padding:0}.hero,.card,.section{box-shadow:none}.hero{background:#111827!important;-webkit-print-color-adjust:exact;print-color-adjust:exact}.section{break-inside:avoid}.footer{margin-top:10px}}
|
||||
</style></head><body><main>
|
||||
<section class="hero"><div class="eyebrow">CASAN assurance dossier · contract v${report.schema_version}</div><h1>${escapeHtml(report.title)}</h1><div class="scope">${scope.map((item) => `<span>${escapeHtml(item)}</span>`).join('')}</div><div class="verdict ${verdictClass}">${escapeHtml(report.verdict)}</div><div class="meta">Generated ${escapeHtml(report.generated_at)} · ${escapeHtml(report.report_id)} · Freshness ${escapeHtml(report.freshness.status)}</div></section>
|
||||
<section class="grid"><div class="card"><span>Governed runs</span><strong>${number(report.summary.runs)}</strong></div><div class="card"><span>Failure rate</span><strong>${report.summary.failure_rate_pct}%</strong></div><div class="card"><span>P95 latency</span><strong>${number(report.summary.latency_ms.p95)} ms</strong></div><div class="card"><span>Provider tokens</span><strong>${number(report.summary.tokens.provider_total)}</strong></div><div class="card"><span>Actual provider cost</span><strong>${money(report.summary.cost_usd.provider_actual)}</strong></div><div class="card"><span>Estimated cost</span><strong>${money(report.summary.cost_usd.estimated)}</strong></div><div class="card"><span>Alerts</span><strong>${number(report.summary.alerts)}</strong></div><div class="card"><span>Retries</span><strong>${number(report.summary.retries)}</strong></div></section>
|
||||
<section class="grid"><div class="card"><span>Governed runs</span><strong>${number(report.summary.runs)}</strong></div><div class="card"><span>Failure rate</span><strong>${report.summary.failure_rate_pct}%</strong></div><div class="card"><span>P95 latency</span><strong>${number(report.summary.latency_ms.p95)} ms</strong></div><div class="card"><span>Provider tokens</span><strong>${report.summary.coverage.token_records > 0 ? number(report.summary.tokens.provider_total ?? report.summary.tokens.total ?? 0) : 'Unavailable'}</strong></div><div class="card"><span>Actual provider cost</span><strong>${report.summary.cost_usd.provider_actual !== null ? money(report.summary.cost_usd.provider_actual) : 'Unavailable'}</strong></div><div class="card"><span>Token coverage</span><strong>${report.summary.coverage.token_pct}%</strong></div><div class="card"><span>Cost coverage</span><strong>${report.summary.coverage.cost_pct}%</strong></div><div class="card"><span>Retries</span><strong>${number(report.summary.retries)}</strong></div></section>
|
||||
<section class="section"><h2>Verdict findings</h2><ul class="findings">${findings}</ul></section>
|
||||
<section class="section"><h2>Evidence freshness</h2><div class="table-wrap"><table><thead><tr><th>Source</th><th>Presence</th><th>State</th><th>Age</th><th>Records</th><th>Path</th></tr></thead><tbody>${sourceRows}</tbody></table></div></section>
|
||||
<section class="section"><h2>Step breakdown</h2><div class="table-wrap"><table><thead><tr><th>Step</th><th>Runs</th><th>Failures</th><th>Avg latency</th><th>Tokens</th><th>Cost</th></tr></thead><tbody>${breakdownRows(report.details.by_step)}</tbody></table></div></section>
|
||||
|
||||
@@ -37,10 +37,18 @@ export interface H6ReportSummary {
|
||||
failure_rate_pct: number;
|
||||
retries: number;
|
||||
latency_ms: { average: number; p50: number; p95: number; p99: number; max: number };
|
||||
tokens: { input: number; output: number; total: number; provider_total: number };
|
||||
cost_usd: { provider_actual: number; estimated: number };
|
||||
tokens: { input: number | null; output: number | null; total: number | null; provider_total: number | null };
|
||||
cost_usd: { provider_actual: number | null; estimated: number | null };
|
||||
provider_calls: number;
|
||||
alerts: number;
|
||||
coverage: {
|
||||
runtime_records: number;
|
||||
token_records: number;
|
||||
cost_records: number;
|
||||
token_pct: number;
|
||||
cost_pct: number;
|
||||
quality: { complete: number; partial: number; insufficient: number; unknown: number };
|
||||
};
|
||||
}
|
||||
|
||||
export interface H6Breakdown {
|
||||
@@ -48,8 +56,8 @@ export interface H6Breakdown {
|
||||
runs: number;
|
||||
failures: number;
|
||||
latency_avg_ms: number;
|
||||
tokens: number;
|
||||
cost_usd: number;
|
||||
tokens: number | null;
|
||||
cost_usd: number | null;
|
||||
}
|
||||
|
||||
export interface H6ReportDetails {
|
||||
@@ -66,6 +74,7 @@ export type H6Report = HarnessReport<H6ReportSummary, H6ReportDetails>;
|
||||
|
||||
const safeFilter = /^[a-zA-Z0-9][a-zA-Z0-9._:-]{0,127}$/;
|
||||
const numberValue = (value: unknown): number => typeof value === 'number' && Number.isFinite(value) ? value : 0;
|
||||
const hasNumber = (value: unknown): boolean => typeof value === 'number' && Number.isFinite(value);
|
||||
const stringValue = (value: unknown, fallback: string): string => typeof value === 'string' && value.trim() ? value.trim() : fallback;
|
||||
const round = (value: number, digits = 2): number => Number(value.toFixed(digits));
|
||||
const configuredThreshold = (name: string, fallback: number): number => {
|
||||
@@ -148,8 +157,13 @@ function grouped(rows: TelemetryRow[], keyOf: (row: TelemetryRow) => string): H6
|
||||
runs: records.length,
|
||||
failures: records.filter((row) => stringValue(row.status, 'unknown') === 'failed').length,
|
||||
latency_avg_ms: latencies.length ? Math.round(latencies.reduce((total, value) => total + value, 0) / latencies.length) : 0,
|
||||
tokens: sum(records, 'total_tokens'),
|
||||
cost_usd: round(sum(records, records.some((row) => row.cost_usd !== undefined) ? 'cost_usd' : 'cost_estimate'), 6),
|
||||
tokens: records.some((row) => hasNumber(row.total_tokens))
|
||||
? sum(records, 'total_tokens') : null,
|
||||
cost_usd: records.some((row) => hasNumber(row.cost_usd))
|
||||
? round(sum(records, 'cost_usd'), 6)
|
||||
: records.some((row) => hasNumber(row.cost_estimate))
|
||||
? round(sum(records, 'cost_estimate'), 6)
|
||||
: null,
|
||||
};
|
||||
}).sort((left, right) => right.runs - left.runs || left.key.localeCompare(right.key));
|
||||
}
|
||||
@@ -195,6 +209,29 @@ export function buildH6Report(input: H6ReportInput, query: H6ReportQuery): H6Rep
|
||||
const failed = metrics.filter((row) => stringValue(row.status, 'unknown') === 'failed').length;
|
||||
const degraded = metrics.filter((row) => stringValue(row.status, 'unknown') === 'degraded').length;
|
||||
const success = metrics.filter((row) => ['success', 'pass', 'passed'].includes(stringValue(row.status, 'unknown'))).length;
|
||||
const providerTokenRuns = new Set(provider.filter((row) => hasNumber(row.total_tokens)).map(rowRun));
|
||||
const providerCostRuns = new Set(provider.filter((row) => hasNumber(row.cost_usd)).map(rowRun));
|
||||
const tokenRecords = metrics.filter((row) => (
|
||||
hasNumber(row.total_tokens)
|
||||
|| hasNumber(row.input_tokens)
|
||||
|| hasNumber(row.output_tokens)
|
||||
|| providerTokenRuns.has(rowRun(row))
|
||||
)).length;
|
||||
const costRecords = metrics.filter((row) => (
|
||||
hasNumber(row.cost_estimate)
|
||||
|| providerCostRuns.has(rowRun(row))
|
||||
)).length;
|
||||
const tokenCoverage = metrics.length ? round((tokenRecords / metrics.length) * 100, 1) : 0;
|
||||
const costCoverage = metrics.length ? round((costRecords / metrics.length) * 100, 1) : 0;
|
||||
const qualityCounts = { complete: 0, partial: 0, insufficient: 0, unknown: 0 };
|
||||
for (const row of metrics) {
|
||||
const quality = stringValue(row.telemetry_quality, 'unknown');
|
||||
if (quality === 'complete' || quality === 'partial' || quality === 'insufficient') {
|
||||
qualityCounts[quality] += 1;
|
||||
} else {
|
||||
qualityCounts.unknown += 1;
|
||||
}
|
||||
}
|
||||
const sourceEvidence = evidenceSources(input);
|
||||
const primary = sourceEvidence.find((source) => source.source === 'metrics');
|
||||
const failureRate = metrics.length ? round((failed / metrics.length) * 100, 1) : 0;
|
||||
@@ -223,7 +260,19 @@ export function buildH6Report(input: H6ReportInput, query: H6ReportQuery): H6Rep
|
||||
const alertCount = alertCounts.reduce((total, entry) => total + entry.count, 0);
|
||||
if (alertCount > 0) findings.push({ severity: 'warning', code: 'ALERTS_PRESENT', message: `${alertCount} alert signal(s) require review.`, metric: 'alerts', value: alertCount });
|
||||
if (degraded > 0) findings.push({ severity: 'warning', code: 'DEGRADED_RUNS_PRESENT', message: `${degraded} degraded run(s) are present in the selected scope.`, metric: 'degraded', value: degraded });
|
||||
if (metrics.length > 0 && (tokenCoverage < 100 || costCoverage < 100)) {
|
||||
findings.push({
|
||||
severity: 'warning',
|
||||
code: 'TELEMETRY_COVERAGE_GAP',
|
||||
message: 'Some runs do not have reliable provider token or cost attribution.',
|
||||
metric: 'token_cost_coverage_pct',
|
||||
value: `${tokenCoverage}/${costCoverage}`,
|
||||
threshold: '100/100',
|
||||
});
|
||||
}
|
||||
if (provider.length === 0) warnings.push('No provider usage records matched the selected scope; token and actual-cost breakdown may be incomplete.');
|
||||
if (metrics.length > 0 && tokenCoverage < 100) warnings.push(`${tokenCoverage}% of runtime records have reliable token attribution; unavailable values remain null, never zero.`);
|
||||
if (metrics.length > 0 && costCoverage < 100) warnings.push(`${costCoverage}% of runtime records have reliable cost attribution; unavailable values remain null, never zero.`);
|
||||
for (const source of sourceEvidence) {
|
||||
if (source.source === 'metrics') continue;
|
||||
if (!source.present) warnings.push(`Optional ${source.source} telemetry source is missing; its breakdown is unavailable.`);
|
||||
@@ -243,7 +292,7 @@ export function buildH6Report(input: H6ReportInput, query: H6ReportQuery): H6Rep
|
||||
const costSources = grouped(metrics, (row) => stringValue(row.cost_source, 'unknown')).map((entry) => ({
|
||||
source: entry.key,
|
||||
records: entry.runs,
|
||||
cost_usd: entry.cost_usd,
|
||||
cost_usd: entry.cost_usd ?? 0,
|
||||
}));
|
||||
const scope: HarnessReportScope = { project: query.project, from: query.from, to: query.to, run: query.run };
|
||||
|
||||
@@ -272,15 +321,37 @@ export function buildH6Report(input: H6ReportInput, query: H6ReportQuery): H6Rep
|
||||
p99: percentile(latencies, 99),
|
||||
max: latencies.length ? Math.max(...latencies) : 0,
|
||||
},
|
||||
tokens: { input: sum(metrics, 'input_tokens'), output: sum(metrics, 'output_tokens'), total: sum(metrics, 'total_tokens'), provider_total: sum(provider, 'total_tokens') },
|
||||
cost_usd: { provider_actual: round(sum(provider, 'cost_usd'), 6), estimated: round(sum(metrics, 'cost_estimate'), 6) },
|
||||
tokens: {
|
||||
input: metrics.some((row) => hasNumber(row.input_tokens)) ? sum(metrics, 'input_tokens') : null,
|
||||
output: metrics.some((row) => hasNumber(row.output_tokens)) ? sum(metrics, 'output_tokens') : null,
|
||||
total: metrics.some((row) => hasNumber(row.total_tokens)) ? sum(metrics, 'total_tokens') : null,
|
||||
provider_total: provider.some((row) => hasNumber(row.total_tokens)) ? sum(provider, 'total_tokens') : null,
|
||||
},
|
||||
cost_usd: {
|
||||
provider_actual: provider.some((row) => hasNumber(row.cost_usd)) ? round(sum(provider, 'cost_usd'), 6) : null,
|
||||
estimated: metrics.some((row) => hasNumber(row.cost_estimate)) ? round(sum(metrics, 'cost_estimate'), 6) : null,
|
||||
},
|
||||
provider_calls: provider.length,
|
||||
alerts: alertCount,
|
||||
coverage: {
|
||||
runtime_records: metrics.length,
|
||||
token_records: tokenRecords,
|
||||
cost_records: costRecords,
|
||||
token_pct: tokenCoverage,
|
||||
cost_pct: costCoverage,
|
||||
quality: qualityCounts,
|
||||
},
|
||||
},
|
||||
thresholds: { failure_rate_pct: failureThreshold, p95_latency_ms: p95Threshold, freshness_age_s: STALE_AFTER_S },
|
||||
findings,
|
||||
evidence_sources: sourceEvidence,
|
||||
data_quality: { status: !primary?.present ? 'insufficient' : warnings.length ? 'partial' : 'complete', warnings },
|
||||
data_quality: {
|
||||
status: (
|
||||
!primary?.present
|
||||
|| (metrics.length > 0 && tokenRecords === 0 && costRecords === 0)
|
||||
) ? 'insufficient' : warnings.length ? 'partial' : 'complete',
|
||||
warnings,
|
||||
},
|
||||
available_filters: { projects: availableProjects, runs: availableRuns },
|
||||
details: {
|
||||
by_status: counted(metrics, (row) => stringValue(row.status, 'unknown')).map(({ key, count: countValue }) => ({ status: key, count: countValue })),
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { BadRequestException, Controller, Get, Header, Inject, Query, Res } from '@nestjs/common';
|
||||
import { BadRequestException, Controller, Get, Header, Inject, Param, Query, Res } from '@nestjs/common';
|
||||
import type { Response } from 'express';
|
||||
import { ok } from '../common/api-response.js';
|
||||
import { parseH6ReportQuery, type H6ReportQueryParams } from './h6-report.js';
|
||||
@@ -34,4 +34,24 @@ export class ReportsController {
|
||||
response.setHeader('Content-Disposition', `attachment; filename="casan-h6-report-${stamp}.${format}"`);
|
||||
response.send(body);
|
||||
}
|
||||
|
||||
@Get('run/:traceId')
|
||||
run(@Param('traceId') traceId: string) {
|
||||
return ok(this.reports.run(traceId));
|
||||
}
|
||||
|
||||
@Get('run/:traceId/export')
|
||||
@Header('Cache-Control', 'no-store')
|
||||
exportRun(
|
||||
@Param('traceId') traceId: string,
|
||||
@Query('format') rawFormat: string | undefined,
|
||||
@Res() response: Response,
|
||||
) {
|
||||
const format = rawFormat ?? 'json';
|
||||
if (format !== 'json' && format !== 'html') throw new BadRequestException('RUN_REPORT_INVALID_FORMAT');
|
||||
const report = this.reports.run(traceId);
|
||||
response.type(format === 'html' ? 'text/html; charset=utf-8' : 'application/json; charset=utf-8');
|
||||
response.setHeader('Content-Disposition', `attachment; filename="casan-run-${traceId}.${format}"`);
|
||||
response.send(this.reports.serializeRun(report, format));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { ReportsController } from './reports.controller.js';
|
||||
import { ReportsService } from './reports.service.js';
|
||||
import { TelemetryModule } from '../telemetry/telemetry.module.js';
|
||||
|
||||
@Module({
|
||||
imports: [TelemetryModule],
|
||||
controllers: [ReportsController],
|
||||
providers: [ReportsService],
|
||||
exports: [ReportsService],
|
||||
|
||||
@@ -1,12 +1,20 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { Inject, Injectable, NotFoundException } from '@nestjs/common';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { PATHS, telemetryFreshness } from '../common/app-root.js';
|
||||
import { readJsonl } from '../telemetry/telemetry.reader.js';
|
||||
import { TelemetryService } from '../telemetry/telemetry.service.js';
|
||||
import { buildH6Report, type H6Report, type H6ReportQuery } from './h6-report.js';
|
||||
import { renderH6ReportHtml } from './h6-report.html.js';
|
||||
import { HARNESS_REPORT_CATALOG } from './report.contract.js';
|
||||
import { buildRunAssuranceReport, type RunAssuranceReport } from './run-report.js';
|
||||
import { renderRunAssuranceHtml } from './run-report.html.js';
|
||||
import { APP_ROOT } from '../common/app-root.js';
|
||||
|
||||
@Injectable()
|
||||
export class ReportsService {
|
||||
constructor(@Inject(TelemetryService) private readonly telemetry: TelemetryService) {}
|
||||
|
||||
catalog() {
|
||||
return { schema_version: 1, reports: HARNESS_REPORT_CATALOG };
|
||||
}
|
||||
@@ -25,4 +33,35 @@ export class ReportsService {
|
||||
serializeH6(report: H6Report, format: 'json' | 'html'): string {
|
||||
return format === 'html' ? renderH6ReportHtml(report) : `${JSON.stringify(report, null, 2)}\n`;
|
||||
}
|
||||
|
||||
run(traceId: string): RunAssuranceReport {
|
||||
if (!/^[a-zA-Z0-9][a-zA-Z0-9._:-]{0,127}$/.test(traceId)) {
|
||||
throw new NotFoundException('CASAN_RUN_NOT_FOUND');
|
||||
}
|
||||
const graph = this.telemetry.traceGraph(traceId);
|
||||
const traceResult = this.telemetry.run(traceId);
|
||||
const metrics = readJsonl(PATHS.metrics);
|
||||
const metric = [...metrics].reverse().find((row) => row.trace_id === traceId) ?? null;
|
||||
let config: Record<string, unknown> = {};
|
||||
try {
|
||||
config = JSON.parse(readFileSync(join(APP_ROOT, '.casan', 'config.json'), 'utf8')) as Record<string, unknown>;
|
||||
} catch {
|
||||
config = {};
|
||||
}
|
||||
const report = buildRunAssuranceReport({
|
||||
traceId,
|
||||
graph,
|
||||
trace: traceResult.trace,
|
||||
metric,
|
||||
config,
|
||||
});
|
||||
if (report.verdict === 'not_found') throw new NotFoundException('CASAN_RUN_NOT_FOUND');
|
||||
return report;
|
||||
}
|
||||
|
||||
serializeRun(report: RunAssuranceReport, format: 'json' | 'html'): string {
|
||||
return format === 'html'
|
||||
? renderRunAssuranceHtml(report)
|
||||
: `${JSON.stringify(report, null, 2)}\n`;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
import type { RunAssuranceReport } from './run-report.js';
|
||||
|
||||
const escapeHtml = (value: unknown): string => String(value ?? '')
|
||||
.replaceAll('&', '&')
|
||||
.replaceAll('<', '<')
|
||||
.replaceAll('>', '>')
|
||||
.replaceAll('"', '"')
|
||||
.replaceAll("'", ''');
|
||||
|
||||
const badgeClass = (status: string): string => {
|
||||
if (status === 'pass' || status === 'certified') return 'pass';
|
||||
if (status === 'warning' || status === 'in_progress') return 'warn';
|
||||
if (status === 'queued' || status === 'skipped') return 'muted';
|
||||
return 'fail';
|
||||
};
|
||||
|
||||
export function renderRunAssuranceHtml(report: RunAssuranceReport): string {
|
||||
const gates = report.gates.map((gate) => `
|
||||
<article class="gate">
|
||||
<div class="gate-head"><strong>${escapeHtml(gate.title)}</strong><span class="badge ${badgeClass(gate.status)}">${escapeHtml(gate.status)}</span></div>
|
||||
<p>${escapeHtml(gate.reason)}</p>
|
||||
<small>${escapeHtml(gate.updated_at ?? 'No timestamp')}</small>
|
||||
<details><summary>Evidence fields</summary><pre>${escapeHtml(JSON.stringify(gate.evidence, null, 2))}</pre></details>
|
||||
</article>`).join('');
|
||||
const tokenValue = report.summary.token_usage_available ? 'Available' : 'Unavailable';
|
||||
const costValue = report.summary.cost_available ? 'Available' : 'Unavailable';
|
||||
return `<!doctype html>
|
||||
<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<title>CASAN assurance receipt · ${escapeHtml(report.trace_id)}</title>
|
||||
<style>
|
||||
:root{color-scheme:light;--ink:#0f172a;--muted:#64748b;--line:#e2e8f0;--panel:#fff;--bg:#f8fafc;--indigo:#4f46e5}
|
||||
*{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--ink);font:14px/1.55 Inter,ui-sans-serif,system-ui,-apple-system,sans-serif}
|
||||
main{max-width:1160px;margin:0 auto;padding:40px 24px 64px}.hero{position:relative;overflow:hidden;border-radius:28px;background:#111827;color:#fff;padding:32px;box-shadow:0 24px 60px rgba(15,23,42,.18)}
|
||||
.hero:after{content:"";position:absolute;right:-80px;top:-100px;width:280px;height:280px;border-radius:50%;background:rgba(99,102,241,.3);filter:blur(50px)}.eyebrow{color:#a5b4fc;font-size:11px;font-weight:800;letter-spacing:.18em;text-transform:uppercase}
|
||||
h1{position:relative;margin:8px 0 2px;font-size:30px;letter-spacing:-.035em}.trace{position:relative;color:#94a3b8;font:12px ui-monospace,SFMono-Regular,monospace;word-break:break-all}
|
||||
.summary{display:grid;grid-template-columns:repeat(4,minmax(0,1fr));gap:12px;margin:18px 0}.metric,.section{border:1px solid var(--line);background:var(--panel);border-radius:18px;padding:18px}.metric span{display:block;color:var(--muted);font-size:11px;font-weight:700;text-transform:uppercase;letter-spacing:.08em}.metric strong{display:block;margin-top:8px;font-size:22px}
|
||||
.section{margin-top:16px}.section h2{margin:0 0 14px;font-size:16px}.rail{display:grid;grid-template-columns:repeat(7,minmax(0,1fr));gap:10px}.gate{min-width:0;border:1px solid var(--line);border-radius:15px;padding:14px;background:#fff}.gate-head{display:flex;align-items:flex-start;justify-content:space-between;gap:8px}.gate p{min-height:44px;color:#475569;font-size:12px}.gate small{color:#94a3b8;font-size:10px}.gate details{margin-top:10px;color:#64748b;font-size:10px}.gate summary{cursor:pointer;font-weight:700}.gate pre{max-height:180px;overflow:auto;white-space:pre-wrap;word-break:break-word;border-radius:8px;background:#f8fafc;padding:8px;font-size:9px}.badge{border-radius:999px;padding:3px 7px;font-size:9px;font-weight:800;text-transform:uppercase}.pass{background:#dcfce7;color:#166534}.warn{background:#fef3c7;color:#92400e}.fail{background:#ffe4e6;color:#9f1239}.muted{background:#f1f5f9;color:#64748b}
|
||||
.note{color:var(--muted);font-size:12px}.footer{margin-top:18px;color:#94a3b8;font-size:11px}
|
||||
@media(max-width:900px){.summary{grid-template-columns:repeat(2,1fr)}.rail{grid-template-columns:1fr 1fr}}@media(max-width:520px){main{padding:18px 12px}.hero{padding:22px}.rail,.summary{grid-template-columns:1fr}}
|
||||
@media print{body{background:#fff}main{max-width:none;padding:0}.hero,.metric,.section{box-shadow:none;break-inside:avoid}}
|
||||
</style></head><body><main>
|
||||
<section class="hero"><div class="eyebrow">CASAN · evidence-backed assurance</div><h1>${escapeHtml(report.verdict.replace('_', ' ').toUpperCase())}</h1><div class="trace">${escapeHtml(report.trace_id)}</div></section>
|
||||
<section class="summary">
|
||||
<div class="metric"><span>Harness gates</span><strong>${report.summary.gates_observed}/7</strong></div>
|
||||
<div class="metric"><span>H6 quality</span><strong>${escapeHtml(report.summary.telemetry_quality)}</strong></div>
|
||||
<div class="metric"><span>Token usage</span><strong>${tokenValue}</strong></div>
|
||||
<div class="metric"><span>Cost</span><strong>${costValue}</strong></div>
|
||||
<div class="metric"><span>Duration</span><strong>${report.summary.duration_ms === null ? 'Unavailable' : `${report.summary.duration_ms} ms`}</strong></div>
|
||||
<div class="metric"><span>Tool calls</span><strong>${report.summary.tool_calls}</strong></div>
|
||||
<div class="metric"><span>Failures</span><strong>${report.summary.failures}</strong></div>
|
||||
<div class="metric"><span>Evidence source</span><strong>${report.source.trace_found && report.source.graph_found ? 'Verified' : 'Partial'}</strong></div>
|
||||
</section>
|
||||
<section class="section"><h2>Live assurance rail · H1 → H7</h2><div class="rail">${gates}</div></section>
|
||||
<section class="section"><h2>Certification</h2><p>Strength: <strong>${escapeHtml(report.certification.strength ?? 'unknown')}</strong></p><p class="note">${escapeHtml(report.certification.reasons.join(' · ') || 'No certification reason recorded.')}</p></section>
|
||||
<div class="footer">Generated ${escapeHtml(report.generated_at)} · Project ${escapeHtml(report.project.id)} · Edition ${escapeHtml(report.project.edition ?? 'unknown')} · Maturity ${escapeHtml(report.project.maturity?.status ?? 'not assessed')}. No maturity score or unavailable telemetry value is hard-coded.</div>
|
||||
</main></body></html>`;
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
export interface RunGateSnapshot {
|
||||
id: string;
|
||||
title: string;
|
||||
description: string;
|
||||
status: string;
|
||||
reason: string;
|
||||
updated_at: string | null;
|
||||
evidence: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface RunAssuranceReport {
|
||||
schema_version: 1;
|
||||
report_id: string;
|
||||
generated_at: string;
|
||||
trace_id: string;
|
||||
project: {
|
||||
id: string;
|
||||
edition: string | null;
|
||||
maturity: { level: number | null; status: string } | null;
|
||||
};
|
||||
verdict: 'certified' | 'non_certified' | 'in_progress' | 'not_found';
|
||||
certification: {
|
||||
strength: string | null;
|
||||
reasons: string[];
|
||||
finalized_at: string | null;
|
||||
};
|
||||
summary: {
|
||||
gates_observed: number;
|
||||
gates_total: 7;
|
||||
tool_calls: number;
|
||||
failures: number;
|
||||
duration_ms: number | null;
|
||||
telemetry_quality: string;
|
||||
token_usage_available: boolean;
|
||||
cost_available: boolean;
|
||||
};
|
||||
gates: RunGateSnapshot[];
|
||||
h6: Record<string, unknown> | null;
|
||||
source: {
|
||||
trace_found: boolean;
|
||||
graph_found: boolean;
|
||||
metric_found: boolean;
|
||||
};
|
||||
}
|
||||
|
||||
type Row = Record<string, unknown>;
|
||||
|
||||
const stringValue = (value: unknown, fallback = ''): string => (
|
||||
typeof value === 'string' && value.trim() ? value.trim() : fallback
|
||||
);
|
||||
const numberValue = (value: unknown, fallback = 0): number => (
|
||||
typeof value === 'number' && Number.isFinite(value) ? value : fallback
|
||||
);
|
||||
const hasNumber = (value: unknown): boolean => (
|
||||
typeof value === 'number' && Number.isFinite(value)
|
||||
);
|
||||
|
||||
export function buildRunAssuranceReport(input: {
|
||||
traceId: string;
|
||||
graph: {
|
||||
found: boolean;
|
||||
terminal: boolean;
|
||||
progress: number;
|
||||
nodes: RunGateSnapshot[];
|
||||
};
|
||||
trace: Row | null;
|
||||
metric: Row | null;
|
||||
config: Row;
|
||||
now?: Date;
|
||||
}): RunAssuranceReport {
|
||||
const now = input.now ?? new Date();
|
||||
const trace = input.trace ?? {};
|
||||
const metric = input.metric ?? {};
|
||||
const certified = trace.certified === true;
|
||||
const traceFound = input.trace !== null;
|
||||
const metricFound = input.metric !== null;
|
||||
const verdict = (
|
||||
!traceFound && !input.graph.found && !metricFound ? 'not_found'
|
||||
: !input.graph.terminal && !trace.finalized_at ? 'in_progress'
|
||||
: certified ? 'certified' : 'non_certified'
|
||||
);
|
||||
const maturity = input.config.maturity && typeof input.config.maturity === 'object'
|
||||
? input.config.maturity as Record<string, unknown>
|
||||
: null;
|
||||
const duration = (
|
||||
hasNumber(metric.duration_ms) ? Number(metric.duration_ms)
|
||||
: hasNumber(metric.latency_ms) ? Number(metric.latency_ms)
|
||||
: null
|
||||
);
|
||||
const costObject = metric.cost && typeof metric.cost === 'object'
|
||||
? metric.cost as Record<string, unknown>
|
||||
: {};
|
||||
return {
|
||||
schema_version: 1,
|
||||
report_id: `RUN-${input.traceId}`,
|
||||
generated_at: now.toISOString(),
|
||||
trace_id: input.traceId,
|
||||
project: {
|
||||
id: stringValue(
|
||||
input.config.project_id ?? trace.project_id ?? metric.project_id,
|
||||
'unknown',
|
||||
),
|
||||
edition: stringValue(
|
||||
input.config.edition ?? input.config.target_level_name,
|
||||
) || null,
|
||||
maturity: maturity ? {
|
||||
level: hasNumber(maturity.level) ? Number(maturity.level) : null,
|
||||
status: stringValue(maturity.status, 'not_assessed'),
|
||||
} : null,
|
||||
},
|
||||
verdict,
|
||||
certification: {
|
||||
strength: stringValue(
|
||||
trace.certification_strength ?? metric.certification_strength,
|
||||
) || null,
|
||||
reasons: Array.isArray(trace.certification_reasons)
|
||||
? trace.certification_reasons.filter(
|
||||
(value): value is string => typeof value === 'string')
|
||||
: [],
|
||||
finalized_at: stringValue(trace.finalized_at ?? metric.finished_at) || null,
|
||||
},
|
||||
summary: {
|
||||
gates_observed: input.graph.progress,
|
||||
gates_total: 7,
|
||||
tool_calls: numberValue(trace.tool_calls ?? metric.tool_calls),
|
||||
failures: numberValue(trace.failures ?? metric.failures),
|
||||
duration_ms: duration,
|
||||
telemetry_quality: stringValue(metric.telemetry_quality, 'unknown'),
|
||||
token_usage_available: (
|
||||
hasNumber(metric.total_tokens)
|
||||
|| hasNumber(metric.input_tokens)
|
||||
|| hasNumber(metric.output_tokens)
|
||||
),
|
||||
cost_available: (
|
||||
hasNumber(metric.cost_estimate)
|
||||
|| hasNumber(costObject.amount)
|
||||
),
|
||||
},
|
||||
gates: input.graph.nodes,
|
||||
h6: input.metric,
|
||||
source: {
|
||||
trace_found: traceFound,
|
||||
graph_found: input.graph.found,
|
||||
metric_found: metricFound,
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -16,6 +16,11 @@ export class TelemetryController {
|
||||
return ok(this.svc.overview());
|
||||
}
|
||||
|
||||
@Get('project')
|
||||
project() {
|
||||
return ok(this.svc.projectProfile());
|
||||
}
|
||||
|
||||
// Production reverse proxy policy must restrict this aggregate-only endpoint
|
||||
// to the monitoring network / service account.
|
||||
@Get('metrics')
|
||||
|
||||
@@ -5,5 +5,6 @@ import { TelemetryService } from './telemetry.service.js';
|
||||
@Module({
|
||||
controllers: [TelemetryController],
|
||||
providers: [TelemetryService],
|
||||
exports: [TelemetryService],
|
||||
})
|
||||
export class TelemetryModule {}
|
||||
|
||||
@@ -59,9 +59,9 @@ const pct = (part: number, total: number) => (total > 0 ? Math.round((part / tot
|
||||
|
||||
function gateStatus(value: unknown): HarnessGateStatus {
|
||||
const status = String(value ?? '').toLowerCase();
|
||||
if (['success', 'pass', 'passed', 'allow', 'allowed', 'answered'].includes(status)) return 'pass';
|
||||
if (['warn', 'warning'].includes(status)) return 'warning';
|
||||
if (['block', 'blocked', 'deny', 'denied'].includes(status)) return 'blocked';
|
||||
if (['success', 'pass', 'passed', 'allow', 'allowed', 'answered', 'opened', 'certified'].includes(status)) return 'pass';
|
||||
if (['warn', 'warning', 'degraded', 'partial', 'insufficient'].includes(status)) return 'warning';
|
||||
if (['block', 'blocked', 'deny', 'denied', 'flag', 'non_certified'].includes(status)) return 'blocked';
|
||||
if (['fail', 'failed', 'error'].includes(status)) return 'error';
|
||||
if (status === 'running') return 'running';
|
||||
if (status === 'skipped') return 'skipped';
|
||||
@@ -232,11 +232,23 @@ export class TelemetryService {
|
||||
|
||||
const runs = metrics.length;
|
||||
const latencies = metrics.map((m) => num(m.latency_ms)).filter((x) => x > 0);
|
||||
const latestMetric = metrics.length ? metrics[metrics.length - 1] : null;
|
||||
const latestTraceId = typeof latestMetric?.trace_id === 'string' ? latestMetric.trace_id : null;
|
||||
const latestTrace = latestTraceId ? readTrace(PATHS.traceDir, latestTraceId) : null;
|
||||
const latestGraph = latestTraceId ? this.traceGraph(latestTraceId) : null;
|
||||
return {
|
||||
...this.freshness(),
|
||||
totals: {
|
||||
runs,
|
||||
total_cost: sum(metrics, 'cost_estimate'),
|
||||
token_coverage_pct: pct(
|
||||
count(metrics, (m) => m.total_tokens != null || m.input_tokens != null || m.output_tokens != null),
|
||||
runs,
|
||||
),
|
||||
cost_coverage_pct: pct(
|
||||
count(metrics, (m) => m.cost_estimate != null || (m.cost && m.cost.amount != null)),
|
||||
runs,
|
||||
),
|
||||
avg_latency_ms: latencies.length ? Math.round(latencies.reduce((a, b) => a + b, 0) / latencies.length) : 0,
|
||||
failures: count(metrics, (m) => m.status === 'failed'),
|
||||
hallucination_signals: sum(metrics, 'hallucination_signals'),
|
||||
@@ -260,6 +272,39 @@ export class TelemetryService {
|
||||
head: readHead(PATHS.auditHead),
|
||||
last_decision: audit.length ? audit[audit.length - 1].decision ?? null : null,
|
||||
},
|
||||
latest_assurance: latestTraceId && latestMetric && latestGraph ? {
|
||||
trace_id: latestTraceId,
|
||||
certified: latestTrace?.certified === true,
|
||||
certification_strength: latestTrace?.certification_strength ?? latestMetric.certification_strength ?? null,
|
||||
telemetry_quality: latestMetric.telemetry_quality ?? 'unknown',
|
||||
duration_ms: latestMetric.duration_ms ?? latestMetric.latency_ms ?? null,
|
||||
tool_calls: latestMetric.tool_calls ?? 0,
|
||||
failures: latestMetric.failures ?? 0,
|
||||
graph: latestGraph,
|
||||
} : null,
|
||||
};
|
||||
}
|
||||
|
||||
projectProfile() {
|
||||
const config = readJson(join(APP_ROOT, '.casan', 'config.json')) as Row | null;
|
||||
const maturity = config?.maturity && typeof config.maturity === 'object'
|
||||
? config.maturity as Row
|
||||
: null;
|
||||
return {
|
||||
project_id: String(config?.project_id ?? APP_ROOT.split('/').pop() ?? 'unknown'),
|
||||
app_root: APP_ROOT,
|
||||
edition: String(config?.edition ?? config?.target_level_name ?? 'unknown'),
|
||||
edition_status: String(config?.edition_status ?? 'unknown'),
|
||||
maturity: {
|
||||
level: typeof maturity?.level === 'number' ? maturity.level : null,
|
||||
status: String(maturity?.status ?? 'not_assessed'),
|
||||
evidence: typeof maturity?.evidence === 'string' ? maturity.evidence : null,
|
||||
},
|
||||
enforcement_mode: String(config?.enforcement_mode ?? 'unknown'),
|
||||
integration_mode: String(config?.integration_mode ?? 'unknown'),
|
||||
clients: Array.isArray(config?.clients)
|
||||
? config.clients.filter((value): value is string => typeof value === 'string')
|
||||
: [],
|
||||
};
|
||||
}
|
||||
|
||||
@@ -351,13 +396,45 @@ export class TelemetryService {
|
||||
},
|
||||
});
|
||||
}
|
||||
const gateMap: Record<string, string> = {
|
||||
H1: 'H1-context',
|
||||
H2: 'H2-tool',
|
||||
H3: 'H3-eval',
|
||||
H4: 'H4-security',
|
||||
H5: 'H5-governance',
|
||||
H6: 'H6-agentops',
|
||||
H7: 'H7-orchestration',
|
||||
};
|
||||
for (const evidence of arr(legacyTrace.evidence)) {
|
||||
const harnesses = String(evidence.h ?? '').split('/');
|
||||
for (const harness of harnesses) {
|
||||
const mappedGate = gateMap[harness];
|
||||
if (!mappedGate) continue;
|
||||
events.push({
|
||||
timestamp: String(evidence.at ?? legacyTrace.finalized_at ?? ''),
|
||||
trace_id: safeId,
|
||||
gate_id: mappedGate,
|
||||
status: gateStatus(evidence.decision),
|
||||
reason: `${String(evidence.kind ?? 'legacy-evidence')}: ${String(evidence.detail ?? '')}`,
|
||||
evidence: {
|
||||
kind: evidence.kind ?? null,
|
||||
decision: evidence.decision ?? null,
|
||||
detail: evidence.detail ?? null,
|
||||
certification_strength: legacyTrace.certification_strength ?? null,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const metric of readJsonl<Row>(PATHS.metrics).filter((row) => row.trace_id === safeId)) {
|
||||
const telemetryQuality = String(metric.telemetry_quality ?? '');
|
||||
events.push({
|
||||
timestamp: String(metric.timestamp ?? ''),
|
||||
trace_id: safeId,
|
||||
gate_id: 'H6-agentops',
|
||||
status: gateStatus(metric.status),
|
||||
status: ['partial', 'insufficient'].includes(telemetryQuality)
|
||||
? 'warning'
|
||||
: gateStatus(metric.status),
|
||||
reason: String(metric.step ?? 'Legacy runtime metric'),
|
||||
evidence: {
|
||||
latency_ms: metric.latency_ms ?? null,
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { BadRequestException, ServiceUnavailableException } from '@nestjs/common';
|
||||
import { IngestService, expectedSignature, stableJson } from '../src/ingest/ingest.service.js';
|
||||
|
||||
test('ingest canonical JSON is stable across object key order', () => {
|
||||
assert.equal(stableJson({ z: 1, a: { y: 2, b: 3 } }), '{"a":{"b":3,"y":2},"z":1}');
|
||||
});
|
||||
|
||||
test('ingest is disabled unless an HMAC secret is configured', () => {
|
||||
const previous = process.env.CASAN_CP_INGEST_TOKEN;
|
||||
delete process.env.CASAN_CP_INGEST_TOKEN;
|
||||
try {
|
||||
assert.throws(() => new IngestService().ingest({}, '0', 'none'), ServiceUnavailableException);
|
||||
} finally {
|
||||
if (previous === undefined) delete process.env.CASAN_CP_INGEST_TOKEN;
|
||||
else process.env.CASAN_CP_INGEST_TOKEN = previous;
|
||||
}
|
||||
});
|
||||
|
||||
test('signed ingest rejects raw prompt/content fields before persistence', () => {
|
||||
const previous = process.env.CASAN_CP_INGEST_TOKEN;
|
||||
process.env.CASAN_CP_INGEST_TOKEN = 'test-only-secret';
|
||||
const timestamp = String(Math.floor(Date.now() / 1000));
|
||||
const body = {
|
||||
schema_version: 1,
|
||||
sent_at: new Date().toISOString(),
|
||||
project_id: 'safe-project',
|
||||
trace_id: 'safe-trace',
|
||||
receipt: {},
|
||||
metric: { trace_id: 'safe-trace', prompt: 'must-not-cross-boundary' },
|
||||
trace: { trace_id: 'safe-trace' },
|
||||
events: [],
|
||||
};
|
||||
try {
|
||||
const signature = expectedSignature('test-only-secret', timestamp, body);
|
||||
assert.throws(
|
||||
() => new IngestService().ingest(body, timestamp, signature),
|
||||
BadRequestException,
|
||||
);
|
||||
} finally {
|
||||
if (previous === undefined) delete process.env.CASAN_CP_INGEST_TOKEN;
|
||||
else process.env.CASAN_CP_INGEST_TOKEN = previous;
|
||||
}
|
||||
});
|
||||
@@ -8,6 +8,8 @@ import { sourceFreshness } from '../src/common/app-root.js';
|
||||
import { buildH6Report, parseH6ReportQuery, type H6ReportInput } from '../src/reports/h6-report.js';
|
||||
import { renderH6ReportHtml } from '../src/reports/h6-report.html.js';
|
||||
import { HARNESS_REPORT_CATALOG } from '../src/reports/report.contract.js';
|
||||
import { buildRunAssuranceReport } from '../src/reports/run-report.js';
|
||||
import { renderRunAssuranceHtml } from '../src/reports/run-report.html.js';
|
||||
|
||||
const NOW = new Date('2026-07-20T12:00:00.000Z');
|
||||
|
||||
@@ -68,6 +70,8 @@ test('H6 report filters project/time/run and aggregates measured evidence', () =
|
||||
assert.equal(report.summary.latency_ms.p50, 100);
|
||||
assert.equal(report.summary.latency_ms.p95, 9000);
|
||||
assert.equal(report.summary.tokens.provider_total, 28);
|
||||
assert.equal(report.summary.coverage.token_pct, 100);
|
||||
assert.equal(report.summary.coverage.cost_pct, 100);
|
||||
assert.equal(report.summary.alerts, 2);
|
||||
assert.deepEqual(report.details.by_alert, [
|
||||
{ alert: 'execution-failed', count: 1 },
|
||||
@@ -105,6 +109,30 @@ test('H6 report makes stale optional sources explicit in data quality', () => {
|
||||
assert.ok(report.data_quality.warnings.some((warning) => warning.includes('alerts telemetry source is missing')));
|
||||
});
|
||||
|
||||
test('H6 report never presents missing Codex usage as zero-cost coverage', () => {
|
||||
const input = fixture();
|
||||
input.metrics = [{
|
||||
timestamp: '2026-07-19T10:00:00Z',
|
||||
trace_id: 'codex-null',
|
||||
project_id: 'basic-design',
|
||||
status: 'success',
|
||||
latency_ms: 244090,
|
||||
total_tokens: null,
|
||||
cost_estimate: null,
|
||||
telemetry_quality: 'insufficient',
|
||||
}];
|
||||
input.provider = [];
|
||||
const report = buildH6Report(input, parseH6ReportQuery({ run: 'codex-null' }));
|
||||
assert.equal(report.summary.coverage.token_pct, 0);
|
||||
assert.equal(report.summary.coverage.cost_pct, 0);
|
||||
assert.equal(report.summary.tokens.total, null);
|
||||
assert.equal(report.summary.cost_usd.provider_actual, null);
|
||||
assert.equal(report.summary.cost_usd.estimated, null);
|
||||
assert.equal(report.data_quality.status, 'insufficient');
|
||||
assert.equal(report.verdict, 'attention');
|
||||
assert.ok(report.findings.some((finding) => finding.code === 'TELEMETRY_COVERAGE_GAP'));
|
||||
});
|
||||
|
||||
test('HTML export is standalone, escaped and contains no hard-coded maturity score', () => {
|
||||
const input = fixture();
|
||||
input.metrics[0].step = '<script>alert(1)</script>';
|
||||
@@ -117,3 +145,49 @@ test('HTML export is standalone, escaped and contains no hard-coded maturity sco
|
||||
assert.doesNotMatch(html, /Average\s+\d|\/100|218 core tests/i);
|
||||
assert.match(html, /No maturity score or telemetry value is hard-coded/);
|
||||
});
|
||||
|
||||
test('per-run assurance export carries H1-H7 and truthful H6 availability', () => {
|
||||
const nodes = ['Context', 'Tool', 'Eval', 'Security', 'Governance', 'AgentOps', 'Orchestration']
|
||||
.map((title, index) => ({
|
||||
id: `H${index + 1}`,
|
||||
title: `H${index + 1} · ${title}`,
|
||||
description: title,
|
||||
status: index === 5 ? 'warning' : 'pass',
|
||||
reason: index === 5 ? 'provider usage unavailable' : 'evidence verified',
|
||||
updated_at: NOW.toISOString(),
|
||||
evidence: {},
|
||||
}));
|
||||
const report = buildRunAssuranceReport({
|
||||
traceId: 'trace-safe',
|
||||
graph: { found: true, terminal: true, progress: 7, nodes },
|
||||
trace: {
|
||||
certified: true,
|
||||
certification_strength: 'project_hook',
|
||||
certification_reasons: ['evidence_complete'],
|
||||
finalized_at: NOW.toISOString(),
|
||||
tool_calls: 4,
|
||||
},
|
||||
metric: {
|
||||
trace_id: 'trace-safe',
|
||||
telemetry_quality: 'insufficient',
|
||||
duration_ms: 1200,
|
||||
total_tokens: null,
|
||||
cost_estimate: null,
|
||||
},
|
||||
config: {
|
||||
project_id: 'basic-design',
|
||||
edition: 'core',
|
||||
maturity: { level: null, status: 'not_assessed' },
|
||||
},
|
||||
now: NOW,
|
||||
});
|
||||
assert.equal(report.verdict, 'certified');
|
||||
assert.equal(report.summary.gates_observed, 7);
|
||||
assert.equal(report.summary.token_usage_available, false);
|
||||
assert.equal(report.summary.cost_available, false);
|
||||
const html = renderRunAssuranceHtml(report);
|
||||
assert.match(html, /Live assurance rail · H1 → H7/);
|
||||
assert.match(html, /Evidence fields/);
|
||||
assert.match(html, /Unavailable/);
|
||||
assert.doesNotMatch(html, /\$0(?:\.0+)?/);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user