feat: add production assurance dashboard flow

This commit is contained in:
thanhnv
2026-07-28 21:49:15 +07:00
parent ee5d1f7af5
commit cce3cbfd42
39 changed files with 1873 additions and 91 deletions
@@ -0,0 +1,45 @@
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { BadRequestException, ServiceUnavailableException } from '@nestjs/common';
import { IngestService, expectedSignature, stableJson } from '../src/ingest/ingest.service.js';
test('ingest canonical JSON is stable across object key order', () => {
assert.equal(stableJson({ z: 1, a: { y: 2, b: 3 } }), '{"a":{"b":3,"y":2},"z":1}');
});
test('ingest is disabled unless an HMAC secret is configured', () => {
const previous = process.env.CASAN_CP_INGEST_TOKEN;
delete process.env.CASAN_CP_INGEST_TOKEN;
try {
assert.throws(() => new IngestService().ingest({}, '0', 'none'), ServiceUnavailableException);
} finally {
if (previous === undefined) delete process.env.CASAN_CP_INGEST_TOKEN;
else process.env.CASAN_CP_INGEST_TOKEN = previous;
}
});
test('signed ingest rejects raw prompt/content fields before persistence', () => {
const previous = process.env.CASAN_CP_INGEST_TOKEN;
process.env.CASAN_CP_INGEST_TOKEN = 'test-only-secret';
const timestamp = String(Math.floor(Date.now() / 1000));
const body = {
schema_version: 1,
sent_at: new Date().toISOString(),
project_id: 'safe-project',
trace_id: 'safe-trace',
receipt: {},
metric: { trace_id: 'safe-trace', prompt: 'must-not-cross-boundary' },
trace: { trace_id: 'safe-trace' },
events: [],
};
try {
const signature = expectedSignature('test-only-secret', timestamp, body);
assert.throws(
() => new IngestService().ingest(body, timestamp, signature),
BadRequestException,
);
} finally {
if (previous === undefined) delete process.env.CASAN_CP_INGEST_TOKEN;
else process.env.CASAN_CP_INGEST_TOKEN = previous;
}
});
@@ -8,6 +8,8 @@ import { sourceFreshness } from '../src/common/app-root.js';
import { buildH6Report, parseH6ReportQuery, type H6ReportInput } from '../src/reports/h6-report.js';
import { renderH6ReportHtml } from '../src/reports/h6-report.html.js';
import { HARNESS_REPORT_CATALOG } from '../src/reports/report.contract.js';
import { buildRunAssuranceReport } from '../src/reports/run-report.js';
import { renderRunAssuranceHtml } from '../src/reports/run-report.html.js';
const NOW = new Date('2026-07-20T12:00:00.000Z');
@@ -68,6 +70,8 @@ test('H6 report filters project/time/run and aggregates measured evidence', () =
assert.equal(report.summary.latency_ms.p50, 100);
assert.equal(report.summary.latency_ms.p95, 9000);
assert.equal(report.summary.tokens.provider_total, 28);
assert.equal(report.summary.coverage.token_pct, 100);
assert.equal(report.summary.coverage.cost_pct, 100);
assert.equal(report.summary.alerts, 2);
assert.deepEqual(report.details.by_alert, [
{ alert: 'execution-failed', count: 1 },
@@ -105,6 +109,30 @@ test('H6 report makes stale optional sources explicit in data quality', () => {
assert.ok(report.data_quality.warnings.some((warning) => warning.includes('alerts telemetry source is missing')));
});
test('H6 report never presents missing Codex usage as zero-cost coverage', () => {
const input = fixture();
input.metrics = [{
timestamp: '2026-07-19T10:00:00Z',
trace_id: 'codex-null',
project_id: 'basic-design',
status: 'success',
latency_ms: 244090,
total_tokens: null,
cost_estimate: null,
telemetry_quality: 'insufficient',
}];
input.provider = [];
const report = buildH6Report(input, parseH6ReportQuery({ run: 'codex-null' }));
assert.equal(report.summary.coverage.token_pct, 0);
assert.equal(report.summary.coverage.cost_pct, 0);
assert.equal(report.summary.tokens.total, null);
assert.equal(report.summary.cost_usd.provider_actual, null);
assert.equal(report.summary.cost_usd.estimated, null);
assert.equal(report.data_quality.status, 'insufficient');
assert.equal(report.verdict, 'attention');
assert.ok(report.findings.some((finding) => finding.code === 'TELEMETRY_COVERAGE_GAP'));
});
test('HTML export is standalone, escaped and contains no hard-coded maturity score', () => {
const input = fixture();
input.metrics[0].step = '<script>alert(1)</script>';
@@ -117,3 +145,49 @@ test('HTML export is standalone, escaped and contains no hard-coded maturity sco
assert.doesNotMatch(html, /Average\s+\d|\/100|218 core tests/i);
assert.match(html, /No maturity score or telemetry value is hard-coded/);
});
test('per-run assurance export carries H1-H7 and truthful H6 availability', () => {
const nodes = ['Context', 'Tool', 'Eval', 'Security', 'Governance', 'AgentOps', 'Orchestration']
.map((title, index) => ({
id: `H${index + 1}`,
title: `H${index + 1} · ${title}`,
description: title,
status: index === 5 ? 'warning' : 'pass',
reason: index === 5 ? 'provider usage unavailable' : 'evidence verified',
updated_at: NOW.toISOString(),
evidence: {},
}));
const report = buildRunAssuranceReport({
traceId: 'trace-safe',
graph: { found: true, terminal: true, progress: 7, nodes },
trace: {
certified: true,
certification_strength: 'project_hook',
certification_reasons: ['evidence_complete'],
finalized_at: NOW.toISOString(),
tool_calls: 4,
},
metric: {
trace_id: 'trace-safe',
telemetry_quality: 'insufficient',
duration_ms: 1200,
total_tokens: null,
cost_estimate: null,
},
config: {
project_id: 'basic-design',
edition: 'core',
maturity: { level: null, status: 'not_assessed' },
},
now: NOW,
});
assert.equal(report.verdict, 'certified');
assert.equal(report.summary.gates_observed, 7);
assert.equal(report.summary.token_usage_available, false);
assert.equal(report.summary.cost_available, false);
const html = renderRunAssuranceHtml(report);
assert.match(html, /Live assurance rail · H1 → H7/);
assert.match(html, /Evidence fields/);
assert.match(html, /Unavailable/);
assert.doesNotMatch(html, /\$0(?:\.0+)?/);
});