feat: add production assurance dashboard flow
This commit is contained in:
@@ -0,0 +1,45 @@
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { BadRequestException, ServiceUnavailableException } from '@nestjs/common';
|
||||
import { IngestService, expectedSignature, stableJson } from '../src/ingest/ingest.service.js';
|
||||
|
||||
test('ingest canonical JSON is stable across object key order', () => {
|
||||
assert.equal(stableJson({ z: 1, a: { y: 2, b: 3 } }), '{"a":{"b":3,"y":2},"z":1}');
|
||||
});
|
||||
|
||||
test('ingest is disabled unless an HMAC secret is configured', () => {
|
||||
const previous = process.env.CASAN_CP_INGEST_TOKEN;
|
||||
delete process.env.CASAN_CP_INGEST_TOKEN;
|
||||
try {
|
||||
assert.throws(() => new IngestService().ingest({}, '0', 'none'), ServiceUnavailableException);
|
||||
} finally {
|
||||
if (previous === undefined) delete process.env.CASAN_CP_INGEST_TOKEN;
|
||||
else process.env.CASAN_CP_INGEST_TOKEN = previous;
|
||||
}
|
||||
});
|
||||
|
||||
test('signed ingest rejects raw prompt/content fields before persistence', () => {
|
||||
const previous = process.env.CASAN_CP_INGEST_TOKEN;
|
||||
process.env.CASAN_CP_INGEST_TOKEN = 'test-only-secret';
|
||||
const timestamp = String(Math.floor(Date.now() / 1000));
|
||||
const body = {
|
||||
schema_version: 1,
|
||||
sent_at: new Date().toISOString(),
|
||||
project_id: 'safe-project',
|
||||
trace_id: 'safe-trace',
|
||||
receipt: {},
|
||||
metric: { trace_id: 'safe-trace', prompt: 'must-not-cross-boundary' },
|
||||
trace: { trace_id: 'safe-trace' },
|
||||
events: [],
|
||||
};
|
||||
try {
|
||||
const signature = expectedSignature('test-only-secret', timestamp, body);
|
||||
assert.throws(
|
||||
() => new IngestService().ingest(body, timestamp, signature),
|
||||
BadRequestException,
|
||||
);
|
||||
} finally {
|
||||
if (previous === undefined) delete process.env.CASAN_CP_INGEST_TOKEN;
|
||||
else process.env.CASAN_CP_INGEST_TOKEN = previous;
|
||||
}
|
||||
});
|
||||
@@ -8,6 +8,8 @@ import { sourceFreshness } from '../src/common/app-root.js';
|
||||
import { buildH6Report, parseH6ReportQuery, type H6ReportInput } from '../src/reports/h6-report.js';
|
||||
import { renderH6ReportHtml } from '../src/reports/h6-report.html.js';
|
||||
import { HARNESS_REPORT_CATALOG } from '../src/reports/report.contract.js';
|
||||
import { buildRunAssuranceReport } from '../src/reports/run-report.js';
|
||||
import { renderRunAssuranceHtml } from '../src/reports/run-report.html.js';
|
||||
|
||||
const NOW = new Date('2026-07-20T12:00:00.000Z');
|
||||
|
||||
@@ -68,6 +70,8 @@ test('H6 report filters project/time/run and aggregates measured evidence', () =
|
||||
assert.equal(report.summary.latency_ms.p50, 100);
|
||||
assert.equal(report.summary.latency_ms.p95, 9000);
|
||||
assert.equal(report.summary.tokens.provider_total, 28);
|
||||
assert.equal(report.summary.coverage.token_pct, 100);
|
||||
assert.equal(report.summary.coverage.cost_pct, 100);
|
||||
assert.equal(report.summary.alerts, 2);
|
||||
assert.deepEqual(report.details.by_alert, [
|
||||
{ alert: 'execution-failed', count: 1 },
|
||||
@@ -105,6 +109,30 @@ test('H6 report makes stale optional sources explicit in data quality', () => {
|
||||
assert.ok(report.data_quality.warnings.some((warning) => warning.includes('alerts telemetry source is missing')));
|
||||
});
|
||||
|
||||
test('H6 report never presents missing Codex usage as zero-cost coverage', () => {
|
||||
const input = fixture();
|
||||
input.metrics = [{
|
||||
timestamp: '2026-07-19T10:00:00Z',
|
||||
trace_id: 'codex-null',
|
||||
project_id: 'basic-design',
|
||||
status: 'success',
|
||||
latency_ms: 244090,
|
||||
total_tokens: null,
|
||||
cost_estimate: null,
|
||||
telemetry_quality: 'insufficient',
|
||||
}];
|
||||
input.provider = [];
|
||||
const report = buildH6Report(input, parseH6ReportQuery({ run: 'codex-null' }));
|
||||
assert.equal(report.summary.coverage.token_pct, 0);
|
||||
assert.equal(report.summary.coverage.cost_pct, 0);
|
||||
assert.equal(report.summary.tokens.total, null);
|
||||
assert.equal(report.summary.cost_usd.provider_actual, null);
|
||||
assert.equal(report.summary.cost_usd.estimated, null);
|
||||
assert.equal(report.data_quality.status, 'insufficient');
|
||||
assert.equal(report.verdict, 'attention');
|
||||
assert.ok(report.findings.some((finding) => finding.code === 'TELEMETRY_COVERAGE_GAP'));
|
||||
});
|
||||
|
||||
test('HTML export is standalone, escaped and contains no hard-coded maturity score', () => {
|
||||
const input = fixture();
|
||||
input.metrics[0].step = '<script>alert(1)</script>';
|
||||
@@ -117,3 +145,49 @@ test('HTML export is standalone, escaped and contains no hard-coded maturity sco
|
||||
assert.doesNotMatch(html, /Average\s+\d|\/100|218 core tests/i);
|
||||
assert.match(html, /No maturity score or telemetry value is hard-coded/);
|
||||
});
|
||||
|
||||
test('per-run assurance export carries H1-H7 and truthful H6 availability', () => {
|
||||
const nodes = ['Context', 'Tool', 'Eval', 'Security', 'Governance', 'AgentOps', 'Orchestration']
|
||||
.map((title, index) => ({
|
||||
id: `H${index + 1}`,
|
||||
title: `H${index + 1} · ${title}`,
|
||||
description: title,
|
||||
status: index === 5 ? 'warning' : 'pass',
|
||||
reason: index === 5 ? 'provider usage unavailable' : 'evidence verified',
|
||||
updated_at: NOW.toISOString(),
|
||||
evidence: {},
|
||||
}));
|
||||
const report = buildRunAssuranceReport({
|
||||
traceId: 'trace-safe',
|
||||
graph: { found: true, terminal: true, progress: 7, nodes },
|
||||
trace: {
|
||||
certified: true,
|
||||
certification_strength: 'project_hook',
|
||||
certification_reasons: ['evidence_complete'],
|
||||
finalized_at: NOW.toISOString(),
|
||||
tool_calls: 4,
|
||||
},
|
||||
metric: {
|
||||
trace_id: 'trace-safe',
|
||||
telemetry_quality: 'insufficient',
|
||||
duration_ms: 1200,
|
||||
total_tokens: null,
|
||||
cost_estimate: null,
|
||||
},
|
||||
config: {
|
||||
project_id: 'basic-design',
|
||||
edition: 'core',
|
||||
maturity: { level: null, status: 'not_assessed' },
|
||||
},
|
||||
now: NOW,
|
||||
});
|
||||
assert.equal(report.verdict, 'certified');
|
||||
assert.equal(report.summary.gates_observed, 7);
|
||||
assert.equal(report.summary.token_usage_available, false);
|
||||
assert.equal(report.summary.cost_available, false);
|
||||
const html = renderRunAssuranceHtml(report);
|
||||
assert.match(html, /Live assurance rail · H1 → H7/);
|
||||
assert.match(html, /Evidence fields/);
|
||||
assert.match(html, /Unavailable/);
|
||||
assert.doesNotMatch(html, /\$0(?:\.0+)?/);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user