194 lines
9.7 KiB
TypeScript
194 lines
9.7 KiB
TypeScript
import { test } from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import { BadRequestException } from '@nestjs/common';
|
|
import { mkdtempSync, utimesSync, writeFileSync } from 'node:fs';
|
|
import { tmpdir } from 'node:os';
|
|
import { join } from 'node:path';
|
|
import { sourceFreshness } from '../src/common/app-root.js';
|
|
import { buildH6Report, parseH6ReportQuery, type H6ReportInput } from '../src/reports/h6-report.js';
|
|
import { renderH6ReportHtml } from '../src/reports/h6-report.html.js';
|
|
import { HARNESS_REPORT_CATALOG } from '../src/reports/report.contract.js';
|
|
import { buildRunAssuranceReport } from '../src/reports/run-report.js';
|
|
import { renderRunAssuranceHtml } from '../src/reports/run-report.html.js';
|
|
|
|
const NOW = new Date('2026-07-20T12:00:00.000Z');
|
|
|
|
function fixture(): H6ReportInput {
|
|
return {
|
|
now: NOW,
|
|
metrics: [
|
|
{ timestamp: '2026-07-19T08:00:00Z', trace_id: 'run-a', project: 'okr', status: 'success', step: 'generate', latency_ms: 100, input_tokens: 10, output_tokens: 20, total_tokens: 30, cost_estimate: 0.001, cost_source: 'word_count_estimate', retry_count: 0, alerts: [] },
|
|
{ timestamp: '2026-07-19T09:00:00Z', trace_id: 'run-b', project: 'okr', status: 'failed', step: 'test', latency_ms: 9000, input_tokens: 20, output_tokens: 5, total_tokens: 25, cost_estimate: 0.002, cost_source: 'provider_telemetry', retry_count: 2, alerts: ['high-latency', 'execution-failed'] },
|
|
{ timestamp: '2026-07-18T09:00:00Z', trace_id: 'run-c', project: 'desk', status: 'success', step: 'test', latency_ms: 250, total_tokens: 10, cost_estimate: 0.0001, cost_source: 'word_count_estimate' },
|
|
],
|
|
provider: [
|
|
{ timestamp: '2026-07-19T08:00:00Z', run_id: 'run-a', project: 'okr', provider: 'ollama', model: 'ornith:9b', status: 'success', step: 'generate', latency_ms: 80, total_tokens: 28, cost_usd: 0 },
|
|
{ timestamp: '2026-07-18T09:00:00Z', run_id: 'run-c', project: 'desk', provider: 'openai', model: 'gpt-test', status: 'success', step: 'test', latency_ms: 200, total_tokens: 10, cost_usd: 0.01 },
|
|
],
|
|
alerts: [
|
|
{ timestamp: '2026-07-19T09:00:00Z', trace_id: 'run-b', project: 'okr', body: { 'alert.type': 'execution-failed' } },
|
|
],
|
|
sourceFreshness: [
|
|
{ source: 'metrics', path: '/workspace/.specify/logs/cost/metrics.jsonl', present: true, updated_at: '2026-07-20T11:59:00Z', age_s: 60, stale: false, required: true },
|
|
{ source: 'provider', path: '/workspace/.specify/logs/level5/provider-usage.jsonl', present: true, updated_at: '2026-07-20T11:58:00Z', age_s: 120, stale: false, required: false },
|
|
{ source: 'alerts', path: '/workspace/.specify/agentops/alerts.log', present: true, updated_at: '2026-07-20T11:57:00Z', age_s: 180, stale: false, required: false },
|
|
],
|
|
};
|
|
}
|
|
|
|
test('H1-H7 share one versioned report catalog contract', () => {
|
|
assert.deepEqual(HARNESS_REPORT_CATALOG.map((entry) => entry.id), ['H1', 'H2', 'H3', 'H4', 'H5', 'H6', 'H7']);
|
|
assert.ok(HARNESS_REPORT_CATALOG.every((entry) => entry.contract_version === 1));
|
|
assert.equal(HARNESS_REPORT_CATALOG.find((entry) => entry.id === 'H6')?.availability, 'implemented');
|
|
});
|
|
|
|
test('telemetry freshness is source-specific and missing required data fails loud', () => {
|
|
const directory = mkdtempSync(join(tmpdir(), 'casan-h6-'));
|
|
const metricsPath = join(directory, 'metrics.jsonl');
|
|
writeFileSync(metricsPath, '{}\n');
|
|
const oldTime = new Date(NOW.getTime() - 7200 * 1000);
|
|
utimesSync(metricsPath, oldTime, oldTime);
|
|
const staleMetrics = sourceFreshness('metrics', metricsPath, true, NOW.getTime());
|
|
const missingMetrics = sourceFreshness('metrics', join(directory, 'missing.jsonl'), true, NOW.getTime());
|
|
assert.equal(staleMetrics.present, true);
|
|
assert.equal(staleMetrics.age_s, 7200);
|
|
assert.equal(staleMetrics.stale, true);
|
|
assert.deepEqual(
|
|
{ present: missingMetrics.present, stale: missingMetrics.stale, required: missingMetrics.required },
|
|
{ present: false, stale: true, required: true },
|
|
);
|
|
});
|
|
|
|
test('H6 report filters project/time/run and aggregates measured evidence', () => {
|
|
const query = parseH6ReportQuery({ project: 'okr', from: '2026-07-19', to: '2026-07-19' });
|
|
const report = buildH6Report(fixture(), query);
|
|
assert.equal(report.schema_version, 1);
|
|
assert.equal(report.harness, 'H6');
|
|
assert.equal(report.summary.runs, 2);
|
|
assert.equal(report.summary.failed, 1);
|
|
assert.equal(report.summary.retries, 2);
|
|
assert.equal(report.summary.latency_ms.p50, 100);
|
|
assert.equal(report.summary.latency_ms.p95, 9000);
|
|
assert.equal(report.summary.tokens.provider_total, 28);
|
|
assert.equal(report.summary.coverage.token_pct, 100);
|
|
assert.equal(report.summary.coverage.cost_pct, 100);
|
|
assert.equal(report.summary.alerts, 2);
|
|
assert.deepEqual(report.details.by_alert, [
|
|
{ alert: 'execution-failed', count: 1 },
|
|
{ alert: 'high-latency', count: 1 },
|
|
]);
|
|
assert.equal(report.available_filters.projects.length, 2);
|
|
assert.equal(report.freshness.status, 'live');
|
|
assert.equal(report.verdict, 'fail');
|
|
assert.ok(report.findings.some((finding) => finding.code === 'FAILURE_RATE_BREACH'));
|
|
});
|
|
|
|
test('H6 report supports exact run filtering and truthful no-data verdict', () => {
|
|
const selected = buildH6Report(fixture(), parseH6ReportQuery({ run: 'run-a' }));
|
|
assert.equal(selected.summary.runs, 1);
|
|
assert.equal(selected.summary.provider_calls, 1);
|
|
assert.equal(selected.verdict, 'pass');
|
|
const empty = buildH6Report(fixture(), parseH6ReportQuery({ run: 'not-found' }));
|
|
assert.equal(empty.verdict, 'no_data');
|
|
assert.equal(empty.summary.runs, 0);
|
|
});
|
|
|
|
test('H6 query validation rejects traversal, inverted dates and invalid limits', () => {
|
|
assert.throws(() => parseH6ReportQuery({ project: '../secret' }), BadRequestException);
|
|
assert.throws(() => parseH6ReportQuery({ from: '2026-07-20', to: '2026-07-19' }), BadRequestException);
|
|
assert.throws(() => parseH6ReportQuery({ limit: '5000' }), BadRequestException);
|
|
});
|
|
|
|
test('H6 report makes stale optional sources explicit in data quality', () => {
|
|
const input = fixture();
|
|
input.sourceFreshness[1] = { ...input.sourceFreshness[1], stale: true, age_s: 7200 };
|
|
input.sourceFreshness[2] = { ...input.sourceFreshness[2], present: false, updated_at: null, age_s: null, stale: false };
|
|
const report = buildH6Report(input, parseH6ReportQuery({ run: 'run-a' }));
|
|
assert.equal(report.data_quality.status, 'partial');
|
|
assert.ok(report.data_quality.warnings.some((warning) => warning.includes('provider telemetry source is stale')));
|
|
assert.ok(report.data_quality.warnings.some((warning) => warning.includes('alerts telemetry source is missing')));
|
|
});
|
|
|
|
test('H6 report never presents missing Codex usage as zero-cost coverage', () => {
|
|
const input = fixture();
|
|
input.metrics = [{
|
|
timestamp: '2026-07-19T10:00:00Z',
|
|
trace_id: 'codex-null',
|
|
project_id: 'basic-design',
|
|
status: 'success',
|
|
latency_ms: 244090,
|
|
total_tokens: null,
|
|
cost_estimate: null,
|
|
telemetry_quality: 'insufficient',
|
|
}];
|
|
input.provider = [];
|
|
const report = buildH6Report(input, parseH6ReportQuery({ run: 'codex-null' }));
|
|
assert.equal(report.summary.coverage.token_pct, 0);
|
|
assert.equal(report.summary.coverage.cost_pct, 0);
|
|
assert.equal(report.summary.tokens.total, null);
|
|
assert.equal(report.summary.cost_usd.provider_actual, null);
|
|
assert.equal(report.summary.cost_usd.estimated, null);
|
|
assert.equal(report.data_quality.status, 'insufficient');
|
|
assert.equal(report.verdict, 'attention');
|
|
assert.ok(report.findings.some((finding) => finding.code === 'TELEMETRY_COVERAGE_GAP'));
|
|
});
|
|
|
|
test('HTML export is standalone, escaped and contains no hard-coded maturity score', () => {
|
|
const input = fixture();
|
|
input.metrics[0].step = '<script>alert(1)</script>';
|
|
const report = buildH6Report(input, parseH6ReportQuery({ project: 'okr' }));
|
|
const html = renderH6ReportHtml(report);
|
|
assert.match(html, /^<!doctype html>/);
|
|
assert.match(html, /H6 · AgentOps Report/);
|
|
assert.match(html, /<script>alert\(1\)<\/script>/);
|
|
assert.doesNotMatch(html, /<script>alert\(1\)<\/script>/);
|
|
assert.doesNotMatch(html, /Average\s+\d|\/100|218 core tests/i);
|
|
assert.match(html, /No maturity score or telemetry value is hard-coded/);
|
|
});
|
|
|
|
test('per-run assurance export carries H1-H7 and truthful H6 availability', () => {
|
|
const nodes = ['Context', 'Tool', 'Eval', 'Security', 'Governance', 'AgentOps', 'Orchestration']
|
|
.map((title, index) => ({
|
|
id: `H${index + 1}`,
|
|
title: `H${index + 1} · ${title}`,
|
|
description: title,
|
|
status: index === 5 ? 'warning' : 'pass',
|
|
reason: index === 5 ? 'provider usage unavailable' : 'evidence verified',
|
|
updated_at: NOW.toISOString(),
|
|
evidence: {},
|
|
}));
|
|
const report = buildRunAssuranceReport({
|
|
traceId: 'trace-safe',
|
|
graph: { found: true, terminal: true, progress: 7, nodes },
|
|
trace: {
|
|
certified: true,
|
|
certification_strength: 'project_hook',
|
|
certification_reasons: ['evidence_complete'],
|
|
finalized_at: NOW.toISOString(),
|
|
tool_calls: 4,
|
|
},
|
|
metric: {
|
|
trace_id: 'trace-safe',
|
|
telemetry_quality: 'insufficient',
|
|
duration_ms: 1200,
|
|
total_tokens: null,
|
|
cost_estimate: null,
|
|
},
|
|
config: {
|
|
project_id: 'basic-design',
|
|
edition: 'core',
|
|
maturity: { level: null, status: 'not_assessed' },
|
|
},
|
|
now: NOW,
|
|
});
|
|
assert.equal(report.verdict, 'certified');
|
|
assert.equal(report.summary.gates_observed, 7);
|
|
assert.equal(report.summary.token_usage_available, false);
|
|
assert.equal(report.summary.cost_available, false);
|
|
const html = renderRunAssuranceHtml(report);
|
|
assert.match(html, /Live assurance rail · H1 → H7/);
|
|
assert.match(html, /Evidence fields/);
|
|
assert.match(html, /Unavailable/);
|
|
assert.doesNotMatch(html, /\$0(?:\.0+)?/);
|
|
});
|