import { test } from 'node:test'; import assert from 'node:assert/strict'; import { BadRequestException } from '@nestjs/common'; import { mkdtempSync, utimesSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { sourceFreshness } from '../src/common/app-root.js'; import { buildH6Report, parseH6ReportQuery, type H6ReportInput } from '../src/reports/h6-report.js'; import { renderH6ReportHtml } from '../src/reports/h6-report.html.js'; import { HARNESS_REPORT_CATALOG } from '../src/reports/report.contract.js'; import { buildRunAssuranceReport } from '../src/reports/run-report.js'; import { renderRunAssuranceHtml } from '../src/reports/run-report.html.js'; const NOW = new Date('2026-07-20T12:00:00.000Z'); function fixture(): H6ReportInput { return { now: NOW, metrics: [ { timestamp: '2026-07-19T08:00:00Z', trace_id: 'run-a', project: 'okr', status: 'success', step: 'generate', latency_ms: 100, input_tokens: 10, output_tokens: 20, total_tokens: 30, cost_estimate: 0.001, cost_source: 'word_count_estimate', retry_count: 0, alerts: [] }, { timestamp: '2026-07-19T09:00:00Z', trace_id: 'run-b', project: 'okr', status: 'failed', step: 'test', latency_ms: 9000, input_tokens: 20, output_tokens: 5, total_tokens: 25, cost_estimate: 0.002, cost_source: 'provider_telemetry', retry_count: 2, alerts: ['high-latency', 'execution-failed'] }, { timestamp: '2026-07-18T09:00:00Z', trace_id: 'run-c', project: 'desk', status: 'success', step: 'test', latency_ms: 250, total_tokens: 10, cost_estimate: 0.0001, cost_source: 'word_count_estimate' }, ], provider: [ { timestamp: '2026-07-19T08:00:00Z', run_id: 'run-a', project: 'okr', provider: 'ollama', model: 'ornith:9b', status: 'success', step: 'generate', latency_ms: 80, total_tokens: 28, cost_usd: 0 }, { timestamp: '2026-07-18T09:00:00Z', run_id: 'run-c', project: 'desk', provider: 'openai', model: 'gpt-test', status: 'success', step: 'test', latency_ms: 200, total_tokens: 10, cost_usd: 0.01 }, ], alerts: [ { timestamp: '2026-07-19T09:00:00Z', trace_id: 'run-b', project: 'okr', body: { 'alert.type': 'execution-failed' } }, ], sourceFreshness: [ { source: 'metrics', path: '/workspace/.specify/logs/cost/metrics.jsonl', present: true, updated_at: '2026-07-20T11:59:00Z', age_s: 60, stale: false, required: true }, { source: 'provider', path: '/workspace/.specify/logs/level5/provider-usage.jsonl', present: true, updated_at: '2026-07-20T11:58:00Z', age_s: 120, stale: false, required: false }, { source: 'alerts', path: '/workspace/.specify/agentops/alerts.log', present: true, updated_at: '2026-07-20T11:57:00Z', age_s: 180, stale: false, required: false }, ], }; } test('H1-H7 share one versioned report catalog contract', () => { assert.deepEqual(HARNESS_REPORT_CATALOG.map((entry) => entry.id), ['H1', 'H2', 'H3', 'H4', 'H5', 'H6', 'H7']); assert.ok(HARNESS_REPORT_CATALOG.every((entry) => entry.contract_version === 1)); assert.equal(HARNESS_REPORT_CATALOG.find((entry) => entry.id === 'H6')?.availability, 'implemented'); }); test('telemetry freshness is source-specific and missing required data fails loud', () => { const directory = mkdtempSync(join(tmpdir(), 'casan-h6-')); const metricsPath = join(directory, 'metrics.jsonl'); writeFileSync(metricsPath, '{}\n'); const oldTime = new Date(NOW.getTime() - 7200 * 1000); utimesSync(metricsPath, oldTime, oldTime); const staleMetrics = sourceFreshness('metrics', metricsPath, true, NOW.getTime()); const missingMetrics = sourceFreshness('metrics', join(directory, 'missing.jsonl'), true, NOW.getTime()); assert.equal(staleMetrics.present, true); assert.equal(staleMetrics.age_s, 7200); assert.equal(staleMetrics.stale, true); assert.deepEqual( { present: missingMetrics.present, stale: missingMetrics.stale, required: missingMetrics.required }, { present: false, stale: true, required: true }, ); }); test('H6 report filters project/time/run and aggregates measured evidence', () => { const query = parseH6ReportQuery({ project: 'okr', from: '2026-07-19', to: '2026-07-19' }); const report = buildH6Report(fixture(), query); assert.equal(report.schema_version, 1); assert.equal(report.harness, 'H6'); assert.equal(report.summary.runs, 2); assert.equal(report.summary.failed, 1); assert.equal(report.summary.retries, 2); assert.equal(report.summary.latency_ms.p50, 100); assert.equal(report.summary.latency_ms.p95, 9000); assert.equal(report.summary.tokens.provider_total, 28); assert.equal(report.summary.coverage.token_pct, 100); assert.equal(report.summary.coverage.cost_pct, 100); assert.equal(report.summary.alerts, 2); assert.deepEqual(report.details.by_alert, [ { alert: 'execution-failed', count: 1 }, { alert: 'high-latency', count: 1 }, ]); assert.equal(report.available_filters.projects.length, 2); assert.equal(report.freshness.status, 'live'); assert.equal(report.verdict, 'fail'); assert.ok(report.findings.some((finding) => finding.code === 'FAILURE_RATE_BREACH')); }); test('H6 report supports exact run filtering and truthful no-data verdict', () => { const selected = buildH6Report(fixture(), parseH6ReportQuery({ run: 'run-a' })); assert.equal(selected.summary.runs, 1); assert.equal(selected.summary.provider_calls, 1); assert.equal(selected.verdict, 'pass'); const empty = buildH6Report(fixture(), parseH6ReportQuery({ run: 'not-found' })); assert.equal(empty.verdict, 'no_data'); assert.equal(empty.summary.runs, 0); }); test('H6 query validation rejects traversal, inverted dates and invalid limits', () => { assert.throws(() => parseH6ReportQuery({ project: '../secret' }), BadRequestException); assert.throws(() => parseH6ReportQuery({ from: '2026-07-20', to: '2026-07-19' }), BadRequestException); assert.throws(() => parseH6ReportQuery({ limit: '5000' }), BadRequestException); }); test('H6 report makes stale optional sources explicit in data quality', () => { const input = fixture(); input.sourceFreshness[1] = { ...input.sourceFreshness[1], stale: true, age_s: 7200 }; input.sourceFreshness[2] = { ...input.sourceFreshness[2], present: false, updated_at: null, age_s: null, stale: false }; const report = buildH6Report(input, parseH6ReportQuery({ run: 'run-a' })); assert.equal(report.data_quality.status, 'partial'); assert.ok(report.data_quality.warnings.some((warning) => warning.includes('provider telemetry source is stale'))); assert.ok(report.data_quality.warnings.some((warning) => warning.includes('alerts telemetry source is missing'))); }); test('H6 report never presents missing Codex usage as zero-cost coverage', () => { const input = fixture(); input.metrics = [{ timestamp: '2026-07-19T10:00:00Z', trace_id: 'codex-null', project_id: 'basic-design', status: 'success', latency_ms: 244090, total_tokens: null, cost_estimate: null, telemetry_quality: 'insufficient', }]; input.provider = []; const report = buildH6Report(input, parseH6ReportQuery({ run: 'codex-null' })); assert.equal(report.summary.coverage.token_pct, 0); assert.equal(report.summary.coverage.cost_pct, 0); assert.equal(report.summary.tokens.total, null); assert.equal(report.summary.cost_usd.provider_actual, null); assert.equal(report.summary.cost_usd.estimated, null); assert.equal(report.data_quality.status, 'insufficient'); assert.equal(report.verdict, 'attention'); assert.ok(report.findings.some((finding) => finding.code === 'TELEMETRY_COVERAGE_GAP')); }); test('HTML export is standalone, escaped and contains no hard-coded maturity score', () => { const input = fixture(); input.metrics[0].step = ''; const report = buildH6Report(input, parseH6ReportQuery({ project: 'okr' })); const html = renderH6ReportHtml(report); assert.match(html, /^/); assert.match(html, /H6 · AgentOps Report/); assert.match(html, /<script>alert\(1\)<\/script>/); assert.doesNotMatch(html, /