diff --git a/.specify/logs/audit/security.jsonl b/.specify/logs/audit/security.jsonl index 6b63149..c4f0671 100644 --- a/.specify/logs/audit/security.jsonl +++ b/.specify/logs/audit/security.jsonl @@ -537,3 +537,6 @@ {"timestamp":"2026-07-19T04:00:29Z","trace_id":"trace-1784433629-809","harness":"H4-security","mode":"output","status":"pass","action":"allow","risk_level":"low","matched_categories":[],"input_hash":"e39bb3117bcde6f297fe51ac7d86546bbfbb7b17339821001d681b95a766019b","output_hash":"665a7b0ed97ad39aeec1aab585b6a0e3779b261f7d0a6734f7f4cb07672a3354"} {"timestamp":"2026-07-19T04:01:15Z","trace_id":"trace-1784433675-137","harness":"H4-security","mode":"input","status":"pass","action":"alert","risk_level":"medium","matched_categories":["semantic-availability"],"input_hash":"152e2c124ba6f23a03165ccb19ef9f04d6eb1155858101e8f741bb89f2245234","output_hash":"152e2c124ba6f23a03165ccb19ef9f04d6eb1155858101e8f741bb89f2245234"} {"timestamp":"2026-07-19T04:01:16Z","trace_id":"trace-1784433676-759","harness":"H4-security","mode":"output","status":"pass","action":"allow","risk_level":"low","matched_categories":[],"input_hash":"e39bb3117bcde6f297fe51ac7d86546bbfbb7b17339821001d681b95a766019b","output_hash":"e39bb3117bcde6f297fe51ac7d86546bbfbb7b17339821001d681b95a766019b"} +{"timestamp":"2026-07-19T05:18:02Z","trace_id":"trace-1784438282-2705","harness":"H4-security","mode":"input","status":"pass","action":"alert","risk_level":"medium","matched_categories":["semantic-availability"],"input_hash":"3084720034103b1c979a3ae80f6f59e360833eaa6a8c88d45381820edfbe2886","output_hash":"3084720034103b1c979a3ae80f6f59e360833eaa6a8c88d45381820edfbe2886"} +{"timestamp":"2026-07-19T05:18:03Z","trace_id":"trace-1784438283-3332","harness":"H4-security","mode":"output","status":"pass","action":"allow","risk_level":"low","matched_categories":[],"input_hash":"92848c78682b10335aedc8fa6ebfb8e9b7b5f17d9709df94b7742bda7412631e","output_hash":"92848c78682b10335aedc8fa6ebfb8e9b7b5f17d9709df94b7742bda7412631e"} +{"timestamp":"2026-07-19T05:18:20Z","trace_id":"trace-1784438300-3900","harness":"H4-security","mode":"input","status":"pass","action":"allow","risk_level":"low","matched_categories":[],"input_hash":"cbd0ed3c59982438aadbf2372bf8e463b9e2a884a4e452472683bca515616767","output_hash":"14487285dc9cada36542a27770878b55e3929878a40ebb527b67feef56fd5b8e"} diff --git a/docs/guides/CASAN_LOCAL_FULL_STACK.md b/docs/guides/CASAN_LOCAL_FULL_STACK.md index 59500d0..676f336 100644 --- a/docs/guides/CASAN_LOCAL_FULL_STACK.md +++ b/docs/guides/CASAN_LOCAL_FULL_STACK.md @@ -197,6 +197,26 @@ docker volume rm casan-local-prod_minio-data This is destructive and cannot be undone. Do not use it if the MinIO bucket has evidence you need to retain. +### Python used by the local auth bridge + +`local-full.sh` validates the Python interpreter before starting the host-side +provider auth bridge. On macOS it prefers `/usr/bin/python3`, avoiding an old +Intel-only framework Python that may be first in `PATH` but terminated by macOS. +To select another working Python explicitly, set `CASAN_PYTHON_BIN`: + +```bash +CASAN_PYTHON_BIN=/opt/homebrew/bin/python3 \ + bash packages/casan-harness/scripts/bash/local-full.sh start +``` + +The launcher also validates Docker's configured credential helper. If macOS +terminates that helper, CASAN uses an isolated, temporary anonymous Docker +configuration for this local stack's public images while preserving the active +Docker Desktop socket and Compose plugin. The same fallback uses Docker's classic +builder if the bundled BuildKit metadata helper is also terminated. It does not +edit `~/.docker/config.json` or delete stored registry credentials. Reinstall or +update Docker Desktop later to repair the helper globally. + ## Current scope and next phase This local environment includes every currently implemented CASAN component. diff --git a/docs/plans/CASAN_PLAN_19_HARNESS_REPORTING.md b/docs/plans/CASAN_PLAN_19_HARNESS_REPORTING.md new file mode 100644 index 0000000..bd1cdbd --- /dev/null +++ b/docs/plans/CASAN_PLAN_19_HARNESS_REPORTING.md @@ -0,0 +1,90 @@ +# CASAN Plan-19 — Harness Reporting Contract + +## Purpose + +Turn harness evidence into reviewable, portable reports. H6 is the first implemented +report; H1–H7 share one contract so later reports do not invent incompatible envelopes, +filter semantics or verdict fields. + +## Source-of-truth boundary + +H6 is a read-only projection over three canonical files. The report API and the existing +Control Panel resolve these paths through the same `app-root.ts` contract; harness writers +resolve them through `casan-paths.sh`. + +| Source | Canonical environment variable | Default path | Required | +|---|---|---|---| +| Runtime metrics | `CASAN_TELEMETRY_METRICS_LOG` | `.specify/logs/cost/metrics.jsonl` | yes | +| Provider usage | `CASAN_TELEMETRY_PROVIDER_LOG` | `.specify/logs/level5/provider-usage.jsonl` | no | +| Alerts | `CASAN_TELEMETRY_ALERTS_LOG` | `.specify/agentops/alerts.log` | no | + +Legacy names (`CASAN_DASHBOARD_METRICS`, `CASAN_CP_PROVIDER_USAGE`, +`CASAN_PROVIDER_LOG`, `CASAN_DASHBOARD_ALERTS`, and chat-specific metrics aliases) are +compatibility fallbacks only. New deployments should set the canonical variables. + +Freshness is source-specific and based on filesystem modification time. A source becomes +stale when its age exceeds `CASAN_DASHBOARD_STALE_S` (default `3600` seconds). Missing +required metrics produce a `missing` freshness status and an insufficient-data verdict. + +## Common H1–H7 contract + +`packages/casan-control-panel/backend/src/reports/report.contract.ts` defines schema +version `1`. Every harness report has: + +- identity: `report_id`, `harness`, `title`, `generated_at`; +- scope: `project`, `from`, `to`, `run`; +- decision: `verdict`, `verdict_reasons`, and explicit `thresholds`; +- evidence: source-level freshness, record counts and findings; +- quality: `complete`, `partial` or `insufficient` with warnings; +- presentation-neutral `summary` and `details` data. + +The catalog endpoint marks H6 as `implemented`; H1–H5 and H7 are `contract_ready`. This +is deliberate: report presence must not be confused with measured evidence or a passing +score. + +## H6 API and UI + +| Capability | Route | +|---|---| +| Report catalog | `GET /api/v1/reports` | +| H6 JSON API | `GET /api/v1/reports/h6` | +| Independent export | `GET /api/v1/reports/h6/export?format=json|html` | +| Browser report | `/reports/h6` | + +Optional filters are `project`, `from`, `to`, and `run`; `limit` only controls recent-row +detail and does not change summary aggregation. Project/run identifiers are allowlisted to +safe identifier characters, dates must parse as ISO-compatible values, and inverted ranges +fail with `400`. + +The H6 summary includes runtime counts and outcomes, failure rate, retries, latency +distribution, local/provider tokens, estimated/provider-reported cost, provider calls and +alert signals. Verdicts are evidence-derived: + +- `no_data`: no runtime or provider record matches the scope; +- `fail`: required metrics are missing or failure rate exceeds its threshold; +- `attention`: stale telemetry, failures, degraded runs, high P95 latency or alerts exist; +- `pass`: data is present and no finding crosses a configured threshold. + +Defaults are `5%` maximum failure rate and `5000 ms` maximum P95 latency, configurable +through `CASAN_H6_FAILURE_RATE_THRESHOLD_PCT` and +`CASAN_H6_P95_LATENCY_THRESHOLD_MS`. + +## Export integrity + +HTML and JSON exports are generated from the exact report object returned by the H6 service. +The HTML artifact is standalone and print-ready; JSON keeps schema version `1` for automated +consumers. Neither export reads a second data source or contains a hard-coded maturity +score. Source paths outside the app root are represented as `[external]/` so an +export does not disclose host directory structure. + +## Verification + +```bash +npm run build -w @casan/control-panel-backend +npm run build -w @casan/control-panel-frontend +cd packages/casan-control-panel/backend +node --import tsx --test test/reports.test.ts +``` + +The report tests cover the common catalog, filtered aggregation, exact-run/no-data behavior, +query validation, HTML escaping and absence of hard-coded scoring. diff --git a/infra/local-prod/docker-compose.yml b/infra/local-prod/docker-compose.yml index cc63efa..cea1cd5 100644 --- a/infra/local-prod/docker-compose.yml +++ b/infra/local-prod/docker-compose.yml @@ -107,9 +107,10 @@ services: environment: CASAN_DASHBOARD_BIND: 0.0.0.0 CASAN_DASHBOARD_STALE_S: "315360000" - CASAN_DASHBOARD_METRICS: /workspace/.specify/logs/cost/metrics.jsonl + CASAN_TELEMETRY_METRICS_LOG: /workspace/.specify/logs/cost/metrics.jsonl CASAN_DASHBOARD_HTML: /workspace/docs/output/casan/central-agentops-dashboard.html - CASAN_DASHBOARD_ALERTS: /workspace/.specify/agentops/alerts.log + CASAN_TELEMETRY_PROVIDER_LOG: /workspace/.specify/logs/level5/provider-usage.jsonl + CASAN_TELEMETRY_ALERTS_LOG: /workspace/.specify/agentops/alerts.log volumes: - ../..:/workspace:ro expose: diff --git a/packages/casan-control-panel/README.md b/packages/casan-control-panel/README.md index d461d8b..38619e5 100644 --- a/packages/casan-control-panel/README.md +++ b/packages/casan-control-panel/README.md @@ -23,6 +23,17 @@ Open http://127.0.0.1:5174 — panels show REAL metrics from `.specify/logs/**`. `incidents` · `tools` · `traceability` · `drift` · `cost` · `GET /healthz` (200 fresh / 503 stale — fail-loud, mirrors `dashboard-server.py`). +Harness reports: + +- `GET /api/v1/reports` — versioned H1–H7 report catalog and availability. +- `GET /api/v1/reports/h6?project=&from=&to=&run=` — H6 AgentOps + report from the same runtime metrics, provider usage and alert files as the Control Panel. +- `GET /api/v1/reports/h6/export?format=html|json&...` — standalone, print-ready HTML + or machine-readable JSON export. Both formats serialize the same filtered report object; + no maturity score is hard-coded. +- `/reports/h6` — UI report view with project, time-range and run filters plus source-level + freshness and data-quality warnings. + Metrics export: `GET /api/v1/metrics` provides Prometheus text exposition for aggregate freshness, run/failure/cost/token and H4/H5/action/incident counters. It intentionally contains no tenant, actor, trace, prompt or Evidence Pack @@ -121,7 +132,11 @@ FinOps/SLO: Data sources + aggregation mirror `packages/casan-harness/tests/generate-agentops-dashboard.py`. App root + telemetry paths resolve via the same marker walk-up as `casan-paths.sh` -(`.specify` or `packages/casan-harness`) and honor `CASAN_DASHBOARD_*` env overrides. +(`.specify` or `packages/casan-harness`). The canonical source variables are +`CASAN_TELEMETRY_METRICS_LOG`, `CASAN_TELEMETRY_PROVIDER_LOG`, and +`CASAN_TELEMETRY_ALERTS_LOG`; legacy dashboard/control-panel names remain accepted as +compatibility aliases. Freshness is calculated independently from each file's mtime using +`CASAN_DASHBOARD_STALE_S` (default `3600`). ## Security posture (MVP) Binds `127.0.0.1` by default. Refuses a non-loopback bind under `CASAN_PROFILE=prod` / diff --git a/packages/casan-control-panel/backend/src/app.module.ts b/packages/casan-control-panel/backend/src/app.module.ts index 5ceb6df..98bd9b6 100644 --- a/packages/casan-control-panel/backend/src/app.module.ts +++ b/packages/casan-control-panel/backend/src/app.module.ts @@ -9,9 +9,10 @@ import { ProviderAuthModule } from './provider-auth/provider-auth.module.js'; import { GoalsModule } from './goals/goals.module.js'; import { EvidenceModule } from './evidence/evidence.module.js'; import { SessionController } from './session/session.controller.js'; +import { ReportsModule } from './reports/reports.module.js'; @Module({ - imports: [TelemetryModule, SettingsModule, KillSwitchModule, ApprovalsModule, ChatModule, ProviderAuthModule, GoalsModule, EvidenceModule], + imports: [TelemetryModule, SettingsModule, KillSwitchModule, ApprovalsModule, ChatModule, ProviderAuthModule, GoalsModule, EvidenceModule, ReportsModule], controllers: [HealthController, SessionController], }) export class AppModule {} diff --git a/packages/casan-control-panel/backend/src/common/app-root.ts b/packages/casan-control-panel/backend/src/common/app-root.ts index af8bb46..cfd2c1a 100644 --- a/packages/casan-control-panel/backend/src/common/app-root.ts +++ b/packages/casan-control-panel/backend/src/common/app-root.ts @@ -3,8 +3,8 @@ // Mirrors packages/casan-harness/scripts/bash/casan-paths.sh: walk UP from a start dir // for a marker (`.specify` state dir OR `packages/casan-harness`), so the API reads the // real harness telemetry whether launched from the repo, a workspace subdir, or a bundle. -// Every path is overridable by the same CASAN_DASHBOARD_* env vars the legacy -// dashboard-server.py honors, so console + dashboard read identical sources. +// Canonical CASAN_TELEMETRY_* variables are shared with harness writers; legacy +// dashboard/control-panel names remain accepted so upgrades do not split sources. import { existsSync, statSync } from 'node:fs'; import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -27,35 +27,74 @@ export const APP_ROOT = process.env.CASAN_APP_ROOT ? resolve(process.env.CASAN_APP_ROOT) : findAppRoot(HERE); -const env = (k: string, fallback: string) => (process.env[k] ? resolve(process.env[k]!) : join(APP_ROOT, fallback)); +const env = (keys: string[], fallback: string) => { + const configured = keys.map((key) => process.env[key]).find((value): value is string => Boolean(value)); + return configured ? resolve(configured) : join(APP_ROOT, fallback); +}; // Runtime telemetry (regenerated) + generated reports. Names match the harness layout. export const PATHS = { - metrics: env('CASAN_DASHBOARD_METRICS', '.specify/logs/cost/metrics.jsonl'), - providerUsage: env('CASAN_CP_PROVIDER_USAGE', '.specify/logs/level5/provider-usage.jsonl'), - audit: env('CASAN_CP_AUDIT', '.specify/logs/audit/audit.jsonl'), - auditHead: env('CASAN_CP_AUDIT_HEAD', '.specify/logs/audit/audit-head.txt'), - security: env('CASAN_CP_SECURITY', '.specify/logs/audit/security.jsonl'), - toolRegistry: env('CASAN_CP_TOOL_REGISTRY', '.specify/logs/level5/tool-registry.jsonl'), - actionGate: env('CASAN_CP_ACTION_GATE', '.specify/logs/level5/action-gate.jsonl'), - fallback: env('CASAN_CP_FALLBACK', '.specify/logs/level5/fallback.jsonl'), - incidents: env('CASAN_CP_INCIDENTS', '.specify/logs/level5/incidents.jsonl'), - alerts: env('CASAN_DASHBOARD_ALERTS', '.specify/agentops/alerts.log'), - traceDir: env('CASAN_CP_TRACE_DIR', '.specify/logs/trace'), - traceEventDir: env('CASAN_CP_TRACE_EVENT_DIR', '.specify/logs/trace-events'), - traceability: env('CASAN_CP_TRACEABILITY', 'docs/output/casan/traceability-matrix.json'), - drift: env('CASAN_CP_DRIFT', 'docs/output/casan/level5-evidence/09-drift-report.json'), - businessKpi: env('CASAN_CP_KPI', 'docs/output/casan/level5-evidence/14-business-kpi-report.json'), - benignFp: env('CASAN_CP_BENIGN_FP', 'docs/output/casan/benign-fp-report.json'), - scoringReport: env('CASAN_CP_SCORING_REPORT', 'docs/output/casan/phase3-real-run-scoring.md'), - approvalInbox: env('CASAN_CP_APPROVAL_INBOX', '.specify/level5/approval-inbox.json'), - chatAudit: env('CASAN_CP_CHAT_AUDIT', '.specify/logs/chat/chat-turns.jsonl'), - delegationPolicy: env('CASAN_CP_DELEGATION_POLICY', 'packages/casan-harness/config/delegation-policy.yaml'), - selfImprove: env('CASAN_CP_SELF_IMPROVE', 'packages/casan-harness/scripts/bash/self-improve.py'), - evidencePacks: env('CASAN_CP_EVIDENCE_PACKS', 'docs/output/casan/evidence-packs'), + metrics: env(['CASAN_TELEMETRY_METRICS_LOG', 'CASAN_DASHBOARD_METRICS'], '.specify/logs/cost/metrics.jsonl'), + providerUsage: env(['CASAN_TELEMETRY_PROVIDER_LOG', 'CASAN_CP_PROVIDER_USAGE'], '.specify/logs/level5/provider-usage.jsonl'), + alerts: env(['CASAN_TELEMETRY_ALERTS_LOG', 'CASAN_DASHBOARD_ALERTS'], '.specify/agentops/alerts.log'), + audit: env(['CASAN_CP_AUDIT'], '.specify/logs/audit/audit.jsonl'), + auditHead: env(['CASAN_CP_AUDIT_HEAD'], '.specify/logs/audit/audit-head.txt'), + security: env(['CASAN_CP_SECURITY'], '.specify/logs/audit/security.jsonl'), + toolRegistry: env(['CASAN_CP_TOOL_REGISTRY'], '.specify/logs/level5/tool-registry.jsonl'), + actionGate: env(['CASAN_CP_ACTION_GATE'], '.specify/logs/level5/action-gate.jsonl'), + fallback: env(['CASAN_CP_FALLBACK'], '.specify/logs/level5/fallback.jsonl'), + incidents: env(['CASAN_CP_INCIDENTS'], '.specify/logs/level5/incidents.jsonl'), + traceDir: env(['CASAN_CP_TRACE_DIR'], '.specify/logs/trace'), + traceEventDir: env(['CASAN_CP_TRACE_EVENT_DIR'], '.specify/logs/trace-events'), + traceability: env(['CASAN_CP_TRACEABILITY'], 'docs/output/casan/traceability-matrix.json'), + drift: env(['CASAN_CP_DRIFT'], 'docs/output/casan/level5-evidence/09-drift-report.json'), + businessKpi: env(['CASAN_CP_KPI'], 'docs/output/casan/level5-evidence/14-business-kpi-report.json'), + benignFp: env(['CASAN_CP_BENIGN_FP'], 'docs/output/casan/benign-fp-report.json'), + scoringReport: env(['CASAN_CP_SCORING_REPORT'], 'docs/output/casan/phase3-real-run-scoring.md'), + approvalInbox: env(['CASAN_CP_APPROVAL_INBOX'], '.specify/level5/approval-inbox.json'), + chatAudit: env(['CASAN_CP_CHAT_AUDIT'], '.specify/logs/chat/chat-turns.jsonl'), + delegationPolicy: env(['CASAN_CP_DELEGATION_POLICY'], 'packages/casan-harness/config/delegation-policy.yaml'), + selfImprove: env(['CASAN_CP_SELF_IMPROVE'], 'packages/casan-harness/scripts/bash/self-improve.py'), + evidencePacks: env(['CASAN_CP_EVIDENCE_PACKS'], 'docs/output/casan/evidence-packs'), }; -export const STALE_AFTER_S = Number(process.env.CASAN_DASHBOARD_STALE_S ?? 3600); +const configuredStaleAfter = Number(process.env.CASAN_DASHBOARD_STALE_S ?? 3600); +export const STALE_AFTER_S = Number.isFinite(configuredStaleAfter) && configuredStaleAfter >= 0 + ? configuredStaleAfter + : 3600; + +export interface SourceFreshness { + source: 'metrics' | 'provider' | 'alerts'; + path: string; + present: boolean; + updated_at: string | null; + age_s: number | null; + stale: boolean; + required: boolean; +} + +export function sourceFreshness( + source: SourceFreshness['source'], + path: string, + required: boolean, + nowMs = Date.now(), +): SourceFreshness { + try { + const mtime = statSync(path).mtime; + const age = Math.max(0, Math.floor((nowMs - mtime.getTime()) / 1000)); + return { source, path, present: true, updated_at: mtime.toISOString(), age_s: age, stale: age > STALE_AFTER_S, required }; + } catch { + return { source, path, present: false, updated_at: null, age_s: null, stale: required, required }; + } +} + +export function telemetryFreshness(nowMs = Date.now()): SourceFreshness[] { + return [ + sourceFreshness('metrics', PATHS.metrics, true, nowMs), + sourceFreshness('provider', PATHS.providerUsage, false, nowMs), + sourceFreshness('alerts', PATHS.alerts, false, nowMs), + ]; +} // Freshness of the primary metrics feed (mtime), used by /healthz + `stale` flags. export function metricsAgeSeconds(): number | null { diff --git a/packages/casan-control-panel/backend/src/reports/h6-report.html.ts b/packages/casan-control-panel/backend/src/reports/h6-report.html.ts new file mode 100644 index 0000000..bc845b7 --- /dev/null +++ b/packages/casan-control-panel/backend/src/reports/h6-report.html.ts @@ -0,0 +1,61 @@ +import type { H6Breakdown, H6Report } from './h6-report.js'; + +const escapeHtml = (value: unknown): string => String(value ?? '') + .replaceAll('&', '&') + .replaceAll('<', '<') + .replaceAll('>', '>') + .replaceAll('"', '"') + .replaceAll("'", '''); + +const number = (value: number): string => new Intl.NumberFormat('en-US').format(value); +const money = (value: number): string => `$${value.toFixed(6)}`; + +function breakdownRows(rows: H6Breakdown[]): string { + if (rows.length === 0) return 'No records in the selected scope.'; + return rows.map((row) => ` + ${escapeHtml(row.key)}${number(row.runs)}${number(row.failures)} + ${number(row.latency_avg_ms)} ms${number(row.tokens)}${money(row.cost_usd)} + `).join(''); +} + +export function renderH6ReportHtml(report: H6Report): string { + const verdictClass = report.verdict === 'pass' ? 'pass' : report.verdict === 'fail' ? 'fail' : report.verdict === 'attention' ? 'attention' : 'neutral'; + const scope = [ + report.scope.project ? `Project: ${report.scope.project}` : 'All projects', + report.scope.run ? `Run: ${report.scope.run}` : 'All runs', + report.scope.from ? `From: ${report.scope.from}` : null, + report.scope.to ? `To: ${report.scope.to}` : null, + ].filter((value): value is string => Boolean(value)); + const findings = report.findings.length + ? report.findings.map((finding) => `
  • ${escapeHtml(finding.code)}${escapeHtml(finding.message)}
    • `).join('') + : '
  • NO_FINDINGSNo threshold breach was detected in this scope.
    • '; + const sourceRows = report.evidence_sources.map((source) => ` + ${escapeHtml(source.source)}${source.present ? 'present' : 'missing'}${source.stale ? 'stale' : 'fresh'} + ${source.age_s === null ? '—' : `${number(source.age_s)} s`}${number(source.records)}${escapeHtml(source.path)} + `).join(''); + const warnings = report.data_quality.warnings.length + ? `
      ${report.data_quality.warnings.map((warning) => `
    • ${escapeHtml(warning)}
      • `).join('')}
    ` + : '

    No data-quality warning.

    '; + + return ` + +${escapeHtml(report.title)} · ${escapeHtml(report.report_id)} +
    +
    CASAN assurance dossier · contract v${report.schema_version}

    ${escapeHtml(report.title)}

    ${scope.map((item) => `${escapeHtml(item)}`).join('')}
    ${escapeHtml(report.verdict)}
    Generated ${escapeHtml(report.generated_at)} · ${escapeHtml(report.report_id)} · Freshness ${escapeHtml(report.freshness.status)}
    +
    Governed runs${number(report.summary.runs)}
    Failure rate${report.summary.failure_rate_pct}%
    P95 latency${number(report.summary.latency_ms.p95)} ms
    Provider tokens${number(report.summary.tokens.provider_total)}
    Actual provider cost${money(report.summary.cost_usd.provider_actual)}
    Estimated cost${money(report.summary.cost_usd.estimated)}
    Alerts${number(report.summary.alerts)}
    Retries${number(report.summary.retries)}
    +

    Verdict findings

      ${findings}
    +

    Evidence freshness

    ${sourceRows}
    SourcePresenceStateAgeRecordsPath
    +

    Step breakdown

    ${breakdownRows(report.details.by_step)}
    StepRunsFailuresAvg latencyTokensCost
    +

    Provider/model breakdown

    ${breakdownRows(report.details.by_provider)}
    Provider · modelCallsFailuresAvg latencyTokensCost
    +

    Data quality · ${escapeHtml(report.data_quality.status)}

    ${warnings}
    +
    Generated from CASAN runtime evidence. No maturity score or telemetry value is hard-coded in this document.
    +
    `; +} diff --git a/packages/casan-control-panel/backend/src/reports/h6-report.ts b/packages/casan-control-panel/backend/src/reports/h6-report.ts new file mode 100644 index 0000000..ffb398e --- /dev/null +++ b/packages/casan-control-panel/backend/src/reports/h6-report.ts @@ -0,0 +1,307 @@ +import { BadRequestException } from '@nestjs/common'; +import { basename, relative } from 'node:path'; +import { APP_ROOT, PATHS, STALE_AFTER_S, telemetryFreshness, type SourceFreshness } from '../common/app-root.js'; +import type { HarnessReport, HarnessReportEvidenceSource, HarnessReportFinding, HarnessReportScope } from './report.contract.js'; + +type TelemetryRow = Record; + +export interface H6ReportQuery { + project: string | null; + from: string | null; + to: string | null; + run: string | null; + limit: number; +} + +export interface H6ReportQueryParams { + project?: string; + from?: string; + to?: string; + run?: string; + limit?: string; +} + +export interface H6ReportInput { + metrics: TelemetryRow[]; + provider: TelemetryRow[]; + alerts: TelemetryRow[]; + sourceFreshness: SourceFreshness[]; + now: Date; +} + +export interface H6ReportSummary { + runs: number; + success: number; + failed: number; + degraded: number; + failure_rate_pct: number; + retries: number; + latency_ms: { average: number; p50: number; p95: number; p99: number; max: number }; + tokens: { input: number; output: number; total: number; provider_total: number }; + cost_usd: { provider_actual: number; estimated: number }; + provider_calls: number; + alerts: number; +} + +export interface H6Breakdown { + key: string; + runs: number; + failures: number; + latency_avg_ms: number; + tokens: number; + cost_usd: number; +} + +export interface H6ReportDetails { + by_status: Array<{ status: string; count: number }>; + by_step: H6Breakdown[]; + by_provider: H6Breakdown[]; + by_cost_source: Array<{ source: string; records: number; cost_usd: number }>; + by_alert: Array<{ alert: string; count: number }>; + recent_runs: TelemetryRow[]; + recent_alerts: TelemetryRow[]; +} + +export type H6Report = HarnessReport; + +const safeFilter = /^[a-zA-Z0-9][a-zA-Z0-9._:-]{0,127}$/; +const numberValue = (value: unknown): number => typeof value === 'number' && Number.isFinite(value) ? value : 0; +const stringValue = (value: unknown, fallback: string): string => typeof value === 'string' && value.trim() ? value.trim() : fallback; +const round = (value: number, digits = 2): number => Number(value.toFixed(digits)); +const configuredThreshold = (name: string, fallback: number): number => { + const parsed = Number(process.env[name] ?? fallback); + return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback; +}; +const rowProject = (row: TelemetryRow): string => stringValue(row.project ?? row.project_id, 'default'); +const rowRun = (row: TelemetryRow): string => stringValue(row.run_id ?? row.trace_id, 'unattributed'); +const rowTime = (row: TelemetryRow): number | null => { + const timestamp = typeof row.timestamp === 'string' ? Date.parse(row.timestamp) : Number.NaN; + return Number.isFinite(timestamp) ? timestamp : null; +}; + +function parseDate(value: string | undefined, label: 'from' | 'to'): string | null { + if (!value) return null; + const dateOnly = /^\d{4}-\d{2}-\d{2}$/.test(value); + const normalized = dateOnly ? `${value}T${label === 'from' ? '00:00:00.000' : '23:59:59.999'}Z` : value; + const parsed = Date.parse(normalized); + if (!Number.isFinite(parsed)) throw new BadRequestException(`H6_REPORT_INVALID_${label.toUpperCase()}`); + return new Date(parsed).toISOString(); +} + +function parseFilter(value: string | undefined, label: 'project' | 'run'): string | null { + if (!value) return null; + if (!safeFilter.test(value)) throw new BadRequestException(`H6_REPORT_INVALID_${label.toUpperCase()}`); + return value; +} + +export function parseH6ReportQuery(raw: H6ReportQueryParams): H6ReportQuery { + const from = parseDate(raw.from, 'from'); + const to = parseDate(raw.to, 'to'); + if (from && to && Date.parse(from) > Date.parse(to)) throw new BadRequestException('H6_REPORT_INVALID_TIME_RANGE'); + const parsedLimit = Number(raw.limit ?? 50); + if (!Number.isInteger(parsedLimit) || parsedLimit < 1 || parsedLimit > 200) { + throw new BadRequestException('H6_REPORT_INVALID_LIMIT'); + } + return { + project: parseFilter(raw.project, 'project'), + from, + to, + run: parseFilter(raw.run, 'run'), + limit: parsedLimit, + }; +} + +function filtered(rows: TelemetryRow[], query: H6ReportQuery): TelemetryRow[] { + const fromMs = query.from ? Date.parse(query.from) : null; + const toMs = query.to ? Date.parse(query.to) : null; + return rows.filter((row) => { + if (query.project && rowProject(row) !== query.project) return false; + if (query.run && rowRun(row) !== query.run) return false; + const timestamp = rowTime(row); + if (fromMs !== null && (timestamp === null || timestamp < fromMs)) return false; + if (toMs !== null && (timestamp === null || timestamp > toMs)) return false; + return true; + }); +} + +function percentile(values: number[], percentileValue: number): number { + if (values.length === 0) return 0; + const sorted = [...values].sort((left, right) => left - right); + const index = Math.max(0, Math.ceil((percentileValue / 100) * sorted.length) - 1); + return sorted[index]; +} + +function sum(rows: TelemetryRow[], key: string): number { + return rows.reduce((total, row) => total + numberValue(row[key]), 0); +} + +function grouped(rows: TelemetryRow[], keyOf: (row: TelemetryRow) => string): H6Breakdown[] { + const groups = new Map(); + for (const row of rows) { + const key = keyOf(row); + groups.set(key, [...(groups.get(key) ?? []), row]); + } + return [...groups.entries()].map(([key, records]) => { + const latencies = records.map((row) => numberValue(row.latency_ms)).filter((value) => value > 0); + return { + key, + runs: records.length, + failures: records.filter((row) => stringValue(row.status, 'unknown') === 'failed').length, + latency_avg_ms: latencies.length ? Math.round(latencies.reduce((total, value) => total + value, 0) / latencies.length) : 0, + tokens: sum(records, 'total_tokens'), + cost_usd: round(sum(records, records.some((row) => row.cost_usd !== undefined) ? 'cost_usd' : 'cost_estimate'), 6), + }; + }).sort((left, right) => right.runs - left.runs || left.key.localeCompare(right.key)); +} + +function counted(rows: TelemetryRow[], keyOf: (row: TelemetryRow) => string): Array<{ key: string; count: number }> { + const counts = new Map(); + for (const row of rows) { + const key = keyOf(row); + counts.set(key, (counts.get(key) ?? 0) + 1); + } + return [...counts.entries()].map(([key, countValue]) => ({ key, count: countValue })) + .sort((left, right) => right.count - left.count || left.key.localeCompare(right.key)); +} + +function alertTypes(row: TelemetryRow): string[] { + const structured = row.body && typeof row.body === 'object' ? (row.body as Record)['alert.type'] : undefined; + if (typeof structured === 'string') return [structured]; + if (Array.isArray(row.alerts)) return row.alerts.filter((value): value is string => typeof value === 'string'); + return []; +} + +function evidenceSources(input: H6ReportInput): HarnessReportEvidenceSource[] { + const counts: Record = { + metrics: input.metrics.length, + provider: input.provider.length, + alerts: input.alerts.length, + }; + return input.sourceFreshness.map((source) => { + const appRelativePath = relative(APP_ROOT, source.path) || '.'; + return { + ...source, + path: appRelativePath.startsWith('..') ? `[external]/${basename(source.path)}` : appRelativePath, + records: counts[source.source], + }; + }); +} + +export function buildH6Report(input: H6ReportInput, query: H6ReportQuery): H6Report { + const metrics = filtered(input.metrics, query); + const provider = filtered(input.provider, query); + const alerts = filtered(input.alerts, query); + const latencies = metrics.map((row) => numberValue(row.latency_ms)).filter((value) => value > 0); + const failed = metrics.filter((row) => stringValue(row.status, 'unknown') === 'failed').length; + const degraded = metrics.filter((row) => stringValue(row.status, 'unknown') === 'degraded').length; + const success = metrics.filter((row) => ['success', 'pass', 'passed'].includes(stringValue(row.status, 'unknown'))).length; + const sourceEvidence = evidenceSources(input); + const primary = sourceEvidence.find((source) => source.source === 'metrics'); + const failureRate = metrics.length ? round((failed / metrics.length) * 100, 1) : 0; + const p95 = percentile(latencies, 95); + const failureThreshold = configuredThreshold('CASAN_H6_FAILURE_RATE_THRESHOLD_PCT', 5); + const p95Threshold = configuredThreshold('CASAN_H6_P95_LATENCY_THRESHOLD_MS', 5000); + const findings: HarnessReportFinding[] = []; + const warnings: string[] = []; + + if (!primary?.present) findings.push({ severity: 'critical', code: 'METRICS_MISSING', message: 'The required runtime metrics source is missing.' }); + else if (primary.stale) findings.push({ severity: 'warning', code: 'METRICS_STALE', message: 'The primary runtime metrics source is stale.', metric: 'age_s', value: primary.age_s ?? 'unknown', threshold: STALE_AFTER_S }); + if (failureRate > failureThreshold) findings.push({ severity: 'critical', code: 'FAILURE_RATE_BREACH', message: 'Failure rate exceeds the configured H6 threshold.', metric: 'failure_rate_pct', value: failureRate, threshold: failureThreshold }); + else if (failed > 0) findings.push({ severity: 'warning', code: 'FAILURES_PRESENT', message: `${failed} failed run(s) are present in the selected scope.`, metric: 'failed', value: failed }); + if (p95 > p95Threshold) findings.push({ severity: 'warning', code: 'P95_LATENCY_BREACH', message: 'P95 latency exceeds the configured H6 threshold.', metric: 'p95_latency_ms', value: p95, threshold: p95Threshold }); + const alertFingerprints = new Map>(); + for (const row of [...metrics, ...alerts]) { + for (const alert of alertTypes(row)) { + const run = rowRun(row); + const fingerprint = `${run === 'unattributed' ? stringValue(row.timestamp, 'unknown-time') : run}:${alert}`; + const known = alertFingerprints.get(alert) ?? new Set(); + known.add(fingerprint); + alertFingerprints.set(alert, known); + } + } + const alertCounts = [...alertFingerprints.entries()].map(([alert, fingerprints]) => ({ alert, count: fingerprints.size })); + const alertCount = alertCounts.reduce((total, entry) => total + entry.count, 0); + if (alertCount > 0) findings.push({ severity: 'warning', code: 'ALERTS_PRESENT', message: `${alertCount} alert signal(s) require review.`, metric: 'alerts', value: alertCount }); + if (degraded > 0) findings.push({ severity: 'warning', code: 'DEGRADED_RUNS_PRESENT', message: `${degraded} degraded run(s) are present in the selected scope.`, metric: 'degraded', value: degraded }); + if (provider.length === 0) warnings.push('No provider usage records matched the selected scope; token and actual-cost breakdown may be incomplete.'); + for (const source of sourceEvidence) { + if (source.source === 'metrics') continue; + if (!source.present) warnings.push(`Optional ${source.source} telemetry source is missing; its breakdown is unavailable.`); + else if (source.stale) warnings.push(`Optional ${source.source} telemetry source is stale; its breakdown may not reflect recent activity.`); + } + const estimatedRecords = metrics.filter((row) => stringValue(row.cost_source, 'unknown').includes('estimate')).length; + if (estimatedRecords > 0) warnings.push(`${estimatedRecords} metric record(s) use estimated rather than provider-reported cost.`); + const unattributed = [...input.metrics, ...input.provider].filter((row) => row.project === undefined && row.project_id === undefined).length; + if (unattributed > 0) warnings.push(`${unattributed} source record(s) have no project attribute and are classified as project "default".`); + + const critical = findings.some((finding) => finding.severity === 'critical'); + const warning = findings.some((finding) => finding.severity === 'warning'); + const verdict = metrics.length === 0 && provider.length === 0 ? 'no_data' : critical ? 'fail' : warning ? 'attention' : 'pass'; + const freshnessStatus = !primary?.present ? 'missing' : primary.stale ? 'stale' : 'live'; + const availableProjects = [...new Set([...input.metrics, ...input.provider, ...input.alerts].map(rowProject))].sort(); + const availableRuns = [...new Set([...input.metrics, ...input.provider, ...input.alerts].map(rowRun).filter((run) => run !== 'unattributed'))].sort(); + const costSources = grouped(metrics, (row) => stringValue(row.cost_source, 'unknown')).map((entry) => ({ + source: entry.key, + records: entry.runs, + cost_usd: entry.cost_usd, + })); + const scope: HarnessReportScope = { project: query.project, from: query.from, to: query.to, run: query.run }; + + return { + schema_version: 1, + report_id: `H6-${input.now.toISOString().replace(/[-:.TZ]/g, '').slice(0, 14)}`, + harness: 'H6', + title: 'H6 · AgentOps Report', + description: 'Evidence-backed runtime, token, cost, failure, retry and alert telemetry.', + generated_at: input.now.toISOString(), + scope, + verdict, + verdict_reasons: findings.map((finding) => finding.code), + freshness: { status: freshnessStatus, stale_after_s: STALE_AFTER_S, primary_age_s: primary?.age_s ?? null, sources: sourceEvidence }, + summary: { + runs: metrics.length, + success, + failed, + degraded, + failure_rate_pct: failureRate, + retries: sum(metrics, 'retry_count'), + latency_ms: { + average: latencies.length ? Math.round(latencies.reduce((total, value) => total + value, 0) / latencies.length) : 0, + p50: percentile(latencies, 50), + p95, + p99: percentile(latencies, 99), + max: latencies.length ? Math.max(...latencies) : 0, + }, + tokens: { input: sum(metrics, 'input_tokens'), output: sum(metrics, 'output_tokens'), total: sum(metrics, 'total_tokens'), provider_total: sum(provider, 'total_tokens') }, + cost_usd: { provider_actual: round(sum(provider, 'cost_usd'), 6), estimated: round(sum(metrics, 'cost_estimate'), 6) }, + provider_calls: provider.length, + alerts: alertCount, + }, + thresholds: { failure_rate_pct: failureThreshold, p95_latency_ms: p95Threshold, freshness_age_s: STALE_AFTER_S }, + findings, + evidence_sources: sourceEvidence, + data_quality: { status: !primary?.present ? 'insufficient' : warnings.length ? 'partial' : 'complete', warnings }, + available_filters: { projects: availableProjects, runs: availableRuns }, + details: { + by_status: counted(metrics, (row) => stringValue(row.status, 'unknown')).map(({ key, count: countValue }) => ({ status: key, count: countValue })), + by_step: grouped(metrics, (row) => stringValue(row.step, 'unknown-step')), + by_provider: grouped(provider, (row) => `${stringValue(row.provider, 'unknown-provider')} · ${stringValue(row.model, 'unknown-model')}`), + by_cost_source: costSources, + by_alert: alertCounts.sort((left, right) => right.count - left.count || left.alert.localeCompare(right.alert)), + recent_runs: [...metrics].slice(-query.limit).reverse(), + recent_alerts: [...alerts].slice(-query.limit).reverse(), + }, + }; +} + +export function defaultH6ReportInput(now = new Date()): H6ReportInput { + return { + metrics: [], + provider: [], + alerts: [], + sourceFreshness: telemetryFreshness(now.getTime()), + now, + }; +} + +export const H6_REPORT_PATHS = PATHS; diff --git a/packages/casan-control-panel/backend/src/reports/report.contract.ts b/packages/casan-control-panel/backend/src/reports/report.contract.ts new file mode 100644 index 0000000..b926b9a --- /dev/null +++ b/packages/casan-control-panel/backend/src/reports/report.contract.ts @@ -0,0 +1,83 @@ +import type { SourceFreshness } from '../common/app-root.js'; + +export const HARNESS_REPORT_DEFINITIONS = [ + { id: 'H1', slug: 'context', title: 'H1 · Context', description: 'Context selection, prompt contract and risk classification' }, + { id: 'H2', slug: 'tool', title: 'H2 · Tool', description: 'Allowlisted tools, actions and execution boundaries' }, + { id: 'H3', slug: 'evaluation', title: 'H3 · Evaluation', description: 'Quality evaluation, traceability and acceptance evidence' }, + { id: 'H4', slug: 'security', title: 'H4 · Security', description: 'Input, output and artifact security controls' }, + { id: 'H5', slug: 'governance', title: 'H5 · Governance', description: 'Policy decisions, approvals and audit integrity' }, + { id: 'H6', slug: 'agentops', title: 'H6 · AgentOps', description: 'Runtime, token, cost, failure and alert telemetry' }, + { id: 'H7', slug: 'orchestration', title: 'H7 · Orchestration', description: 'Final outcome, certification and rollback evidence' }, +] as const; + +export type HarnessReportId = typeof HARNESS_REPORT_DEFINITIONS[number]['id']; +export type HarnessReportVerdict = 'pass' | 'attention' | 'fail' | 'no_data'; +export type ReportExportFormat = 'json' | 'html'; + +export interface HarnessReportScope { + project: string | null; + from: string | null; + to: string | null; + run: string | null; +} + +export interface HarnessReportFinding { + severity: 'info' | 'warning' | 'critical'; + code: string; + message: string; + metric?: string; + value?: number | string; + threshold?: number | string; +} + +export interface HarnessReportEvidenceSource extends SourceFreshness { + records: number; +} + +export interface HarnessReport { + schema_version: 1; + report_id: string; + harness: HarnessReportId; + title: string; + description: string; + generated_at: string; + scope: HarnessReportScope; + verdict: HarnessReportVerdict; + verdict_reasons: string[]; + freshness: { + status: 'live' | 'stale' | 'missing'; + stale_after_s: number; + primary_age_s: number | null; + sources: HarnessReportEvidenceSource[]; + }; + summary: TSummary; + thresholds: Record; + findings: HarnessReportFinding[]; + evidence_sources: HarnessReportEvidenceSource[]; + data_quality: { + status: 'complete' | 'partial' | 'insufficient'; + warnings: string[]; + }; + available_filters: { + projects: string[]; + runs: string[]; + }; + details: TDetails; +} + +export interface HarnessReportCatalogEntry { + id: HarnessReportId; + slug: string; + title: string; + description: string; + contract_version: 1; + endpoint: string; + availability: 'implemented' | 'contract_ready'; +} + +export const HARNESS_REPORT_CATALOG: HarnessReportCatalogEntry[] = HARNESS_REPORT_DEFINITIONS.map((definition) => ({ + ...definition, + contract_version: 1, + endpoint: `/api/v1/reports/${definition.id.toLowerCase()}`, + availability: definition.id === 'H6' ? 'implemented' : 'contract_ready', +})); diff --git a/packages/casan-control-panel/backend/src/reports/reports.controller.ts b/packages/casan-control-panel/backend/src/reports/reports.controller.ts new file mode 100644 index 0000000..d349a68 --- /dev/null +++ b/packages/casan-control-panel/backend/src/reports/reports.controller.ts @@ -0,0 +1,37 @@ +import { BadRequestException, Controller, Get, Header, Inject, Query, Res } from '@nestjs/common'; +import type { Response } from 'express'; +import { ok } from '../common/api-response.js'; +import { parseH6ReportQuery, type H6ReportQueryParams } from './h6-report.js'; +import { ReportsService } from './reports.service.js'; + +interface H6QueryParams extends H6ReportQueryParams { + format?: string; +} + +@Controller('api/v1/reports') +export class ReportsController { + constructor(@Inject(ReportsService) private readonly reports: ReportsService) {} + + @Get() + catalog() { + return ok(this.reports.catalog()); + } + + @Get('h6') + h6(@Query() raw: H6QueryParams) { + return ok(this.reports.h6(parseH6ReportQuery(raw))); + } + + @Get('h6/export') + @Header('Cache-Control', 'no-store') + exportH6(@Query() raw: H6QueryParams, @Res() response: Response) { + const format = raw.format ?? 'json'; + if (format !== 'json' && format !== 'html') throw new BadRequestException('H6_REPORT_INVALID_FORMAT'); + const report = this.reports.h6(parseH6ReportQuery(raw)); + const body = this.reports.serializeH6(report, format); + const stamp = report.generated_at.slice(0, 10); + response.type(format === 'html' ? 'text/html; charset=utf-8' : 'application/json; charset=utf-8'); + response.setHeader('Content-Disposition', `attachment; filename="casan-h6-report-${stamp}.${format}"`); + response.send(body); + } +} diff --git a/packages/casan-control-panel/backend/src/reports/reports.module.ts b/packages/casan-control-panel/backend/src/reports/reports.module.ts new file mode 100644 index 0000000..3a1fd41 --- /dev/null +++ b/packages/casan-control-panel/backend/src/reports/reports.module.ts @@ -0,0 +1,10 @@ +import { Module } from '@nestjs/common'; +import { ReportsController } from './reports.controller.js'; +import { ReportsService } from './reports.service.js'; + +@Module({ + controllers: [ReportsController], + providers: [ReportsService], + exports: [ReportsService], +}) +export class ReportsModule {} diff --git a/packages/casan-control-panel/backend/src/reports/reports.service.ts b/packages/casan-control-panel/backend/src/reports/reports.service.ts new file mode 100644 index 0000000..912ef52 --- /dev/null +++ b/packages/casan-control-panel/backend/src/reports/reports.service.ts @@ -0,0 +1,28 @@ +import { Injectable } from '@nestjs/common'; +import { PATHS, telemetryFreshness } from '../common/app-root.js'; +import { readJsonl } from '../telemetry/telemetry.reader.js'; +import { buildH6Report, type H6Report, type H6ReportQuery } from './h6-report.js'; +import { renderH6ReportHtml } from './h6-report.html.js'; +import { HARNESS_REPORT_CATALOG } from './report.contract.js'; + +@Injectable() +export class ReportsService { + catalog() { + return { schema_version: 1, reports: HARNESS_REPORT_CATALOG }; + } + + h6(query: H6ReportQuery): H6Report { + const now = new Date(); + return buildH6Report({ + metrics: readJsonl(PATHS.metrics), + provider: readJsonl(PATHS.providerUsage), + alerts: readJsonl(PATHS.alerts), + sourceFreshness: telemetryFreshness(now.getTime()), + now, + }, query); + } + + serializeH6(report: H6Report, format: 'json' | 'html'): string { + return format === 'html' ? renderH6ReportHtml(report) : `${JSON.stringify(report, null, 2)}\n`; + } +} diff --git a/packages/casan-control-panel/backend/src/telemetry/telemetry.service.ts b/packages/casan-control-panel/backend/src/telemetry/telemetry.service.ts index 5934401..759a7cf 100644 --- a/packages/casan-control-panel/backend/src/telemetry/telemetry.service.ts +++ b/packages/casan-control-panel/backend/src/telemetry/telemetry.service.ts @@ -5,7 +5,7 @@ import { existsSync, readFileSync, statSync } from 'node:fs'; import { execFileSync } from 'node:child_process'; import { join, relative } from 'node:path'; import { Injectable } from '@nestjs/common'; -import { APP_ROOT, PATHS, isStale, metricsAgeSeconds, STALE_AFTER_S } from '../common/app-root.js'; +import { APP_ROOT, PATHS, isStale, metricsAgeSeconds, STALE_AFTER_S, telemetryFreshness } from '../common/app-root.js'; import { readJsonl, readJson, readHead, readTrace } from './telemetry.reader.js'; type Row = Record; @@ -217,7 +217,7 @@ export function buildChatLoopWidget( export class TelemetryService { private freshness() { const age = metricsAgeSeconds(); - return { stale: isStale(), age_s: age, stale_after_s: STALE_AFTER_S }; + return { stale: isStale(), age_s: age, stale_after_s: STALE_AFTER_S, sources: telemetryFreshness() }; } overview() { diff --git a/packages/casan-control-panel/backend/test/reports.test.ts b/packages/casan-control-panel/backend/test/reports.test.ts new file mode 100644 index 0000000..b0686ec --- /dev/null +++ b/packages/casan-control-panel/backend/test/reports.test.ts @@ -0,0 +1,119 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { BadRequestException } from '@nestjs/common'; +import { mkdtempSync, utimesSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { sourceFreshness } from '../src/common/app-root.js'; +import { buildH6Report, parseH6ReportQuery, type H6ReportInput } from '../src/reports/h6-report.js'; +import { renderH6ReportHtml } from '../src/reports/h6-report.html.js'; +import { HARNESS_REPORT_CATALOG } from '../src/reports/report.contract.js'; + +const NOW = new Date('2026-07-20T12:00:00.000Z'); + +function fixture(): H6ReportInput { + return { + now: NOW, + metrics: [ + { timestamp: '2026-07-19T08:00:00Z', trace_id: 'run-a', project: 'okr', status: 'success', step: 'generate', latency_ms: 100, input_tokens: 10, output_tokens: 20, total_tokens: 30, cost_estimate: 0.001, cost_source: 'word_count_estimate', retry_count: 0, alerts: [] }, + { timestamp: '2026-07-19T09:00:00Z', trace_id: 'run-b', project: 'okr', status: 'failed', step: 'test', latency_ms: 9000, input_tokens: 20, output_tokens: 5, total_tokens: 25, cost_estimate: 0.002, cost_source: 'provider_telemetry', retry_count: 2, alerts: ['high-latency', 'execution-failed'] }, + { timestamp: '2026-07-18T09:00:00Z', trace_id: 'run-c', project: 'desk', status: 'success', step: 'test', latency_ms: 250, total_tokens: 10, cost_estimate: 0.0001, cost_source: 'word_count_estimate' }, + ], + provider: [ + { timestamp: '2026-07-19T08:00:00Z', run_id: 'run-a', project: 'okr', provider: 'ollama', model: 'ornith:9b', status: 'success', step: 'generate', latency_ms: 80, total_tokens: 28, cost_usd: 0 }, + { timestamp: '2026-07-18T09:00:00Z', run_id: 'run-c', project: 'desk', provider: 'openai', model: 'gpt-test', status: 'success', step: 'test', latency_ms: 200, total_tokens: 10, cost_usd: 0.01 }, + ], + alerts: [ + { timestamp: '2026-07-19T09:00:00Z', trace_id: 'run-b', project: 'okr', body: { 'alert.type': 'execution-failed' } }, + ], + sourceFreshness: [ + { source: 'metrics', path: '/workspace/.specify/logs/cost/metrics.jsonl', present: true, updated_at: '2026-07-20T11:59:00Z', age_s: 60, stale: false, required: true }, + { source: 'provider', path: '/workspace/.specify/logs/level5/provider-usage.jsonl', present: true, updated_at: '2026-07-20T11:58:00Z', age_s: 120, stale: false, required: false }, + { source: 'alerts', path: '/workspace/.specify/agentops/alerts.log', present: true, updated_at: '2026-07-20T11:57:00Z', age_s: 180, stale: false, required: false }, + ], + }; +} + +test('H1-H7 share one versioned report catalog contract', () => { + assert.deepEqual(HARNESS_REPORT_CATALOG.map((entry) => entry.id), ['H1', 'H2', 'H3', 'H4', 'H5', 'H6', 'H7']); + assert.ok(HARNESS_REPORT_CATALOG.every((entry) => entry.contract_version === 1)); + assert.equal(HARNESS_REPORT_CATALOG.find((entry) => entry.id === 'H6')?.availability, 'implemented'); +}); + +test('telemetry freshness is source-specific and missing required data fails loud', () => { + const directory = mkdtempSync(join(tmpdir(), 'casan-h6-')); + const metricsPath = join(directory, 'metrics.jsonl'); + writeFileSync(metricsPath, '{}\n'); + const oldTime = new Date(NOW.getTime() - 7200 * 1000); + utimesSync(metricsPath, oldTime, oldTime); + const staleMetrics = sourceFreshness('metrics', metricsPath, true, NOW.getTime()); + const missingMetrics = sourceFreshness('metrics', join(directory, 'missing.jsonl'), true, NOW.getTime()); + assert.equal(staleMetrics.present, true); + assert.equal(staleMetrics.age_s, 7200); + assert.equal(staleMetrics.stale, true); + assert.deepEqual( + { present: missingMetrics.present, stale: missingMetrics.stale, required: missingMetrics.required }, + { present: false, stale: true, required: true }, + ); +}); + +test('H6 report filters project/time/run and aggregates measured evidence', () => { + const query = parseH6ReportQuery({ project: 'okr', from: '2026-07-19', to: '2026-07-19' }); + const report = buildH6Report(fixture(), query); + assert.equal(report.schema_version, 1); + assert.equal(report.harness, 'H6'); + assert.equal(report.summary.runs, 2); + assert.equal(report.summary.failed, 1); + assert.equal(report.summary.retries, 2); + assert.equal(report.summary.latency_ms.p50, 100); + assert.equal(report.summary.latency_ms.p95, 9000); + assert.equal(report.summary.tokens.provider_total, 28); + assert.equal(report.summary.alerts, 2); + assert.deepEqual(report.details.by_alert, [ + { alert: 'execution-failed', count: 1 }, + { alert: 'high-latency', count: 1 }, + ]); + assert.equal(report.available_filters.projects.length, 2); + assert.equal(report.freshness.status, 'live'); + assert.equal(report.verdict, 'fail'); + assert.ok(report.findings.some((finding) => finding.code === 'FAILURE_RATE_BREACH')); +}); + +test('H6 report supports exact run filtering and truthful no-data verdict', () => { + const selected = buildH6Report(fixture(), parseH6ReportQuery({ run: 'run-a' })); + assert.equal(selected.summary.runs, 1); + assert.equal(selected.summary.provider_calls, 1); + assert.equal(selected.verdict, 'pass'); + const empty = buildH6Report(fixture(), parseH6ReportQuery({ run: 'not-found' })); + assert.equal(empty.verdict, 'no_data'); + assert.equal(empty.summary.runs, 0); +}); + +test('H6 query validation rejects traversal, inverted dates and invalid limits', () => { + assert.throws(() => parseH6ReportQuery({ project: '../secret' }), BadRequestException); + assert.throws(() => parseH6ReportQuery({ from: '2026-07-20', to: '2026-07-19' }), BadRequestException); + assert.throws(() => parseH6ReportQuery({ limit: '5000' }), BadRequestException); +}); + +test('H6 report makes stale optional sources explicit in data quality', () => { + const input = fixture(); + input.sourceFreshness[1] = { ...input.sourceFreshness[1], stale: true, age_s: 7200 }; + input.sourceFreshness[2] = { ...input.sourceFreshness[2], present: false, updated_at: null, age_s: null, stale: false }; + const report = buildH6Report(input, parseH6ReportQuery({ run: 'run-a' })); + assert.equal(report.data_quality.status, 'partial'); + assert.ok(report.data_quality.warnings.some((warning) => warning.includes('provider telemetry source is stale'))); + assert.ok(report.data_quality.warnings.some((warning) => warning.includes('alerts telemetry source is missing'))); +}); + +test('HTML export is standalone, escaped and contains no hard-coded maturity score', () => { + const input = fixture(); + input.metrics[0].step = ''; + const report = buildH6Report(input, parseH6ReportQuery({ project: 'okr' })); + const html = renderH6ReportHtml(report); + assert.match(html, /^/); + assert.match(html, /H6 · AgentOps Report/); + assert.match(html, /<script>alert\(1\)<\/script>/); + assert.doesNotMatch(html, /