update report h6
This commit is contained in:
thanhnv
@@ -537,3 +537,6 @@
|
||||
{"timestamp":"2026-07-19T04:00:29Z","trace_id":"trace-1784433629-809","harness":"H4-security","mode":"output","status":"pass","action":"allow","risk_level":"low","matched_categories":[],"input_hash":"e39bb3117bcde6f297fe51ac7d86546bbfbb7b17339821001d681b95a766019b","output_hash":"665a7b0ed97ad39aeec1aab585b6a0e3779b261f7d0a6734f7f4cb07672a3354"}
|
||||
{"timestamp":"2026-07-19T04:01:15Z","trace_id":"trace-1784433675-137","harness":"H4-security","mode":"input","status":"pass","action":"alert","risk_level":"medium","matched_categories":["semantic-availability"],"input_hash":"152e2c124ba6f23a03165ccb19ef9f04d6eb1155858101e8f741bb89f2245234","output_hash":"152e2c124ba6f23a03165ccb19ef9f04d6eb1155858101e8f741bb89f2245234"}
|
||||
{"timestamp":"2026-07-19T04:01:16Z","trace_id":"trace-1784433676-759","harness":"H4-security","mode":"output","status":"pass","action":"allow","risk_level":"low","matched_categories":[],"input_hash":"e39bb3117bcde6f297fe51ac7d86546bbfbb7b17339821001d681b95a766019b","output_hash":"e39bb3117bcde6f297fe51ac7d86546bbfbb7b17339821001d681b95a766019b"}
|
||||
{"timestamp":"2026-07-19T05:18:02Z","trace_id":"trace-1784438282-2705","harness":"H4-security","mode":"input","status":"pass","action":"alert","risk_level":"medium","matched_categories":["semantic-availability"],"input_hash":"3084720034103b1c979a3ae80f6f59e360833eaa6a8c88d45381820edfbe2886","output_hash":"3084720034103b1c979a3ae80f6f59e360833eaa6a8c88d45381820edfbe2886"}
|
||||
{"timestamp":"2026-07-19T05:18:03Z","trace_id":"trace-1784438283-3332","harness":"H4-security","mode":"output","status":"pass","action":"allow","risk_level":"low","matched_categories":[],"input_hash":"92848c78682b10335aedc8fa6ebfb8e9b7b5f17d9709df94b7742bda7412631e","output_hash":"92848c78682b10335aedc8fa6ebfb8e9b7b5f17d9709df94b7742bda7412631e"}
|
||||
{"timestamp":"2026-07-19T05:18:20Z","trace_id":"trace-1784438300-3900","harness":"H4-security","mode":"input","status":"pass","action":"allow","risk_level":"low","matched_categories":[],"input_hash":"cbd0ed3c59982438aadbf2372bf8e463b9e2a884a4e452472683bca515616767","output_hash":"14487285dc9cada36542a27770878b55e3929878a40ebb527b67feef56fd5b8e"}
|
||||
|
||||
@@ -197,6 +197,26 @@ docker volume rm casan-local-prod_minio-data
|
||||
This is destructive and cannot be undone. Do not use it if the MinIO bucket has
|
||||
evidence you need to retain.
|
||||
|
||||
### Python used by the local auth bridge
|
||||
|
||||
`local-full.sh` validates the Python interpreter before starting the host-side
|
||||
provider auth bridge. On macOS it prefers `/usr/bin/python3`, avoiding an old
|
||||
Intel-only framework Python that may be first in `PATH` but terminated by macOS.
|
||||
To select another working Python explicitly, set `CASAN_PYTHON_BIN`:
|
||||
|
||||
```bash
|
||||
CASAN_PYTHON_BIN=/opt/homebrew/bin/python3 \
|
||||
bash packages/casan-harness/scripts/bash/local-full.sh start
|
||||
```
|
||||
|
||||
The launcher also validates Docker's configured credential helper. If macOS
|
||||
terminates that helper, CASAN uses an isolated, temporary anonymous Docker
|
||||
configuration for this local stack's public images while preserving the active
|
||||
Docker Desktop socket and Compose plugin. The same fallback uses Docker's classic
|
||||
builder if the bundled BuildKit metadata helper is also terminated. It does not
|
||||
edit `~/.docker/config.json` or delete stored registry credentials. Reinstall or
|
||||
update Docker Desktop later to repair the helper globally.
|
||||
|
||||
## Current scope and next phase
|
||||
|
||||
This local environment includes every currently implemented CASAN component.
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
# CASAN Plan-19 — Harness Reporting Contract
|
||||
|
||||
## Purpose
|
||||
|
||||
Turn harness evidence into reviewable, portable reports. H6 is the first implemented
|
||||
report; H1–H7 share one contract so later reports do not invent incompatible envelopes,
|
||||
filter semantics or verdict fields.
|
||||
|
||||
## Source-of-truth boundary
|
||||
|
||||
H6 is a read-only projection over three canonical files. The report API and the existing
|
||||
Control Panel resolve these paths through the same `app-root.ts` contract; harness writers
|
||||
resolve them through `casan-paths.sh`.
|
||||
|
||||
| Source | Canonical environment variable | Default path | Required |
|
||||
|---|---|---|---|
|
||||
| Runtime metrics | `CASAN_TELEMETRY_METRICS_LOG` | `.specify/logs/cost/metrics.jsonl` | yes |
|
||||
| Provider usage | `CASAN_TELEMETRY_PROVIDER_LOG` | `.specify/logs/level5/provider-usage.jsonl` | no |
|
||||
| Alerts | `CASAN_TELEMETRY_ALERTS_LOG` | `.specify/agentops/alerts.log` | no |
|
||||
|
||||
Legacy names (`CASAN_DASHBOARD_METRICS`, `CASAN_CP_PROVIDER_USAGE`,
|
||||
`CASAN_PROVIDER_LOG`, `CASAN_DASHBOARD_ALERTS`, and chat-specific metrics aliases) are
|
||||
compatibility fallbacks only. New deployments should set the canonical variables.
|
||||
|
||||
Freshness is source-specific and based on filesystem modification time. A source becomes
|
||||
stale when its age exceeds `CASAN_DASHBOARD_STALE_S` (default `3600` seconds). Missing
|
||||
required metrics produce a `missing` freshness status and an insufficient-data verdict.
|
||||
|
||||
## Common H1–H7 contract
|
||||
|
||||
`packages/casan-control-panel/backend/src/reports/report.contract.ts` defines schema
|
||||
version `1`. Every harness report has:
|
||||
|
||||
- identity: `report_id`, `harness`, `title`, `generated_at`;
|
||||
- scope: `project`, `from`, `to`, `run`;
|
||||
- decision: `verdict`, `verdict_reasons`, and explicit `thresholds`;
|
||||
- evidence: source-level freshness, record counts and findings;
|
||||
- quality: `complete`, `partial` or `insufficient` with warnings;
|
||||
- presentation-neutral `summary` and `details` data.
|
||||
|
||||
The catalog endpoint marks H6 as `implemented`; H1–H5 and H7 are `contract_ready`. This
|
||||
is deliberate: report presence must not be confused with measured evidence or a passing
|
||||
score.
|
||||
|
||||
## H6 API and UI
|
||||
|
||||
| Capability | Route |
|
||||
|---|---|
|
||||
| Report catalog | `GET /api/v1/reports` |
|
||||
| H6 JSON API | `GET /api/v1/reports/h6` |
|
||||
| Independent export | `GET /api/v1/reports/h6/export?format=json|html` |
|
||||
| Browser report | `/reports/h6` |
|
||||
|
||||
Optional filters are `project`, `from`, `to`, and `run`; `limit` only controls recent-row
|
||||
detail and does not change summary aggregation. Project/run identifiers are allowlisted to
|
||||
safe identifier characters, dates must parse as ISO-compatible values, and inverted ranges
|
||||
fail with `400`.
|
||||
|
||||
The H6 summary includes runtime counts and outcomes, failure rate, retries, latency
|
||||
distribution, local/provider tokens, estimated/provider-reported cost, provider calls and
|
||||
alert signals. Verdicts are evidence-derived:
|
||||
|
||||
- `no_data`: no runtime or provider record matches the scope;
|
||||
- `fail`: required metrics are missing or failure rate exceeds its threshold;
|
||||
- `attention`: stale telemetry, failures, degraded runs, high P95 latency or alerts exist;
|
||||
- `pass`: data is present and no finding crosses a configured threshold.
|
||||
|
||||
Defaults are `5%` maximum failure rate and `5000 ms` maximum P95 latency, configurable
|
||||
through `CASAN_H6_FAILURE_RATE_THRESHOLD_PCT` and
|
||||
`CASAN_H6_P95_LATENCY_THRESHOLD_MS`.
|
||||
|
||||
## Export integrity
|
||||
|
||||
HTML and JSON exports are generated from the exact report object returned by the H6 service.
|
||||
The HTML artifact is standalone and print-ready; JSON keeps schema version `1` for automated
|
||||
consumers. Neither export reads a second data source or contains a hard-coded maturity
|
||||
score. Source paths outside the app root are represented as `[external]/<filename>` so an
|
||||
export does not disclose host directory structure.
|
||||
|
||||
## Verification
|
||||
|
||||
```bash
|
||||
npm run build -w @casan/control-panel-backend
|
||||
npm run build -w @casan/control-panel-frontend
|
||||
cd packages/casan-control-panel/backend
|
||||
node --import tsx --test test/reports.test.ts
|
||||
```
|
||||
|
||||
The report tests cover the common catalog, filtered aggregation, exact-run/no-data behavior,
|
||||
query validation, HTML escaping and absence of hard-coded scoring.
|
||||
@@ -107,9 +107,10 @@ services:
|
||||
environment:
|
||||
CASAN_DASHBOARD_BIND: 0.0.0.0
|
||||
CASAN_DASHBOARD_STALE_S: "315360000"
|
||||
CASAN_DASHBOARD_METRICS: /workspace/.specify/logs/cost/metrics.jsonl
|
||||
CASAN_TELEMETRY_METRICS_LOG: /workspace/.specify/logs/cost/metrics.jsonl
|
||||
CASAN_DASHBOARD_HTML: /workspace/docs/output/casan/central-agentops-dashboard.html
|
||||
CASAN_DASHBOARD_ALERTS: /workspace/.specify/agentops/alerts.log
|
||||
CASAN_TELEMETRY_PROVIDER_LOG: /workspace/.specify/logs/level5/provider-usage.jsonl
|
||||
CASAN_TELEMETRY_ALERTS_LOG: /workspace/.specify/agentops/alerts.log
|
||||
volumes:
|
||||
- ../..:/workspace:ro
|
||||
expose:
|
||||
|
||||
@@ -23,6 +23,17 @@ Open http://127.0.0.1:5174 — panels show REAL metrics from `.specify/logs/**`.
|
||||
`incidents` · `tools` · `traceability` · `drift` · `cost` · `GET /healthz` (200 fresh /
|
||||
503 stale — fail-loud, mirrors `dashboard-server.py`).
|
||||
|
||||
Harness reports:
|
||||
|
||||
- `GET /api/v1/reports` — versioned H1–H7 report catalog and availability.
|
||||
- `GET /api/v1/reports/h6?project=<id>&from=<ISO>&to=<ISO>&run=<id>` — H6 AgentOps
|
||||
report from the same runtime metrics, provider usage and alert files as the Control Panel.
|
||||
- `GET /api/v1/reports/h6/export?format=html|json&...` — standalone, print-ready HTML
|
||||
or machine-readable JSON export. Both formats serialize the same filtered report object;
|
||||
no maturity score is hard-coded.
|
||||
- `/reports/h6` — UI report view with project, time-range and run filters plus source-level
|
||||
freshness and data-quality warnings.
|
||||
|
||||
Metrics export: `GET /api/v1/metrics` provides Prometheus text exposition for
|
||||
aggregate freshness, run/failure/cost/token and H4/H5/action/incident counters.
|
||||
It intentionally contains no tenant, actor, trace, prompt or Evidence Pack
|
||||
@@ -121,7 +132,11 @@ FinOps/SLO:
|
||||
|
||||
Data sources + aggregation mirror `packages/casan-harness/tests/generate-agentops-dashboard.py`.
|
||||
App root + telemetry paths resolve via the same marker walk-up as `casan-paths.sh`
|
||||
(`.specify` or `packages/casan-harness`) and honor `CASAN_DASHBOARD_*` env overrides.
|
||||
(`.specify` or `packages/casan-harness`). The canonical source variables are
|
||||
`CASAN_TELEMETRY_METRICS_LOG`, `CASAN_TELEMETRY_PROVIDER_LOG`, and
|
||||
`CASAN_TELEMETRY_ALERTS_LOG`; legacy dashboard/control-panel names remain accepted as
|
||||
compatibility aliases. Freshness is calculated independently from each file's mtime using
|
||||
`CASAN_DASHBOARD_STALE_S` (default `3600`).
|
||||
|
||||
## Security posture (MVP)
|
||||
Binds `127.0.0.1` by default. Refuses a non-loopback bind under `CASAN_PROFILE=prod` /
|
||||
|
||||
@@ -9,9 +9,10 @@ import { ProviderAuthModule } from './provider-auth/provider-auth.module.js';
|
||||
import { GoalsModule } from './goals/goals.module.js';
|
||||
import { EvidenceModule } from './evidence/evidence.module.js';
|
||||
import { SessionController } from './session/session.controller.js';
|
||||
import { ReportsModule } from './reports/reports.module.js';
|
||||
|
||||
@Module({
|
||||
imports: [TelemetryModule, SettingsModule, KillSwitchModule, ApprovalsModule, ChatModule, ProviderAuthModule, GoalsModule, EvidenceModule],
|
||||
imports: [TelemetryModule, SettingsModule, KillSwitchModule, ApprovalsModule, ChatModule, ProviderAuthModule, GoalsModule, EvidenceModule, ReportsModule],
|
||||
controllers: [HealthController, SessionController],
|
||||
})
|
||||
export class AppModule {}
|
||||
|
||||
@@ -3,8 +3,8 @@
|
||||
// Mirrors packages/casan-harness/scripts/bash/casan-paths.sh: walk UP from a start dir
|
||||
// for a marker (`.specify` state dir OR `packages/casan-harness`), so the API reads the
|
||||
// real harness telemetry whether launched from the repo, a workspace subdir, or a bundle.
|
||||
// Every path is overridable by the same CASAN_DASHBOARD_* env vars the legacy
|
||||
// dashboard-server.py honors, so console + dashboard read identical sources.
|
||||
// Canonical CASAN_TELEMETRY_* variables are shared with harness writers; legacy
|
||||
// dashboard/control-panel names remain accepted so upgrades do not split sources.
|
||||
import { existsSync, statSync } from 'node:fs';
|
||||
import { dirname, join, resolve } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
@@ -27,35 +27,74 @@ export const APP_ROOT = process.env.CASAN_APP_ROOT
|
||||
? resolve(process.env.CASAN_APP_ROOT)
|
||||
: findAppRoot(HERE);
|
||||
|
||||
const env = (k: string, fallback: string) => (process.env[k] ? resolve(process.env[k]!) : join(APP_ROOT, fallback));
|
||||
const env = (keys: string[], fallback: string) => {
|
||||
const configured = keys.map((key) => process.env[key]).find((value): value is string => Boolean(value));
|
||||
return configured ? resolve(configured) : join(APP_ROOT, fallback);
|
||||
};
|
||||
|
||||
// Runtime telemetry (regenerated) + generated reports. Names match the harness layout.
|
||||
export const PATHS = {
|
||||
metrics: env('CASAN_DASHBOARD_METRICS', '.specify/logs/cost/metrics.jsonl'),
|
||||
providerUsage: env('CASAN_CP_PROVIDER_USAGE', '.specify/logs/level5/provider-usage.jsonl'),
|
||||
audit: env('CASAN_CP_AUDIT', '.specify/logs/audit/audit.jsonl'),
|
||||
auditHead: env('CASAN_CP_AUDIT_HEAD', '.specify/logs/audit/audit-head.txt'),
|
||||
security: env('CASAN_CP_SECURITY', '.specify/logs/audit/security.jsonl'),
|
||||
toolRegistry: env('CASAN_CP_TOOL_REGISTRY', '.specify/logs/level5/tool-registry.jsonl'),
|
||||
actionGate: env('CASAN_CP_ACTION_GATE', '.specify/logs/level5/action-gate.jsonl'),
|
||||
fallback: env('CASAN_CP_FALLBACK', '.specify/logs/level5/fallback.jsonl'),
|
||||
incidents: env('CASAN_CP_INCIDENTS', '.specify/logs/level5/incidents.jsonl'),
|
||||
alerts: env('CASAN_DASHBOARD_ALERTS', '.specify/agentops/alerts.log'),
|
||||
traceDir: env('CASAN_CP_TRACE_DIR', '.specify/logs/trace'),
|
||||
traceEventDir: env('CASAN_CP_TRACE_EVENT_DIR', '.specify/logs/trace-events'),
|
||||
traceability: env('CASAN_CP_TRACEABILITY', 'docs/output/casan/traceability-matrix.json'),
|
||||
drift: env('CASAN_CP_DRIFT', 'docs/output/casan/level5-evidence/09-drift-report.json'),
|
||||
businessKpi: env('CASAN_CP_KPI', 'docs/output/casan/level5-evidence/14-business-kpi-report.json'),
|
||||
benignFp: env('CASAN_CP_BENIGN_FP', 'docs/output/casan/benign-fp-report.json'),
|
||||
scoringReport: env('CASAN_CP_SCORING_REPORT', 'docs/output/casan/phase3-real-run-scoring.md'),
|
||||
approvalInbox: env('CASAN_CP_APPROVAL_INBOX', '.specify/level5/approval-inbox.json'),
|
||||
chatAudit: env('CASAN_CP_CHAT_AUDIT', '.specify/logs/chat/chat-turns.jsonl'),
|
||||
delegationPolicy: env('CASAN_CP_DELEGATION_POLICY', 'packages/casan-harness/config/delegation-policy.yaml'),
|
||||
selfImprove: env('CASAN_CP_SELF_IMPROVE', 'packages/casan-harness/scripts/bash/self-improve.py'),
|
||||
evidencePacks: env('CASAN_CP_EVIDENCE_PACKS', 'docs/output/casan/evidence-packs'),
|
||||
metrics: env(['CASAN_TELEMETRY_METRICS_LOG', 'CASAN_DASHBOARD_METRICS'], '.specify/logs/cost/metrics.jsonl'),
|
||||
providerUsage: env(['CASAN_TELEMETRY_PROVIDER_LOG', 'CASAN_CP_PROVIDER_USAGE'], '.specify/logs/level5/provider-usage.jsonl'),
|
||||
alerts: env(['CASAN_TELEMETRY_ALERTS_LOG', 'CASAN_DASHBOARD_ALERTS'], '.specify/agentops/alerts.log'),
|
||||
audit: env(['CASAN_CP_AUDIT'], '.specify/logs/audit/audit.jsonl'),
|
||||
auditHead: env(['CASAN_CP_AUDIT_HEAD'], '.specify/logs/audit/audit-head.txt'),
|
||||
security: env(['CASAN_CP_SECURITY'], '.specify/logs/audit/security.jsonl'),
|
||||
toolRegistry: env(['CASAN_CP_TOOL_REGISTRY'], '.specify/logs/level5/tool-registry.jsonl'),
|
||||
actionGate: env(['CASAN_CP_ACTION_GATE'], '.specify/logs/level5/action-gate.jsonl'),
|
||||
fallback: env(['CASAN_CP_FALLBACK'], '.specify/logs/level5/fallback.jsonl'),
|
||||
incidents: env(['CASAN_CP_INCIDENTS'], '.specify/logs/level5/incidents.jsonl'),
|
||||
traceDir: env(['CASAN_CP_TRACE_DIR'], '.specify/logs/trace'),
|
||||
traceEventDir: env(['CASAN_CP_TRACE_EVENT_DIR'], '.specify/logs/trace-events'),
|
||||
traceability: env(['CASAN_CP_TRACEABILITY'], 'docs/output/casan/traceability-matrix.json'),
|
||||
drift: env(['CASAN_CP_DRIFT'], 'docs/output/casan/level5-evidence/09-drift-report.json'),
|
||||
businessKpi: env(['CASAN_CP_KPI'], 'docs/output/casan/level5-evidence/14-business-kpi-report.json'),
|
||||
benignFp: env(['CASAN_CP_BENIGN_FP'], 'docs/output/casan/benign-fp-report.json'),
|
||||
scoringReport: env(['CASAN_CP_SCORING_REPORT'], 'docs/output/casan/phase3-real-run-scoring.md'),
|
||||
approvalInbox: env(['CASAN_CP_APPROVAL_INBOX'], '.specify/level5/approval-inbox.json'),
|
||||
chatAudit: env(['CASAN_CP_CHAT_AUDIT'], '.specify/logs/chat/chat-turns.jsonl'),
|
||||
delegationPolicy: env(['CASAN_CP_DELEGATION_POLICY'], 'packages/casan-harness/config/delegation-policy.yaml'),
|
||||
selfImprove: env(['CASAN_CP_SELF_IMPROVE'], 'packages/casan-harness/scripts/bash/self-improve.py'),
|
||||
evidencePacks: env(['CASAN_CP_EVIDENCE_PACKS'], 'docs/output/casan/evidence-packs'),
|
||||
};
|
||||
|
||||
export const STALE_AFTER_S = Number(process.env.CASAN_DASHBOARD_STALE_S ?? 3600);
|
||||
const configuredStaleAfter = Number(process.env.CASAN_DASHBOARD_STALE_S ?? 3600);
|
||||
export const STALE_AFTER_S = Number.isFinite(configuredStaleAfter) && configuredStaleAfter >= 0
|
||||
? configuredStaleAfter
|
||||
: 3600;
|
||||
|
||||
export interface SourceFreshness {
|
||||
source: 'metrics' | 'provider' | 'alerts';
|
||||
path: string;
|
||||
present: boolean;
|
||||
updated_at: string | null;
|
||||
age_s: number | null;
|
||||
stale: boolean;
|
||||
required: boolean;
|
||||
}
|
||||
|
||||
export function sourceFreshness(
|
||||
source: SourceFreshness['source'],
|
||||
path: string,
|
||||
required: boolean,
|
||||
nowMs = Date.now(),
|
||||
): SourceFreshness {
|
||||
try {
|
||||
const mtime = statSync(path).mtime;
|
||||
const age = Math.max(0, Math.floor((nowMs - mtime.getTime()) / 1000));
|
||||
return { source, path, present: true, updated_at: mtime.toISOString(), age_s: age, stale: age > STALE_AFTER_S, required };
|
||||
} catch {
|
||||
return { source, path, present: false, updated_at: null, age_s: null, stale: required, required };
|
||||
}
|
||||
}
|
||||
|
||||
export function telemetryFreshness(nowMs = Date.now()): SourceFreshness[] {
|
||||
return [
|
||||
sourceFreshness('metrics', PATHS.metrics, true, nowMs),
|
||||
sourceFreshness('provider', PATHS.providerUsage, false, nowMs),
|
||||
sourceFreshness('alerts', PATHS.alerts, false, nowMs),
|
||||
];
|
||||
}
|
||||
|
||||
// Freshness of the primary metrics feed (mtime), used by /healthz + `stale` flags.
|
||||
export function metricsAgeSeconds(): number | null {
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
import type { H6Breakdown, H6Report } from './h6-report.js';
|
||||
|
||||
const escapeHtml = (value: unknown): string => String(value ?? '')
|
||||
.replaceAll('&', '&')
|
||||
.replaceAll('<', '<')
|
||||
.replaceAll('>', '>')
|
||||
.replaceAll('"', '"')
|
||||
.replaceAll("'", ''');
|
||||
|
||||
const number = (value: number): string => new Intl.NumberFormat('en-US').format(value);
|
||||
const money = (value: number): string => `$${value.toFixed(6)}`;
|
||||
|
||||
function breakdownRows(rows: H6Breakdown[]): string {
|
||||
if (rows.length === 0) return '<tr><td colspan="6" class="empty">No records in the selected scope.</td></tr>';
|
||||
return rows.map((row) => `<tr>
|
||||
<td>${escapeHtml(row.key)}</td><td>${number(row.runs)}</td><td>${number(row.failures)}</td>
|
||||
<td>${number(row.latency_avg_ms)} ms</td><td>${number(row.tokens)}</td><td>${money(row.cost_usd)}</td>
|
||||
</tr>`).join('');
|
||||
}
|
||||
|
||||
export function renderH6ReportHtml(report: H6Report): string {
|
||||
const verdictClass = report.verdict === 'pass' ? 'pass' : report.verdict === 'fail' ? 'fail' : report.verdict === 'attention' ? 'attention' : 'neutral';
|
||||
const scope = [
|
||||
report.scope.project ? `Project: ${report.scope.project}` : 'All projects',
|
||||
report.scope.run ? `Run: ${report.scope.run}` : 'All runs',
|
||||
report.scope.from ? `From: ${report.scope.from}` : null,
|
||||
report.scope.to ? `To: ${report.scope.to}` : null,
|
||||
].filter((value): value is string => Boolean(value));
|
||||
const findings = report.findings.length
|
||||
? report.findings.map((finding) => `<li class="${finding.severity}"><strong>${escapeHtml(finding.code)}</strong><span>${escapeHtml(finding.message)}</span></li>`).join('')
|
||||
: '<li class="info"><strong>NO_FINDINGS</strong><span>No threshold breach was detected in this scope.</span></li>';
|
||||
const sourceRows = report.evidence_sources.map((source) => `<tr>
|
||||
<td>${escapeHtml(source.source)}</td><td>${source.present ? 'present' : 'missing'}</td><td>${source.stale ? 'stale' : 'fresh'}</td>
|
||||
<td>${source.age_s === null ? '—' : `${number(source.age_s)} s`}</td><td>${number(source.records)}</td><td><code>${escapeHtml(source.path)}</code></td>
|
||||
</tr>`).join('');
|
||||
const warnings = report.data_quality.warnings.length
|
||||
? `<ul>${report.data_quality.warnings.map((warning) => `<li>${escapeHtml(warning)}</li>`).join('')}</ul>`
|
||||
: '<p>No data-quality warning.</p>';
|
||||
|
||||
return `<!doctype html>
|
||||
<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<title>${escapeHtml(report.title)} · ${escapeHtml(report.report_id)}</title>
|
||||
<style>
|
||||
:root{color-scheme:light;--ink:#172033;--muted:#64748b;--line:#dbe2ec;--paper:#fff;--wash:#f4f6fa;--indigo:#4f46e5;--teal:#0f766e}
|
||||
*{box-sizing:border-box}body{margin:0;background:var(--wash);color:var(--ink);font-family:Inter,ui-sans-serif,system-ui,-apple-system,"Segoe UI",sans-serif;line-height:1.45}
|
||||
main{max-width:1180px;margin:0 auto;padding:42px 28px 72px}.hero{position:relative;overflow:hidden;border:1px solid #cbd5e1;border-radius:20px;background:#111827;color:#f8fafc;padding:30px;box-shadow:0 24px 50px rgba(15,23,42,.14)}
|
||||
.hero:after{content:"";position:absolute;right:-90px;top:-110px;width:300px;height:300px;border-radius:999px;background:radial-gradient(circle,#6366f1 0,transparent 68%);opacity:.5}.eyebrow{font-size:11px;text-transform:uppercase;letter-spacing:.18em;color:#a5b4fc;font-weight:800}h1{font-size:34px;line-height:1.1;margin:8px 0}.scope{display:flex;gap:8px;flex-wrap:wrap;color:#cbd5e1;font-size:12px}.scope span{border:1px solid #334155;border-radius:999px;padding:6px 10px;background:#0f172a}.verdict{position:relative;z-index:1;display:inline-flex;margin-top:22px;border-radius:999px;padding:9px 14px;font-size:12px;text-transform:uppercase;letter-spacing:.12em;font-weight:900}.verdict.pass{background:#d1fae5;color:#065f46}.verdict.fail{background:#fee2e2;color:#991b1b}.verdict.attention{background:#fef3c7;color:#92400e}.verdict.neutral{background:#e2e8f0;color:#334155}
|
||||
.meta{margin-top:10px;color:#94a3b8;font-size:12px}.grid{display:grid;grid-template-columns:repeat(4,minmax(0,1fr));gap:12px;margin-top:18px}.card,.section{background:var(--paper);border:1px solid var(--line);border-radius:16px;box-shadow:0 5px 14px rgba(15,23,42,.04)}.card{padding:17px}.card span{display:block;color:var(--muted);font-size:11px;text-transform:uppercase;letter-spacing:.1em;font-weight:700}.card strong{display:block;margin-top:6px;font-size:26px}.section{padding:22px;margin-top:18px}h2{font-size:16px;margin:0 0 14px}table{border-collapse:collapse;width:100%;font-size:12px}th{text-align:left;color:var(--muted);font-size:10px;text-transform:uppercase;letter-spacing:.1em;padding:9px;border-bottom:1px solid var(--line)}td{padding:10px 9px;border-bottom:1px solid #edf1f5;vertical-align:top}.empty{color:var(--muted);text-align:center;padding:24px}code{font-size:11px;word-break:break-all}.findings{list-style:none;padding:0;margin:0;display:grid;gap:9px}.findings li{display:grid;grid-template-columns:190px 1fr;gap:12px;border-left:4px solid #94a3b8;background:#f8fafc;padding:11px 13px;border-radius:8px}.findings li.warning{border-color:#f59e0b}.findings li.critical{border-color:#ef4444}.findings span{color:#475569}.quality{border-left:4px solid var(--indigo);background:#eef2ff;padding:14px 16px;border-radius:10px;font-size:13px}.quality p,.quality ul{margin:0}.quality ul{padding-left:18px}.footer{margin-top:24px;color:var(--muted);font-size:11px;text-align:center}
|
||||
@media(max-width:800px){main{padding:20px 14px 50px}.grid{grid-template-columns:repeat(2,minmax(0,1fr))}.hero{padding:22px}h1{font-size:28px}.table-wrap{overflow:auto}.findings li{grid-template-columns:1fr}}
|
||||
@media print{body{background:#fff}main{max-width:none;padding:0}.hero,.card,.section{box-shadow:none}.hero{background:#111827!important;-webkit-print-color-adjust:exact;print-color-adjust:exact}.section{break-inside:avoid}.footer{margin-top:10px}}
|
||||
</style></head><body><main>
|
||||
<section class="hero"><div class="eyebrow">CASAN assurance dossier · contract v${report.schema_version}</div><h1>${escapeHtml(report.title)}</h1><div class="scope">${scope.map((item) => `<span>${escapeHtml(item)}</span>`).join('')}</div><div class="verdict ${verdictClass}">${escapeHtml(report.verdict)}</div><div class="meta">Generated ${escapeHtml(report.generated_at)} · ${escapeHtml(report.report_id)} · Freshness ${escapeHtml(report.freshness.status)}</div></section>
|
||||
<section class="grid"><div class="card"><span>Governed runs</span><strong>${number(report.summary.runs)}</strong></div><div class="card"><span>Failure rate</span><strong>${report.summary.failure_rate_pct}%</strong></div><div class="card"><span>P95 latency</span><strong>${number(report.summary.latency_ms.p95)} ms</strong></div><div class="card"><span>Provider tokens</span><strong>${number(report.summary.tokens.provider_total)}</strong></div><div class="card"><span>Actual provider cost</span><strong>${money(report.summary.cost_usd.provider_actual)}</strong></div><div class="card"><span>Estimated cost</span><strong>${money(report.summary.cost_usd.estimated)}</strong></div><div class="card"><span>Alerts</span><strong>${number(report.summary.alerts)}</strong></div><div class="card"><span>Retries</span><strong>${number(report.summary.retries)}</strong></div></section>
|
||||
<section class="section"><h2>Verdict findings</h2><ul class="findings">${findings}</ul></section>
|
||||
<section class="section"><h2>Evidence freshness</h2><div class="table-wrap"><table><thead><tr><th>Source</th><th>Presence</th><th>State</th><th>Age</th><th>Records</th><th>Path</th></tr></thead><tbody>${sourceRows}</tbody></table></div></section>
|
||||
<section class="section"><h2>Step breakdown</h2><div class="table-wrap"><table><thead><tr><th>Step</th><th>Runs</th><th>Failures</th><th>Avg latency</th><th>Tokens</th><th>Cost</th></tr></thead><tbody>${breakdownRows(report.details.by_step)}</tbody></table></div></section>
|
||||
<section class="section"><h2>Provider/model breakdown</h2><div class="table-wrap"><table><thead><tr><th>Provider · model</th><th>Calls</th><th>Failures</th><th>Avg latency</th><th>Tokens</th><th>Cost</th></tr></thead><tbody>${breakdownRows(report.details.by_provider)}</tbody></table></div></section>
|
||||
<section class="section"><h2>Data quality · ${escapeHtml(report.data_quality.status)}</h2><div class="quality">${warnings}</div></section>
|
||||
<div class="footer">Generated from CASAN runtime evidence. No maturity score or telemetry value is hard-coded in this document.</div>
|
||||
</main></body></html>`;
|
||||
}
|
||||
@@ -0,0 +1,307 @@
|
||||
import { BadRequestException } from '@nestjs/common';
|
||||
import { basename, relative } from 'node:path';
|
||||
import { APP_ROOT, PATHS, STALE_AFTER_S, telemetryFreshness, type SourceFreshness } from '../common/app-root.js';
|
||||
import type { HarnessReport, HarnessReportEvidenceSource, HarnessReportFinding, HarnessReportScope } from './report.contract.js';
|
||||
|
||||
type TelemetryRow = Record<string, unknown>;
|
||||
|
||||
export interface H6ReportQuery {
|
||||
project: string | null;
|
||||
from: string | null;
|
||||
to: string | null;
|
||||
run: string | null;
|
||||
limit: number;
|
||||
}
|
||||
|
||||
export interface H6ReportQueryParams {
|
||||
project?: string;
|
||||
from?: string;
|
||||
to?: string;
|
||||
run?: string;
|
||||
limit?: string;
|
||||
}
|
||||
|
||||
export interface H6ReportInput {
|
||||
metrics: TelemetryRow[];
|
||||
provider: TelemetryRow[];
|
||||
alerts: TelemetryRow[];
|
||||
sourceFreshness: SourceFreshness[];
|
||||
now: Date;
|
||||
}
|
||||
|
||||
export interface H6ReportSummary {
|
||||
runs: number;
|
||||
success: number;
|
||||
failed: number;
|
||||
degraded: number;
|
||||
failure_rate_pct: number;
|
||||
retries: number;
|
||||
latency_ms: { average: number; p50: number; p95: number; p99: number; max: number };
|
||||
tokens: { input: number; output: number; total: number; provider_total: number };
|
||||
cost_usd: { provider_actual: number; estimated: number };
|
||||
provider_calls: number;
|
||||
alerts: number;
|
||||
}
|
||||
|
||||
export interface H6Breakdown {
|
||||
key: string;
|
||||
runs: number;
|
||||
failures: number;
|
||||
latency_avg_ms: number;
|
||||
tokens: number;
|
||||
cost_usd: number;
|
||||
}
|
||||
|
||||
export interface H6ReportDetails {
|
||||
by_status: Array<{ status: string; count: number }>;
|
||||
by_step: H6Breakdown[];
|
||||
by_provider: H6Breakdown[];
|
||||
by_cost_source: Array<{ source: string; records: number; cost_usd: number }>;
|
||||
by_alert: Array<{ alert: string; count: number }>;
|
||||
recent_runs: TelemetryRow[];
|
||||
recent_alerts: TelemetryRow[];
|
||||
}
|
||||
|
||||
export type H6Report = HarnessReport<H6ReportSummary, H6ReportDetails>;
|
||||
|
||||
const safeFilter = /^[a-zA-Z0-9][a-zA-Z0-9._:-]{0,127}$/;
|
||||
const numberValue = (value: unknown): number => typeof value === 'number' && Number.isFinite(value) ? value : 0;
|
||||
const stringValue = (value: unknown, fallback: string): string => typeof value === 'string' && value.trim() ? value.trim() : fallback;
|
||||
const round = (value: number, digits = 2): number => Number(value.toFixed(digits));
|
||||
const configuredThreshold = (name: string, fallback: number): number => {
|
||||
const parsed = Number(process.env[name] ?? fallback);
|
||||
return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback;
|
||||
};
|
||||
const rowProject = (row: TelemetryRow): string => stringValue(row.project ?? row.project_id, 'default');
|
||||
const rowRun = (row: TelemetryRow): string => stringValue(row.run_id ?? row.trace_id, 'unattributed');
|
||||
const rowTime = (row: TelemetryRow): number | null => {
|
||||
const timestamp = typeof row.timestamp === 'string' ? Date.parse(row.timestamp) : Number.NaN;
|
||||
return Number.isFinite(timestamp) ? timestamp : null;
|
||||
};
|
||||
|
||||
function parseDate(value: string | undefined, label: 'from' | 'to'): string | null {
|
||||
if (!value) return null;
|
||||
const dateOnly = /^\d{4}-\d{2}-\d{2}$/.test(value);
|
||||
const normalized = dateOnly ? `${value}T${label === 'from' ? '00:00:00.000' : '23:59:59.999'}Z` : value;
|
||||
const parsed = Date.parse(normalized);
|
||||
if (!Number.isFinite(parsed)) throw new BadRequestException(`H6_REPORT_INVALID_${label.toUpperCase()}`);
|
||||
return new Date(parsed).toISOString();
|
||||
}
|
||||
|
||||
function parseFilter(value: string | undefined, label: 'project' | 'run'): string | null {
|
||||
if (!value) return null;
|
||||
if (!safeFilter.test(value)) throw new BadRequestException(`H6_REPORT_INVALID_${label.toUpperCase()}`);
|
||||
return value;
|
||||
}
|
||||
|
||||
export function parseH6ReportQuery(raw: H6ReportQueryParams): H6ReportQuery {
|
||||
const from = parseDate(raw.from, 'from');
|
||||
const to = parseDate(raw.to, 'to');
|
||||
if (from && to && Date.parse(from) > Date.parse(to)) throw new BadRequestException('H6_REPORT_INVALID_TIME_RANGE');
|
||||
const parsedLimit = Number(raw.limit ?? 50);
|
||||
if (!Number.isInteger(parsedLimit) || parsedLimit < 1 || parsedLimit > 200) {
|
||||
throw new BadRequestException('H6_REPORT_INVALID_LIMIT');
|
||||
}
|
||||
return {
|
||||
project: parseFilter(raw.project, 'project'),
|
||||
from,
|
||||
to,
|
||||
run: parseFilter(raw.run, 'run'),
|
||||
limit: parsedLimit,
|
||||
};
|
||||
}
|
||||
|
||||
function filtered(rows: TelemetryRow[], query: H6ReportQuery): TelemetryRow[] {
|
||||
const fromMs = query.from ? Date.parse(query.from) : null;
|
||||
const toMs = query.to ? Date.parse(query.to) : null;
|
||||
return rows.filter((row) => {
|
||||
if (query.project && rowProject(row) !== query.project) return false;
|
||||
if (query.run && rowRun(row) !== query.run) return false;
|
||||
const timestamp = rowTime(row);
|
||||
if (fromMs !== null && (timestamp === null || timestamp < fromMs)) return false;
|
||||
if (toMs !== null && (timestamp === null || timestamp > toMs)) return false;
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
function percentile(values: number[], percentileValue: number): number {
|
||||
if (values.length === 0) return 0;
|
||||
const sorted = [...values].sort((left, right) => left - right);
|
||||
const index = Math.max(0, Math.ceil((percentileValue / 100) * sorted.length) - 1);
|
||||
return sorted[index];
|
||||
}
|
||||
|
||||
function sum(rows: TelemetryRow[], key: string): number {
|
||||
return rows.reduce((total, row) => total + numberValue(row[key]), 0);
|
||||
}
|
||||
|
||||
function grouped(rows: TelemetryRow[], keyOf: (row: TelemetryRow) => string): H6Breakdown[] {
|
||||
const groups = new Map<string, TelemetryRow[]>();
|
||||
for (const row of rows) {
|
||||
const key = keyOf(row);
|
||||
groups.set(key, [...(groups.get(key) ?? []), row]);
|
||||
}
|
||||
return [...groups.entries()].map(([key, records]) => {
|
||||
const latencies = records.map((row) => numberValue(row.latency_ms)).filter((value) => value > 0);
|
||||
return {
|
||||
key,
|
||||
runs: records.length,
|
||||
failures: records.filter((row) => stringValue(row.status, 'unknown') === 'failed').length,
|
||||
latency_avg_ms: latencies.length ? Math.round(latencies.reduce((total, value) => total + value, 0) / latencies.length) : 0,
|
||||
tokens: sum(records, 'total_tokens'),
|
||||
cost_usd: round(sum(records, records.some((row) => row.cost_usd !== undefined) ? 'cost_usd' : 'cost_estimate'), 6),
|
||||
};
|
||||
}).sort((left, right) => right.runs - left.runs || left.key.localeCompare(right.key));
|
||||
}
|
||||
|
||||
function counted(rows: TelemetryRow[], keyOf: (row: TelemetryRow) => string): Array<{ key: string; count: number }> {
|
||||
const counts = new Map<string, number>();
|
||||
for (const row of rows) {
|
||||
const key = keyOf(row);
|
||||
counts.set(key, (counts.get(key) ?? 0) + 1);
|
||||
}
|
||||
return [...counts.entries()].map(([key, countValue]) => ({ key, count: countValue }))
|
||||
.sort((left, right) => right.count - left.count || left.key.localeCompare(right.key));
|
||||
}
|
||||
|
||||
function alertTypes(row: TelemetryRow): string[] {
|
||||
const structured = row.body && typeof row.body === 'object' ? (row.body as Record<string, unknown>)['alert.type'] : undefined;
|
||||
if (typeof structured === 'string') return [structured];
|
||||
if (Array.isArray(row.alerts)) return row.alerts.filter((value): value is string => typeof value === 'string');
|
||||
return [];
|
||||
}
|
||||
|
||||
function evidenceSources(input: H6ReportInput): HarnessReportEvidenceSource[] {
|
||||
const counts: Record<SourceFreshness['source'], number> = {
|
||||
metrics: input.metrics.length,
|
||||
provider: input.provider.length,
|
||||
alerts: input.alerts.length,
|
||||
};
|
||||
return input.sourceFreshness.map((source) => {
|
||||
const appRelativePath = relative(APP_ROOT, source.path) || '.';
|
||||
return {
|
||||
...source,
|
||||
path: appRelativePath.startsWith('..') ? `[external]/${basename(source.path)}` : appRelativePath,
|
||||
records: counts[source.source],
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export function buildH6Report(input: H6ReportInput, query: H6ReportQuery): H6Report {
|
||||
const metrics = filtered(input.metrics, query);
|
||||
const provider = filtered(input.provider, query);
|
||||
const alerts = filtered(input.alerts, query);
|
||||
const latencies = metrics.map((row) => numberValue(row.latency_ms)).filter((value) => value > 0);
|
||||
const failed = metrics.filter((row) => stringValue(row.status, 'unknown') === 'failed').length;
|
||||
const degraded = metrics.filter((row) => stringValue(row.status, 'unknown') === 'degraded').length;
|
||||
const success = metrics.filter((row) => ['success', 'pass', 'passed'].includes(stringValue(row.status, 'unknown'))).length;
|
||||
const sourceEvidence = evidenceSources(input);
|
||||
const primary = sourceEvidence.find((source) => source.source === 'metrics');
|
||||
const failureRate = metrics.length ? round((failed / metrics.length) * 100, 1) : 0;
|
||||
const p95 = percentile(latencies, 95);
|
||||
const failureThreshold = configuredThreshold('CASAN_H6_FAILURE_RATE_THRESHOLD_PCT', 5);
|
||||
const p95Threshold = configuredThreshold('CASAN_H6_P95_LATENCY_THRESHOLD_MS', 5000);
|
||||
const findings: HarnessReportFinding[] = [];
|
||||
const warnings: string[] = [];
|
||||
|
||||
if (!primary?.present) findings.push({ severity: 'critical', code: 'METRICS_MISSING', message: 'The required runtime metrics source is missing.' });
|
||||
else if (primary.stale) findings.push({ severity: 'warning', code: 'METRICS_STALE', message: 'The primary runtime metrics source is stale.', metric: 'age_s', value: primary.age_s ?? 'unknown', threshold: STALE_AFTER_S });
|
||||
if (failureRate > failureThreshold) findings.push({ severity: 'critical', code: 'FAILURE_RATE_BREACH', message: 'Failure rate exceeds the configured H6 threshold.', metric: 'failure_rate_pct', value: failureRate, threshold: failureThreshold });
|
||||
else if (failed > 0) findings.push({ severity: 'warning', code: 'FAILURES_PRESENT', message: `${failed} failed run(s) are present in the selected scope.`, metric: 'failed', value: failed });
|
||||
if (p95 > p95Threshold) findings.push({ severity: 'warning', code: 'P95_LATENCY_BREACH', message: 'P95 latency exceeds the configured H6 threshold.', metric: 'p95_latency_ms', value: p95, threshold: p95Threshold });
|
||||
const alertFingerprints = new Map<string, Set<string>>();
|
||||
for (const row of [...metrics, ...alerts]) {
|
||||
for (const alert of alertTypes(row)) {
|
||||
const run = rowRun(row);
|
||||
const fingerprint = `${run === 'unattributed' ? stringValue(row.timestamp, 'unknown-time') : run}:${alert}`;
|
||||
const known = alertFingerprints.get(alert) ?? new Set<string>();
|
||||
known.add(fingerprint);
|
||||
alertFingerprints.set(alert, known);
|
||||
}
|
||||
}
|
||||
const alertCounts = [...alertFingerprints.entries()].map(([alert, fingerprints]) => ({ alert, count: fingerprints.size }));
|
||||
const alertCount = alertCounts.reduce((total, entry) => total + entry.count, 0);
|
||||
if (alertCount > 0) findings.push({ severity: 'warning', code: 'ALERTS_PRESENT', message: `${alertCount} alert signal(s) require review.`, metric: 'alerts', value: alertCount });
|
||||
if (degraded > 0) findings.push({ severity: 'warning', code: 'DEGRADED_RUNS_PRESENT', message: `${degraded} degraded run(s) are present in the selected scope.`, metric: 'degraded', value: degraded });
|
||||
if (provider.length === 0) warnings.push('No provider usage records matched the selected scope; token and actual-cost breakdown may be incomplete.');
|
||||
for (const source of sourceEvidence) {
|
||||
if (source.source === 'metrics') continue;
|
||||
if (!source.present) warnings.push(`Optional ${source.source} telemetry source is missing; its breakdown is unavailable.`);
|
||||
else if (source.stale) warnings.push(`Optional ${source.source} telemetry source is stale; its breakdown may not reflect recent activity.`);
|
||||
}
|
||||
const estimatedRecords = metrics.filter((row) => stringValue(row.cost_source, 'unknown').includes('estimate')).length;
|
||||
if (estimatedRecords > 0) warnings.push(`${estimatedRecords} metric record(s) use estimated rather than provider-reported cost.`);
|
||||
const unattributed = [...input.metrics, ...input.provider].filter((row) => row.project === undefined && row.project_id === undefined).length;
|
||||
if (unattributed > 0) warnings.push(`${unattributed} source record(s) have no project attribute and are classified as project "default".`);
|
||||
|
||||
const critical = findings.some((finding) => finding.severity === 'critical');
|
||||
const warning = findings.some((finding) => finding.severity === 'warning');
|
||||
const verdict = metrics.length === 0 && provider.length === 0 ? 'no_data' : critical ? 'fail' : warning ? 'attention' : 'pass';
|
||||
const freshnessStatus = !primary?.present ? 'missing' : primary.stale ? 'stale' : 'live';
|
||||
const availableProjects = [...new Set([...input.metrics, ...input.provider, ...input.alerts].map(rowProject))].sort();
|
||||
const availableRuns = [...new Set([...input.metrics, ...input.provider, ...input.alerts].map(rowRun).filter((run) => run !== 'unattributed'))].sort();
|
||||
const costSources = grouped(metrics, (row) => stringValue(row.cost_source, 'unknown')).map((entry) => ({
|
||||
source: entry.key,
|
||||
records: entry.runs,
|
||||
cost_usd: entry.cost_usd,
|
||||
}));
|
||||
const scope: HarnessReportScope = { project: query.project, from: query.from, to: query.to, run: query.run };
|
||||
|
||||
return {
|
||||
schema_version: 1,
|
||||
report_id: `H6-${input.now.toISOString().replace(/[-:.TZ]/g, '').slice(0, 14)}`,
|
||||
harness: 'H6',
|
||||
title: 'H6 · AgentOps Report',
|
||||
description: 'Evidence-backed runtime, token, cost, failure, retry and alert telemetry.',
|
||||
generated_at: input.now.toISOString(),
|
||||
scope,
|
||||
verdict,
|
||||
verdict_reasons: findings.map((finding) => finding.code),
|
||||
freshness: { status: freshnessStatus, stale_after_s: STALE_AFTER_S, primary_age_s: primary?.age_s ?? null, sources: sourceEvidence },
|
||||
summary: {
|
||||
runs: metrics.length,
|
||||
success,
|
||||
failed,
|
||||
degraded,
|
||||
failure_rate_pct: failureRate,
|
||||
retries: sum(metrics, 'retry_count'),
|
||||
latency_ms: {
|
||||
average: latencies.length ? Math.round(latencies.reduce((total, value) => total + value, 0) / latencies.length) : 0,
|
||||
p50: percentile(latencies, 50),
|
||||
p95,
|
||||
p99: percentile(latencies, 99),
|
||||
max: latencies.length ? Math.max(...latencies) : 0,
|
||||
},
|
||||
tokens: { input: sum(metrics, 'input_tokens'), output: sum(metrics, 'output_tokens'), total: sum(metrics, 'total_tokens'), provider_total: sum(provider, 'total_tokens') },
|
||||
cost_usd: { provider_actual: round(sum(provider, 'cost_usd'), 6), estimated: round(sum(metrics, 'cost_estimate'), 6) },
|
||||
provider_calls: provider.length,
|
||||
alerts: alertCount,
|
||||
},
|
||||
thresholds: { failure_rate_pct: failureThreshold, p95_latency_ms: p95Threshold, freshness_age_s: STALE_AFTER_S },
|
||||
findings,
|
||||
evidence_sources: sourceEvidence,
|
||||
data_quality: { status: !primary?.present ? 'insufficient' : warnings.length ? 'partial' : 'complete', warnings },
|
||||
available_filters: { projects: availableProjects, runs: availableRuns },
|
||||
details: {
|
||||
by_status: counted(metrics, (row) => stringValue(row.status, 'unknown')).map(({ key, count: countValue }) => ({ status: key, count: countValue })),
|
||||
by_step: grouped(metrics, (row) => stringValue(row.step, 'unknown-step')),
|
||||
by_provider: grouped(provider, (row) => `${stringValue(row.provider, 'unknown-provider')} · ${stringValue(row.model, 'unknown-model')}`),
|
||||
by_cost_source: costSources,
|
||||
by_alert: alertCounts.sort((left, right) => right.count - left.count || left.alert.localeCompare(right.alert)),
|
||||
recent_runs: [...metrics].slice(-query.limit).reverse(),
|
||||
recent_alerts: [...alerts].slice(-query.limit).reverse(),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function defaultH6ReportInput(now = new Date()): H6ReportInput {
|
||||
return {
|
||||
metrics: [],
|
||||
provider: [],
|
||||
alerts: [],
|
||||
sourceFreshness: telemetryFreshness(now.getTime()),
|
||||
now,
|
||||
};
|
||||
}
|
||||
|
||||
export const H6_REPORT_PATHS = PATHS;
|
||||
@@ -0,0 +1,83 @@
|
||||
import type { SourceFreshness } from '../common/app-root.js';
|
||||
|
||||
export const HARNESS_REPORT_DEFINITIONS = [
|
||||
{ id: 'H1', slug: 'context', title: 'H1 · Context', description: 'Context selection, prompt contract and risk classification' },
|
||||
{ id: 'H2', slug: 'tool', title: 'H2 · Tool', description: 'Allowlisted tools, actions and execution boundaries' },
|
||||
{ id: 'H3', slug: 'evaluation', title: 'H3 · Evaluation', description: 'Quality evaluation, traceability and acceptance evidence' },
|
||||
{ id: 'H4', slug: 'security', title: 'H4 · Security', description: 'Input, output and artifact security controls' },
|
||||
{ id: 'H5', slug: 'governance', title: 'H5 · Governance', description: 'Policy decisions, approvals and audit integrity' },
|
||||
{ id: 'H6', slug: 'agentops', title: 'H6 · AgentOps', description: 'Runtime, token, cost, failure and alert telemetry' },
|
||||
{ id: 'H7', slug: 'orchestration', title: 'H7 · Orchestration', description: 'Final outcome, certification and rollback evidence' },
|
||||
] as const;
|
||||
|
||||
export type HarnessReportId = typeof HARNESS_REPORT_DEFINITIONS[number]['id'];
|
||||
export type HarnessReportVerdict = 'pass' | 'attention' | 'fail' | 'no_data';
|
||||
export type ReportExportFormat = 'json' | 'html';
|
||||
|
||||
export interface HarnessReportScope {
|
||||
project: string | null;
|
||||
from: string | null;
|
||||
to: string | null;
|
||||
run: string | null;
|
||||
}
|
||||
|
||||
export interface HarnessReportFinding {
|
||||
severity: 'info' | 'warning' | 'critical';
|
||||
code: string;
|
||||
message: string;
|
||||
metric?: string;
|
||||
value?: number | string;
|
||||
threshold?: number | string;
|
||||
}
|
||||
|
||||
export interface HarnessReportEvidenceSource extends SourceFreshness {
|
||||
records: number;
|
||||
}
|
||||
|
||||
export interface HarnessReport<TSummary, TDetails> {
|
||||
schema_version: 1;
|
||||
report_id: string;
|
||||
harness: HarnessReportId;
|
||||
title: string;
|
||||
description: string;
|
||||
generated_at: string;
|
||||
scope: HarnessReportScope;
|
||||
verdict: HarnessReportVerdict;
|
||||
verdict_reasons: string[];
|
||||
freshness: {
|
||||
status: 'live' | 'stale' | 'missing';
|
||||
stale_after_s: number;
|
||||
primary_age_s: number | null;
|
||||
sources: HarnessReportEvidenceSource[];
|
||||
};
|
||||
summary: TSummary;
|
||||
thresholds: Record<string, number | string | boolean>;
|
||||
findings: HarnessReportFinding[];
|
||||
evidence_sources: HarnessReportEvidenceSource[];
|
||||
data_quality: {
|
||||
status: 'complete' | 'partial' | 'insufficient';
|
||||
warnings: string[];
|
||||
};
|
||||
available_filters: {
|
||||
projects: string[];
|
||||
runs: string[];
|
||||
};
|
||||
details: TDetails;
|
||||
}
|
||||
|
||||
export interface HarnessReportCatalogEntry {
|
||||
id: HarnessReportId;
|
||||
slug: string;
|
||||
title: string;
|
||||
description: string;
|
||||
contract_version: 1;
|
||||
endpoint: string;
|
||||
availability: 'implemented' | 'contract_ready';
|
||||
}
|
||||
|
||||
export const HARNESS_REPORT_CATALOG: HarnessReportCatalogEntry[] = HARNESS_REPORT_DEFINITIONS.map((definition) => ({
|
||||
...definition,
|
||||
contract_version: 1,
|
||||
endpoint: `/api/v1/reports/${definition.id.toLowerCase()}`,
|
||||
availability: definition.id === 'H6' ? 'implemented' : 'contract_ready',
|
||||
}));
|
||||
@@ -0,0 +1,37 @@
|
||||
import { BadRequestException, Controller, Get, Header, Inject, Query, Res } from '@nestjs/common';
|
||||
import type { Response } from 'express';
|
||||
import { ok } from '../common/api-response.js';
|
||||
import { parseH6ReportQuery, type H6ReportQueryParams } from './h6-report.js';
|
||||
import { ReportsService } from './reports.service.js';
|
||||
|
||||
interface H6QueryParams extends H6ReportQueryParams {
|
||||
format?: string;
|
||||
}
|
||||
|
||||
@Controller('api/v1/reports')
|
||||
export class ReportsController {
|
||||
constructor(@Inject(ReportsService) private readonly reports: ReportsService) {}
|
||||
|
||||
@Get()
|
||||
catalog() {
|
||||
return ok(this.reports.catalog());
|
||||
}
|
||||
|
||||
@Get('h6')
|
||||
h6(@Query() raw: H6QueryParams) {
|
||||
return ok(this.reports.h6(parseH6ReportQuery(raw)));
|
||||
}
|
||||
|
||||
@Get('h6/export')
|
||||
@Header('Cache-Control', 'no-store')
|
||||
exportH6(@Query() raw: H6QueryParams, @Res() response: Response) {
|
||||
const format = raw.format ?? 'json';
|
||||
if (format !== 'json' && format !== 'html') throw new BadRequestException('H6_REPORT_INVALID_FORMAT');
|
||||
const report = this.reports.h6(parseH6ReportQuery(raw));
|
||||
const body = this.reports.serializeH6(report, format);
|
||||
const stamp = report.generated_at.slice(0, 10);
|
||||
response.type(format === 'html' ? 'text/html; charset=utf-8' : 'application/json; charset=utf-8');
|
||||
response.setHeader('Content-Disposition', `attachment; filename="casan-h6-report-${stamp}.${format}"`);
|
||||
response.send(body);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { ReportsController } from './reports.controller.js';
|
||||
import { ReportsService } from './reports.service.js';
|
||||
|
||||
@Module({
|
||||
controllers: [ReportsController],
|
||||
providers: [ReportsService],
|
||||
exports: [ReportsService],
|
||||
})
|
||||
export class ReportsModule {}
|
||||
@@ -0,0 +1,28 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { PATHS, telemetryFreshness } from '../common/app-root.js';
|
||||
import { readJsonl } from '../telemetry/telemetry.reader.js';
|
||||
import { buildH6Report, type H6Report, type H6ReportQuery } from './h6-report.js';
|
||||
import { renderH6ReportHtml } from './h6-report.html.js';
|
||||
import { HARNESS_REPORT_CATALOG } from './report.contract.js';
|
||||
|
||||
@Injectable()
|
||||
export class ReportsService {
|
||||
catalog() {
|
||||
return { schema_version: 1, reports: HARNESS_REPORT_CATALOG };
|
||||
}
|
||||
|
||||
h6(query: H6ReportQuery): H6Report {
|
||||
const now = new Date();
|
||||
return buildH6Report({
|
||||
metrics: readJsonl(PATHS.metrics),
|
||||
provider: readJsonl(PATHS.providerUsage),
|
||||
alerts: readJsonl(PATHS.alerts),
|
||||
sourceFreshness: telemetryFreshness(now.getTime()),
|
||||
now,
|
||||
}, query);
|
||||
}
|
||||
|
||||
serializeH6(report: H6Report, format: 'json' | 'html'): string {
|
||||
return format === 'html' ? renderH6ReportHtml(report) : `${JSON.stringify(report, null, 2)}\n`;
|
||||
}
|
||||
}
|
||||
@@ -5,7 +5,7 @@ import { existsSync, readFileSync, statSync } from 'node:fs';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { join, relative } from 'node:path';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { APP_ROOT, PATHS, isStale, metricsAgeSeconds, STALE_AFTER_S } from '../common/app-root.js';
|
||||
import { APP_ROOT, PATHS, isStale, metricsAgeSeconds, STALE_AFTER_S, telemetryFreshness } from '../common/app-root.js';
|
||||
import { readJsonl, readJson, readHead, readTrace } from './telemetry.reader.js';
|
||||
|
||||
type Row = Record<string, any>;
|
||||
@@ -217,7 +217,7 @@ export function buildChatLoopWidget(
|
||||
export class TelemetryService {
|
||||
private freshness() {
|
||||
const age = metricsAgeSeconds();
|
||||
return { stale: isStale(), age_s: age, stale_after_s: STALE_AFTER_S };
|
||||
return { stale: isStale(), age_s: age, stale_after_s: STALE_AFTER_S, sources: telemetryFreshness() };
|
||||
}
|
||||
|
||||
overview() {
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { BadRequestException } from '@nestjs/common';
|
||||
import { mkdtempSync, utimesSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { sourceFreshness } from '../src/common/app-root.js';
|
||||
import { buildH6Report, parseH6ReportQuery, type H6ReportInput } from '../src/reports/h6-report.js';
|
||||
import { renderH6ReportHtml } from '../src/reports/h6-report.html.js';
|
||||
import { HARNESS_REPORT_CATALOG } from '../src/reports/report.contract.js';
|
||||
|
||||
const NOW = new Date('2026-07-20T12:00:00.000Z');
|
||||
|
||||
function fixture(): H6ReportInput {
|
||||
return {
|
||||
now: NOW,
|
||||
metrics: [
|
||||
{ timestamp: '2026-07-19T08:00:00Z', trace_id: 'run-a', project: 'okr', status: 'success', step: 'generate', latency_ms: 100, input_tokens: 10, output_tokens: 20, total_tokens: 30, cost_estimate: 0.001, cost_source: 'word_count_estimate', retry_count: 0, alerts: [] },
|
||||
{ timestamp: '2026-07-19T09:00:00Z', trace_id: 'run-b', project: 'okr', status: 'failed', step: 'test', latency_ms: 9000, input_tokens: 20, output_tokens: 5, total_tokens: 25, cost_estimate: 0.002, cost_source: 'provider_telemetry', retry_count: 2, alerts: ['high-latency', 'execution-failed'] },
|
||||
{ timestamp: '2026-07-18T09:00:00Z', trace_id: 'run-c', project: 'desk', status: 'success', step: 'test', latency_ms: 250, total_tokens: 10, cost_estimate: 0.0001, cost_source: 'word_count_estimate' },
|
||||
],
|
||||
provider: [
|
||||
{ timestamp: '2026-07-19T08:00:00Z', run_id: 'run-a', project: 'okr', provider: 'ollama', model: 'ornith:9b', status: 'success', step: 'generate', latency_ms: 80, total_tokens: 28, cost_usd: 0 },
|
||||
{ timestamp: '2026-07-18T09:00:00Z', run_id: 'run-c', project: 'desk', provider: 'openai', model: 'gpt-test', status: 'success', step: 'test', latency_ms: 200, total_tokens: 10, cost_usd: 0.01 },
|
||||
],
|
||||
alerts: [
|
||||
{ timestamp: '2026-07-19T09:00:00Z', trace_id: 'run-b', project: 'okr', body: { 'alert.type': 'execution-failed' } },
|
||||
],
|
||||
sourceFreshness: [
|
||||
{ source: 'metrics', path: '/workspace/.specify/logs/cost/metrics.jsonl', present: true, updated_at: '2026-07-20T11:59:00Z', age_s: 60, stale: false, required: true },
|
||||
{ source: 'provider', path: '/workspace/.specify/logs/level5/provider-usage.jsonl', present: true, updated_at: '2026-07-20T11:58:00Z', age_s: 120, stale: false, required: false },
|
||||
{ source: 'alerts', path: '/workspace/.specify/agentops/alerts.log', present: true, updated_at: '2026-07-20T11:57:00Z', age_s: 180, stale: false, required: false },
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
test('H1-H7 share one versioned report catalog contract', () => {
|
||||
assert.deepEqual(HARNESS_REPORT_CATALOG.map((entry) => entry.id), ['H1', 'H2', 'H3', 'H4', 'H5', 'H6', 'H7']);
|
||||
assert.ok(HARNESS_REPORT_CATALOG.every((entry) => entry.contract_version === 1));
|
||||
assert.equal(HARNESS_REPORT_CATALOG.find((entry) => entry.id === 'H6')?.availability, 'implemented');
|
||||
});
|
||||
|
||||
test('telemetry freshness is source-specific and missing required data fails loud', () => {
|
||||
const directory = mkdtempSync(join(tmpdir(), 'casan-h6-'));
|
||||
const metricsPath = join(directory, 'metrics.jsonl');
|
||||
writeFileSync(metricsPath, '{}\n');
|
||||
const oldTime = new Date(NOW.getTime() - 7200 * 1000);
|
||||
utimesSync(metricsPath, oldTime, oldTime);
|
||||
const staleMetrics = sourceFreshness('metrics', metricsPath, true, NOW.getTime());
|
||||
const missingMetrics = sourceFreshness('metrics', join(directory, 'missing.jsonl'), true, NOW.getTime());
|
||||
assert.equal(staleMetrics.present, true);
|
||||
assert.equal(staleMetrics.age_s, 7200);
|
||||
assert.equal(staleMetrics.stale, true);
|
||||
assert.deepEqual(
|
||||
{ present: missingMetrics.present, stale: missingMetrics.stale, required: missingMetrics.required },
|
||||
{ present: false, stale: true, required: true },
|
||||
);
|
||||
});
|
||||
|
||||
test('H6 report filters project/time/run and aggregates measured evidence', () => {
|
||||
const query = parseH6ReportQuery({ project: 'okr', from: '2026-07-19', to: '2026-07-19' });
|
||||
const report = buildH6Report(fixture(), query);
|
||||
assert.equal(report.schema_version, 1);
|
||||
assert.equal(report.harness, 'H6');
|
||||
assert.equal(report.summary.runs, 2);
|
||||
assert.equal(report.summary.failed, 1);
|
||||
assert.equal(report.summary.retries, 2);
|
||||
assert.equal(report.summary.latency_ms.p50, 100);
|
||||
assert.equal(report.summary.latency_ms.p95, 9000);
|
||||
assert.equal(report.summary.tokens.provider_total, 28);
|
||||
assert.equal(report.summary.alerts, 2);
|
||||
assert.deepEqual(report.details.by_alert, [
|
||||
{ alert: 'execution-failed', count: 1 },
|
||||
{ alert: 'high-latency', count: 1 },
|
||||
]);
|
||||
assert.equal(report.available_filters.projects.length, 2);
|
||||
assert.equal(report.freshness.status, 'live');
|
||||
assert.equal(report.verdict, 'fail');
|
||||
assert.ok(report.findings.some((finding) => finding.code === 'FAILURE_RATE_BREACH'));
|
||||
});
|
||||
|
||||
test('H6 report supports exact run filtering and truthful no-data verdict', () => {
|
||||
const selected = buildH6Report(fixture(), parseH6ReportQuery({ run: 'run-a' }));
|
||||
assert.equal(selected.summary.runs, 1);
|
||||
assert.equal(selected.summary.provider_calls, 1);
|
||||
assert.equal(selected.verdict, 'pass');
|
||||
const empty = buildH6Report(fixture(), parseH6ReportQuery({ run: 'not-found' }));
|
||||
assert.equal(empty.verdict, 'no_data');
|
||||
assert.equal(empty.summary.runs, 0);
|
||||
});
|
||||
|
||||
test('H6 query validation rejects traversal, inverted dates and invalid limits', () => {
|
||||
assert.throws(() => parseH6ReportQuery({ project: '../secret' }), BadRequestException);
|
||||
assert.throws(() => parseH6ReportQuery({ from: '2026-07-20', to: '2026-07-19' }), BadRequestException);
|
||||
assert.throws(() => parseH6ReportQuery({ limit: '5000' }), BadRequestException);
|
||||
});
|
||||
|
||||
test('H6 report makes stale optional sources explicit in data quality', () => {
|
||||
const input = fixture();
|
||||
input.sourceFreshness[1] = { ...input.sourceFreshness[1], stale: true, age_s: 7200 };
|
||||
input.sourceFreshness[2] = { ...input.sourceFreshness[2], present: false, updated_at: null, age_s: null, stale: false };
|
||||
const report = buildH6Report(input, parseH6ReportQuery({ run: 'run-a' }));
|
||||
assert.equal(report.data_quality.status, 'partial');
|
||||
assert.ok(report.data_quality.warnings.some((warning) => warning.includes('provider telemetry source is stale')));
|
||||
assert.ok(report.data_quality.warnings.some((warning) => warning.includes('alerts telemetry source is missing')));
|
||||
});
|
||||
|
||||
test('HTML export is standalone, escaped and contains no hard-coded maturity score', () => {
|
||||
const input = fixture();
|
||||
input.metrics[0].step = '<script>alert(1)</script>';
|
||||
const report = buildH6Report(input, parseH6ReportQuery({ project: 'okr' }));
|
||||
const html = renderH6ReportHtml(report);
|
||||
assert.match(html, /^<!doctype html>/);
|
||||
assert.match(html, /H6 · AgentOps Report/);
|
||||
assert.match(html, /<script>alert\(1\)<\/script>/);
|
||||
assert.doesNotMatch(html, /<script>alert\(1\)<\/script>/);
|
||||
assert.doesNotMatch(html, /Average\s+\d|\/100|218 core tests/i);
|
||||
assert.match(html, /No maturity score or telemetry value is hard-coded/);
|
||||
});
|
||||
@@ -13,6 +13,7 @@ import { CommandCenter } from './pages/CommandCenter';
|
||||
import { Chat } from './pages/Chat';
|
||||
import { Goals } from './pages/Goals';
|
||||
import { EvidencePacks } from './pages/EvidencePacks';
|
||||
import { H6ReportPage } from './pages/H6Report';
|
||||
|
||||
export default function App() {
|
||||
return (
|
||||
@@ -31,6 +32,7 @@ export default function App() {
|
||||
<Route path="/chat" element={<Chat />} />
|
||||
<Route path="/goals" element={<Goals />} />
|
||||
<Route path="/evidence-packs" element={<EvidencePacks />} />
|
||||
<Route path="/reports/h6" element={<H6ReportPage />} />
|
||||
<Route path="*" element={<Navigate to="/" replace />} />
|
||||
</Routes>
|
||||
</AppLayout>
|
||||
|
||||
@@ -13,6 +13,7 @@ const PAGE_COPY: Record<string, { title: string; eyebrow: string }> = {
|
||||
'/incidents': { title: 'Incident response', eyebrow: 'Containment' },
|
||||
'/traceability': { title: 'Traceability', eyebrow: 'Evidence graph' },
|
||||
'/finops': { title: 'FinOps & SLO', eyebrow: 'Model economics' },
|
||||
'/reports/h6': { title: 'H6 AgentOps report', eyebrow: 'Assurance dossier' },
|
||||
'/approvals': { title: 'Approval inbox', eyebrow: 'Human-in-the-loop' },
|
||||
'/settings': { title: 'Governed settings', eyebrow: 'Control plane' },
|
||||
};
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import type { ReactNode } from 'react';
|
||||
import { NavLink } from 'react-router-dom';
|
||||
|
||||
type IconName = 'grid' | 'command' | 'chat' | 'goal' | 'runs' | 'shield' | 'governance' | 'incident' | 'trace' | 'evidence' | 'coins' | 'approval' | 'settings';
|
||||
type IconName = 'grid' | 'command' | 'chat' | 'goal' | 'runs' | 'shield' | 'governance' | 'incident' | 'trace' | 'evidence' | 'report' | 'coins' | 'approval' | 'settings';
|
||||
|
||||
interface NavItem { to: string; label: string; icon: IconName; }
|
||||
|
||||
@@ -19,6 +19,7 @@ const NAVIGATION: Array<{ label: string; items: NavItem[] }> = [
|
||||
{ to: '/incidents', label: 'Incidents', icon: 'incident' },
|
||||
{ to: '/traceability', label: 'Traceability', icon: 'trace' },
|
||||
{ to: '/evidence-packs', label: 'Evidence packs', icon: 'evidence' },
|
||||
{ to: '/reports/h6', label: 'H6 AgentOps report', icon: 'report' },
|
||||
] },
|
||||
{ label: 'Control', items: [
|
||||
{ to: '/finops', label: 'FinOps & SLO', icon: 'coins' },
|
||||
@@ -41,6 +42,7 @@ function Icon({ name }: { name: IconName }) {
|
||||
incident: <><path d="M10.3 3.3 2.7 17a2 2 0 0 0 1.7 3h15.2a2 2 0 0 0 1.7-3L13.7 3.3a2 2 0 0 0-3.4 0Z" /><path d="M12 9v4M12 17h.01" /></>,
|
||||
trace: <><circle cx="6" cy="6" r="3" /><circle cx="18" cy="18" r="3" /><circle cx="18" cy="6" r="3" /><path d="m8.6 7.5 6.8 3M9 6h6" /></>,
|
||||
evidence: <><path d="M7 3h7l3 3v15H7a2 2 0 0 1-2-2V5a2 2 0 0 1 2-2Z" /><path d="M14 3v4h4M8.5 12h7M8.5 16h5" /><path d="m9 8.5 1 1 2-2" /></>,
|
||||
report: <><path d="M4 20V10M10 20V4M16 20v-7M22 20H2" /><path d="M3 10h2M9 4h2M15 13h2" /><path d="m16 7 2 2 4-5" /></>,
|
||||
coins: <><ellipse cx="12" cy="5" rx="7" ry="3" /><path d="M5 5v7c0 1.7 3.1 3 7 3s7-1.3 7-3V5M5 12v7c0 1.7 3.1 3 7 3s7-1.3 7-3v-7" /></>,
|
||||
approval: <><path d="M9 11 11 13l4-4" /><path d="M12 22c5-2.1 8-5.3 8-10V5l-8-3-8 3v7c0 4.7 3 7.9 8 10Z" /></>,
|
||||
settings: <><circle cx="12" cy="12" r="3" /><path d="M19.4 15a1.7 1.7 0 0 0 .3 1.9l.1.1-2 2-.1-.1a1.7 1.7 0 0 0-1.9-.3 1.7 1.7 0 0 0-1 1.5v.2h-2.8v-.2a1.7 1.7 0 0 0-1-1.5 1.7 1.7 0 0 0-1.9.3l-.1.1-2-2 .1-.1A1.7 1.7 0 0 0 7.4 15a1.7 1.7 0 0 0-1.5-1H5.7v-2.8h.2a1.7 1.7 0 0 0 1.5-1 1.7 1.7 0 0 0-.3-1.9L7 8.2l2-2 .1.1a1.7 1.7 0 0 0 1.9.3 1.7 1.7 0 0 0 1-1.5v-.2h2.8v.2a1.7 1.7 0 0 0 1 1.5 1.7 1.7 0 0 0 1.9-.3l.1-.1 2 2-.1.1a1.7 1.7 0 0 0-.3 1.9 1.7 1.7 0 0 0 1.5 1h.2V14h-.2a1.7 1.7 0 0 0-1.5 1Z" /></>,
|
||||
|
||||
@@ -21,7 +21,92 @@ async function post<T>(path: string, body: unknown, headers: Record<string, stri
|
||||
return res.data.data as T;
|
||||
}
|
||||
|
||||
export interface Freshness { stale: boolean; age_s: number | null; stale_after_s: number }
|
||||
export interface TelemetrySourceFreshness {
|
||||
source: 'metrics' | 'provider' | 'alerts';
|
||||
path: string;
|
||||
present: boolean;
|
||||
updated_at: string | null;
|
||||
age_s: number | null;
|
||||
stale: boolean;
|
||||
required: boolean;
|
||||
records?: number;
|
||||
}
|
||||
|
||||
export interface Freshness { stale: boolean; age_s: number | null; stale_after_s: number; sources?: TelemetrySourceFreshness[] }
|
||||
|
||||
export type HarnessReportId = 'H1' | 'H2' | 'H3' | 'H4' | 'H5' | 'H6' | 'H7';
|
||||
export type HarnessReportVerdict = 'pass' | 'attention' | 'fail' | 'no_data';
|
||||
|
||||
export interface H6ReportQuery {
|
||||
project?: string;
|
||||
from?: string;
|
||||
to?: string;
|
||||
run?: string;
|
||||
limit?: number;
|
||||
}
|
||||
|
||||
export interface H6ReportFinding {
|
||||
severity: 'info' | 'warning' | 'critical';
|
||||
code: string;
|
||||
message: string;
|
||||
metric?: string;
|
||||
value?: number | string;
|
||||
threshold?: number | string;
|
||||
}
|
||||
|
||||
export interface H6Breakdown {
|
||||
key: string;
|
||||
runs: number;
|
||||
failures: number;
|
||||
latency_avg_ms: number;
|
||||
tokens: number;
|
||||
cost_usd: number;
|
||||
}
|
||||
|
||||
export interface H6Report {
|
||||
schema_version: 1;
|
||||
report_id: string;
|
||||
harness: 'H6';
|
||||
title: string;
|
||||
description: string;
|
||||
generated_at: string;
|
||||
scope: { project: string | null; from: string | null; to: string | null; run: string | null };
|
||||
verdict: HarnessReportVerdict;
|
||||
verdict_reasons: string[];
|
||||
freshness: {
|
||||
status: 'live' | 'stale' | 'missing';
|
||||
stale_after_s: number;
|
||||
primary_age_s: number | null;
|
||||
sources: Array<TelemetrySourceFreshness & { records: number }>;
|
||||
};
|
||||
summary: {
|
||||
runs: number;
|
||||
success: number;
|
||||
failed: number;
|
||||
degraded: number;
|
||||
failure_rate_pct: number;
|
||||
retries: number;
|
||||
latency_ms: { average: number; p50: number; p95: number; p99: number; max: number };
|
||||
tokens: { input: number; output: number; total: number; provider_total: number };
|
||||
cost_usd: { provider_actual: number; estimated: number };
|
||||
provider_calls: number;
|
||||
alerts: number;
|
||||
};
|
||||
thresholds: Record<string, number | string | boolean>;
|
||||
findings: H6ReportFinding[];
|
||||
evidence_sources: Array<TelemetrySourceFreshness & { records: number }>;
|
||||
data_quality: { status: 'complete' | 'partial' | 'insufficient'; warnings: string[] };
|
||||
available_filters: { projects: string[]; runs: string[] };
|
||||
details: {
|
||||
by_status: Array<{ status: string; count: number }>;
|
||||
by_step: H6Breakdown[];
|
||||
by_provider: H6Breakdown[];
|
||||
by_cost_source: Array<{ source: string; records: number; cost_usd: number }>;
|
||||
by_alert: Array<{ alert: string; count: number }>;
|
||||
recent_runs: HarnessRunRecord[];
|
||||
recent_alerts: Array<Record<string, unknown>>;
|
||||
};
|
||||
}
|
||||
|
||||
export interface HarnessRunRecord {
|
||||
timestamp?: string;
|
||||
@@ -478,6 +563,24 @@ function actorHeaders(actor: SettingsActor): Record<string, string> {
|
||||
};
|
||||
}
|
||||
|
||||
function reportQuery(query: H6ReportQuery): string {
|
||||
const params = new URLSearchParams();
|
||||
if (query.project) params.set('project', query.project);
|
||||
if (query.from) params.set('from', query.from);
|
||||
if (query.to) params.set('to', query.to);
|
||||
if (query.run) params.set('run', query.run);
|
||||
if (query.limit) params.set('limit', String(query.limit));
|
||||
const serialized = params.toString();
|
||||
return serialized ? `?${serialized}` : '';
|
||||
}
|
||||
|
||||
export function h6ReportExportUrl(query: H6ReportQuery, format: 'json' | 'html'): string {
|
||||
const base = import.meta.env.VITE_API_BASE_URL ?? '/api/v1';
|
||||
const params = new URLSearchParams(reportQuery(query).replace(/^\?/, ''));
|
||||
params.set('format', format);
|
||||
return `${base}/reports/h6/export?${params.toString()}`;
|
||||
}
|
||||
|
||||
export const api = {
|
||||
session: () => get<SettingsActor>('session'),
|
||||
overview: () => get<Overview>('overview'),
|
||||
@@ -502,6 +605,7 @@ export const api = {
|
||||
incidents: () => get<Freshness & { total: number; kill_switch_scopes: string[]; incidents: any[] }>('incidents'),
|
||||
traceability: () => get<Freshness & { matrix: any }>('traceability'),
|
||||
cost: () => get<Freshness & { provider_tokens: number; provider_cost: number; by_provider: any[]; business_kpi: any }>('cost'),
|
||||
h6Report: (query: H6ReportQuery = {}) => get<H6Report>(`reports/h6${reportQuery(query)}`),
|
||||
command: () => get<CommandCenterState>('command'),
|
||||
evidencePacks: () => get<{ root: string; packs: EvidencePackSummary[] }>('evidence-packs'),
|
||||
evidencePack: (id: string) => get<EvidencePackDetail>(`evidence-packs/${encodeURIComponent(id)}`),
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
import { useQuery } from '@tanstack/react-query';
|
||||
import { useMemo, useState, type FormEvent, type ReactNode } from 'react';
|
||||
import { useSearchParams } from 'react-router-dom';
|
||||
import { api, h6ReportExportUrl, type H6Breakdown, type H6ReportQuery, type HarnessReportVerdict } from '../lib/api';
|
||||
import { Card, StatusBadge } from '../components/ui/Card';
|
||||
|
||||
const money = (value: number): string => `$${value.toFixed(6)}`;
|
||||
const integer = (value: number): string => new Intl.NumberFormat('en-US').format(value);
|
||||
const verdictTone: Record<HarnessReportVerdict, string> = {
|
||||
pass: 'border-emerald-300 bg-emerald-50 text-emerald-900',
|
||||
attention: 'border-amber-300 bg-amber-50 text-amber-950',
|
||||
fail: 'border-rose-300 bg-rose-50 text-rose-950',
|
||||
no_data: 'border-slate-300 bg-slate-50 text-slate-800',
|
||||
};
|
||||
|
||||
function queryFromSearch(search: URLSearchParams): H6ReportQuery {
|
||||
return {
|
||||
project: search.get('project') || undefined,
|
||||
from: search.get('from') || undefined,
|
||||
to: search.get('to') || undefined,
|
||||
run: search.get('run') || undefined,
|
||||
limit: 50,
|
||||
};
|
||||
}
|
||||
|
||||
function FilterField({ label, children }: { label: string; children: ReactNode }) {
|
||||
return <label className="block"><span className="mb-1.5 block text-[10px] font-bold uppercase tracking-[0.13em] text-slate-500">{label}</span>{children}</label>;
|
||||
}
|
||||
|
||||
function MetricCard({ label, value, sub, accent = false }: { label: string; value: string; sub?: string; accent?: boolean }) {
|
||||
return <div className={`relative overflow-hidden rounded-2xl border p-4 ${accent ? 'border-indigo-200 bg-indigo-50/70' : 'border-slate-200 bg-white'}`}>
|
||||
{accent && <span className="absolute -right-5 -top-5 h-16 w-16 rounded-full bg-indigo-200/50 blur-xl" />}
|
||||
<div className="relative text-[10px] font-bold uppercase tracking-[0.12em] text-slate-500">{label}</div>
|
||||
<div className="relative mt-2 text-2xl font-semibold tracking-tight text-slate-900">{value}</div>
|
||||
{sub && <div className="relative mt-1 text-xs text-slate-500">{sub}</div>}
|
||||
</div>;
|
||||
}
|
||||
|
||||
function BreakdownTable({ rows, subject }: { rows: H6Breakdown[]; subject: string }) {
|
||||
return <div className="overflow-x-auto"><table className="w-full min-w-[680px] text-sm">
|
||||
<thead><tr className="border-b border-slate-200 text-left text-[10px] font-bold uppercase tracking-[0.1em] text-slate-400">
|
||||
<th className="py-2.5 pr-3">{subject}</th><th>runs</th><th>failures</th><th>avg latency</th><th>tokens</th><th>cost</th>
|
||||
</tr></thead>
|
||||
<tbody>{rows.map((row) => <tr key={row.key} className="border-b border-slate-100 transition-colors hover:bg-slate-50/80">
|
||||
<td className="max-w-[300px] py-3 pr-3 font-medium text-slate-800">{row.key}</td><td>{integer(row.runs)}</td><td>{integer(row.failures)}</td><td>{integer(row.latency_avg_ms)} ms</td><td>{integer(row.tokens)}</td><td>{money(row.cost_usd)}</td>
|
||||
</tr>)}{rows.length === 0 && <tr><td colSpan={6} className="py-8 text-center text-slate-500">No matching {subject.toLowerCase()} records.</td></tr>}</tbody>
|
||||
</table></div>;
|
||||
}
|
||||
|
||||
function LoadingReport() {
|
||||
return <div className="space-y-4" aria-label="Loading H6 report"><div className="h-44 animate-pulse rounded-3xl bg-slate-200" /><div className="grid grid-cols-2 gap-3 lg:grid-cols-4">{Array.from({ length: 8 }, (_, index) => <div key={index} className="h-28 animate-pulse rounded-2xl bg-slate-200" />)}</div></div>;
|
||||
}
|
||||
|
||||
export function H6ReportPage() {
|
||||
const [searchParams, setSearchParams] = useSearchParams();
|
||||
const query = useMemo(() => queryFromSearch(searchParams), [searchParams]);
|
||||
const [draft, setDraft] = useState(() => ({
|
||||
project: searchParams.get('project') ?? '',
|
||||
from: searchParams.get('from') ?? '',
|
||||
to: searchParams.get('to') ?? '',
|
||||
run: searchParams.get('run') ?? '',
|
||||
}));
|
||||
const report = useQuery({ queryKey: ['reports', 'h6', query], queryFn: () => api.h6Report(query), retry: false });
|
||||
|
||||
const applyFilters = (event: FormEvent<HTMLFormElement>) => {
|
||||
event.preventDefault();
|
||||
const next = new URLSearchParams();
|
||||
if (draft.project) next.set('project', draft.project);
|
||||
if (draft.from) next.set('from', draft.from);
|
||||
if (draft.to) next.set('to', draft.to);
|
||||
if (draft.run) next.set('run', draft.run);
|
||||
setSearchParams(next);
|
||||
};
|
||||
|
||||
const clearFilters = () => {
|
||||
setDraft({ project: '', from: '', to: '', run: '' });
|
||||
setSearchParams({});
|
||||
};
|
||||
|
||||
const inputClass = 'w-full rounded-xl border border-slate-300 bg-white px-3 py-2.5 text-sm text-slate-800 outline-none transition focus:border-indigo-400 focus:ring-2 focus:ring-indigo-100';
|
||||
|
||||
return <div className="space-y-5">
|
||||
<section className="relative overflow-hidden rounded-3xl border border-slate-800 bg-[#111827] px-5 py-6 text-white shadow-[0_22px_55px_rgba(15,23,42,0.16)] sm:px-7">
|
||||
<div className="absolute right-[-60px] top-[-90px] h-64 w-64 rounded-full bg-indigo-500/25 blur-3xl" />
|
||||
<div className="absolute bottom-[-120px] left-1/3 h-56 w-56 rounded-full bg-teal-500/10 blur-3xl" />
|
||||
<div className="relative flex flex-col justify-between gap-5 lg:flex-row lg:items-end">
|
||||
<div><div className="text-[10px] font-bold uppercase tracking-[0.2em] text-indigo-300">H6 · evidence-backed operations</div><h2 className="mt-2 text-2xl font-semibold tracking-tight sm:text-3xl">AgentOps assurance dossier</h2><p className="mt-2 max-w-2xl text-sm leading-6 text-slate-300">One report surface for runtime health, provider usage, cost provenance, failures, retries, alerts and source freshness.</p></div>
|
||||
{report.data && <div className="flex flex-wrap items-center gap-2"><span className={`rounded-full border px-3 py-2 text-xs font-black uppercase tracking-[0.12em] ${verdictTone[report.data.verdict]}`}>{report.data.verdict}</span><span className="rounded-full border border-slate-700 bg-slate-900/70 px-3 py-2 text-xs text-slate-300">{report.data.freshness.status} · {report.data.report_id}</span></div>}
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<Card title="Report scope" right={<span className="text-xs font-normal normal-case tracking-normal text-slate-400">Filters apply to API and both exports</span>}>
|
||||
<form onSubmit={applyFilters} className="grid gap-3 md:grid-cols-2 xl:grid-cols-[1fr_1fr_1fr_1.4fr_auto] xl:items-end">
|
||||
<FilterField label="Project"><select value={draft.project} onChange={(event) => setDraft((current) => ({ ...current, project: event.target.value }))} className={inputClass}><option value="">All projects</option>{report.data?.available_filters.projects.map((project) => <option key={project} value={project}>{project}</option>)}</select></FilterField>
|
||||
<FilterField label="From"><input type="date" value={draft.from} onChange={(event) => setDraft((current) => ({ ...current, from: event.target.value }))} className={inputClass} /></FilterField>
|
||||
<FilterField label="To"><input type="date" value={draft.to} onChange={(event) => setDraft((current) => ({ ...current, to: event.target.value }))} className={inputClass} /></FilterField>
|
||||
<FilterField label="Run / trace"><input list="h6-run-options" value={draft.run} onChange={(event) => setDraft((current) => ({ ...current, run: event.target.value }))} placeholder="All runs" className={inputClass} /><datalist id="h6-run-options">{report.data?.available_filters.runs.map((run) => <option key={run} value={run} />)}</datalist></FilterField>
|
||||
<div className="flex gap-2"><button type="submit" className="rounded-xl bg-indigo-600 px-4 py-2.5 text-sm font-semibold text-white transition hover:bg-indigo-700 focus:outline-none focus:ring-2 focus:ring-indigo-300">Apply</button><button type="button" onClick={clearFilters} className="rounded-xl border border-slate-300 bg-white px-3 py-2.5 text-sm font-semibold text-slate-600 transition hover:bg-slate-50">Clear</button></div>
|
||||
</form>
|
||||
</Card>
|
||||
|
||||
{report.isLoading && <LoadingReport />}
|
||||
{report.isError && <div role="alert" className="rounded-2xl border border-rose-200 bg-rose-50 p-5 text-sm text-rose-800"><div className="font-semibold">The H6 report could not be generated.</div><p className="mt-1 text-rose-700">Check the selected time range and confirm the Control Panel API can read the canonical telemetry sources.</p></div>}
|
||||
|
||||
{report.data && <>
|
||||
<section className="grid grid-cols-2 gap-3 lg:grid-cols-4 xl:grid-cols-8">
|
||||
<MetricCard label="Runs" value={integer(report.data.summary.runs)} sub={`${report.data.summary.success} successful`} accent />
|
||||
<MetricCard label="Failures" value={integer(report.data.summary.failed)} sub={`${report.data.summary.failure_rate_pct}% rate`} />
|
||||
<MetricCard label="Degraded" value={integer(report.data.summary.degraded)} />
|
||||
<MetricCard label="P95 latency" value={`${integer(report.data.summary.latency_ms.p95)}ms`} sub={`P50 ${integer(report.data.summary.latency_ms.p50)}ms`} />
|
||||
<MetricCard label="Provider tokens" value={integer(report.data.summary.tokens.provider_total)} sub={`${report.data.summary.provider_calls} calls`} accent />
|
||||
<MetricCard label="Actual cost" value={money(report.data.summary.cost_usd.provider_actual)} sub="provider reported" />
|
||||
<MetricCard label="Estimated cost" value={money(report.data.summary.cost_usd.estimated)} sub="runtime fallback" />
|
||||
<MetricCard label="Alerts" value={integer(report.data.summary.alerts)} sub={`${report.data.summary.retries} retries`} />
|
||||
</section>
|
||||
|
||||
<div className="grid gap-5 xl:grid-cols-[1.15fr_0.85fr]">
|
||||
<Card title="Verdict findings" right={<StatusBadge value={report.data.verdict} />}>
|
||||
<div className="space-y-2.5">{report.data.findings.map((finding) => <div key={finding.code} className={`rounded-xl border-l-4 p-3.5 ${finding.severity === 'critical' ? 'border-rose-500 bg-rose-50' : finding.severity === 'warning' ? 'border-amber-500 bg-amber-50' : 'border-indigo-500 bg-indigo-50'}`}><div className="flex flex-wrap items-center justify-between gap-2"><span className="font-mono text-xs font-bold text-slate-800">{finding.code}</span>{finding.metric && <span className="text-[11px] text-slate-500">{finding.metric}: {String(finding.value)}{finding.threshold !== undefined ? ` · threshold ${String(finding.threshold)}` : ''}</span>}</div><p className="mt-1 text-sm leading-5 text-slate-600">{finding.message}</p></div>)}{report.data.findings.length === 0 && <div className="rounded-xl border border-emerald-200 bg-emerald-50 p-4 text-sm text-emerald-800">No threshold breach was detected in this scope.</div>}</div>
|
||||
</Card>
|
||||
<Card title="Evidence freshness" right={<StatusBadge value={report.data.freshness.status} />}>
|
||||
<div className="space-y-3">{report.data.evidence_sources.map((source) => <div key={source.source} className="rounded-xl border border-slate-200 bg-slate-50/70 p-3"><div className="flex items-center justify-between gap-3"><span className="text-sm font-semibold capitalize text-slate-800">{source.source}</span><StatusBadge value={!source.present ? 'missing' : source.stale ? 'stale' : 'fresh'} /></div><div className="mt-2 flex justify-between text-xs text-slate-500"><span>{source.records} records</span><span>{source.age_s === null ? 'no timestamp' : `${integer(source.age_s)}s old`}</span></div><div className="mt-2 break-all font-mono text-[10px] leading-4 text-slate-400">{source.path}</div></div>)}</div>
|
||||
</Card>
|
||||
</div>
|
||||
|
||||
<div className="grid gap-5 xl:grid-cols-2"><Card title="Runtime by step"><BreakdownTable rows={report.data.details.by_step} subject="Step" /></Card><Card title="Provider usage"><BreakdownTable rows={report.data.details.by_provider} subject="Provider · model" /></Card></div>
|
||||
|
||||
<div className="grid gap-5 xl:grid-cols-[0.8fr_1.2fr]">
|
||||
<Card title={`Data quality · ${report.data.data_quality.status}`}>
|
||||
{report.data.data_quality.warnings.length > 0 ? <ul className="space-y-2 text-sm leading-5 text-slate-600">{report.data.data_quality.warnings.map((warning) => <li key={warning} className="flex gap-2"><span className="mt-1 h-2 w-2 shrink-0 rounded-full bg-amber-400" />{warning}</li>)}</ul> : <div className="text-sm text-emerald-700">All required sources are present and no estimation warning was detected.</div>}
|
||||
</Card>
|
||||
<Card title="Independent export" right={<span className="text-xs font-normal normal-case tracking-normal text-slate-400">Generated on demand · no hard-coded score</span>}>
|
||||
<p className="text-sm leading-6 text-slate-600">Both exports use the same contract, filters, verdict rules and canonical sources as this screen. JSON is machine-auditable; HTML is print-ready for team review.</p>
|
||||
<div className="mt-4 flex flex-wrap gap-3"><a href={h6ReportExportUrl(query, 'json')} className="rounded-xl bg-slate-900 px-4 py-2.5 text-sm font-semibold text-white transition hover:bg-slate-700">Download JSON</a><a href={h6ReportExportUrl(query, 'html')} className="rounded-xl border border-indigo-300 bg-indigo-50 px-4 py-2.5 text-sm font-semibold text-indigo-700 transition hover:bg-indigo-100">Download HTML</a></div>
|
||||
</Card>
|
||||
</div>
|
||||
</>}
|
||||
</div>;
|
||||
}
|
||||
@@ -27,9 +27,9 @@ LOG_DIR="$CASAN_STATE_ROOT/logs"
|
||||
TRACE_DIR="$LOG_DIR/trace"
|
||||
# SEC-23 (MT-01): telemetry dir is tenant-scoped when CASAN_METRICS_DIR is set
|
||||
# (tenant-paths.sh exports it per tenant); default is the shared path.
|
||||
METRICS_DIR="${CASAN_METRICS_DIR:-$LOG_DIR/cost}"
|
||||
ALERT_LOG="$CASAN_HARNESS_ROOT/agentops/alerts.log"
|
||||
METRICS_LOG="$METRICS_DIR/metrics.jsonl"
|
||||
METRICS_DIR="${CASAN_METRICS_DIR:-$(dirname "$CASAN_TELEMETRY_METRICS_LOG")}"
|
||||
ALERT_LOG="$CASAN_TELEMETRY_ALERTS_LOG"
|
||||
METRICS_LOG="${CASAN_METRICS_LOG:-$CASAN_TELEMETRY_METRICS_LOG}"
|
||||
mkdir -p "$TRACE_DIR" "$METRICS_DIR" "$(dirname "$OUTPUT_FILE")" "$(dirname "$ALERT_LOG")"
|
||||
|
||||
# shellcheck source=tool-audit-lib.sh
|
||||
@@ -127,7 +127,7 @@ COST_SOURCE="word_count_estimate"
|
||||
|
||||
# Prefer real provider usage when telemetry has been imported; the word-count
|
||||
# figure above is an explicit fallback, not presented as a real billed cost.
|
||||
PROVIDER_LOG="$CASAN_STATE_ROOT/logs/level5/provider-usage.jsonl"
|
||||
PROVIDER_LOG="$CASAN_TELEMETRY_PROVIDER_LOG"
|
||||
if [[ -f "$PROVIDER_LOG" ]] && command -v python >/dev/null 2>&1; then
|
||||
# Use real provider telemetry ONLY when a record genuinely matches this step.
|
||||
# Do NOT fall back to an arbitrary record (that would reuse one sample's cost
|
||||
|
||||
@@ -61,6 +61,20 @@ if [[ -z "${CASAN_STATE_ROOT:-}" ]]; then
|
||||
CASAN_STATE_ROOT="$CASAN_APP_ROOT/.specify"
|
||||
fi
|
||||
|
||||
# Canonical H6 telemetry files. Writers and readers must use these three names so
|
||||
# runtime, provider usage and alerts cannot silently drift into different trees.
|
||||
# The older dashboard/control-panel variables remain accepted as compatibility
|
||||
# aliases while downstream deployments move to the CASAN_TELEMETRY_* contract.
|
||||
if [[ -z "${CASAN_TELEMETRY_METRICS_LOG:-}" ]]; then
|
||||
CASAN_TELEMETRY_METRICS_LOG="${CASAN_DASHBOARD_METRICS:-$CASAN_STATE_ROOT/logs/cost/metrics.jsonl}"
|
||||
fi
|
||||
if [[ -z "${CASAN_TELEMETRY_PROVIDER_LOG:-}" ]]; then
|
||||
CASAN_TELEMETRY_PROVIDER_LOG="${CASAN_CP_PROVIDER_USAGE:-${CASAN_PROVIDER_LOG:-$CASAN_STATE_ROOT/logs/level5/provider-usage.jsonl}}"
|
||||
fi
|
||||
if [[ -z "${CASAN_TELEMETRY_ALERTS_LOG:-}" ]]; then
|
||||
CASAN_TELEMETRY_ALERTS_LOG="${CASAN_DASHBOARD_ALERTS:-$CASAN_STATE_ROOT/agentops/alerts.log}"
|
||||
fi
|
||||
|
||||
# Governance root: central-governance mixes harness pub-keys/registries, runtime
|
||||
# policy-manifest state, and a private key. Rooted under STATE (not HARNESS) so the
|
||||
# runtime-regenerated policy-manifest.{json,sig} and the private key never land inside
|
||||
|
||||
@@ -75,7 +75,8 @@ def head_path() -> str:
|
||||
|
||||
|
||||
def metrics_path() -> str:
|
||||
return guarded_override(os.environ["CASAN_CHAT_METRICS_LOG"]) if os.environ.get("CASAN_CHAT_METRICS_LOG") else tenant_path("telemetry/cost/metrics.jsonl", "logs/cost/metrics.jsonl")
|
||||
override = os.environ.get("CASAN_TELEMETRY_METRICS_LOG") or os.environ.get("CASAN_CHAT_METRICS_LOG")
|
||||
return guarded_override(override) if override else tenant_path("telemetry/cost/metrics.jsonl", "logs/cost/metrics.jsonl")
|
||||
|
||||
|
||||
def artifact_dir() -> str:
|
||||
|
||||
@@ -90,7 +90,8 @@ def head_path() -> str:
|
||||
|
||||
|
||||
def metrics_path() -> str:
|
||||
return guarded_override(os.environ["CASAN_CHAT_METRICS_LOG"]) if os.environ.get("CASAN_CHAT_METRICS_LOG") else tenant_path("telemetry/cost/metrics.jsonl", "logs/cost/metrics.jsonl")
|
||||
override = os.environ.get("CASAN_TELEMETRY_METRICS_LOG") or os.environ.get("CASAN_CHAT_METRICS_LOG")
|
||||
return guarded_override(override) if override else tenant_path("telemetry/cost/metrics.jsonl", "logs/cost/metrics.jsonl")
|
||||
|
||||
|
||||
def trace_events_path(trace_id: str) -> str:
|
||||
|
||||
@@ -24,7 +24,7 @@ set -uo pipefail
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$SCRIPT_DIR/casan-paths.sh"
|
||||
ROOT="$CASAN_APP_ROOT"
|
||||
PROVIDER_LOG="${CASAN_PROVIDER_LOG:-$CASAN_STATE_ROOT/logs/level5/provider-usage.jsonl}"
|
||||
PROVIDER_LOG="$CASAN_TELEMETRY_PROVIDER_LOG"
|
||||
CIRCUIT_BREAKER_THRESHOLD="${CIRCUIT_BREAKER_THRESHOLD:-5}"
|
||||
CIRCUIT_WINDOW="${CIRCUIT_WINDOW:-10}"
|
||||
CIRCUIT_WINDOW_FAIL_PCT="${CIRCUIT_WINDOW_FAIL_PCT:-50}"
|
||||
|
||||
@@ -30,7 +30,7 @@ elif [[ -n "${CASAN_TENANT_ID:-}" ]]; then
|
||||
LOG="$(bash "$SCRIPT_DIR/tenant-store.sh" resolve telemetry/provider-usage.jsonl 2>/dev/null)" \
|
||||
|| { echo "COST_SPIKE_TENANT_DENIED" >&2; exit 3; }
|
||||
else
|
||||
LOG="$CASAN_STATE_ROOT/logs/level5/provider-usage.jsonl"
|
||||
LOG="$CASAN_TELEMETRY_PROVIDER_LOG"
|
||||
fi
|
||||
MULT="${2:-3.0}"
|
||||
|
||||
|
||||
@@ -35,9 +35,11 @@ PORT = int(sys.argv[1]) if len(sys.argv) > 1 else 8787
|
||||
DASH = pathlib.Path(os.environ.get(
|
||||
"CASAN_DASHBOARD_HTML", ROOT / "docs" / "output" / "casan" / "central-agentops-dashboard.html"))
|
||||
METRICS = pathlib.Path(os.environ.get(
|
||||
"CASAN_DASHBOARD_METRICS", ROOT / ".specify" / "logs" / "cost" / "metrics.jsonl"))
|
||||
"CASAN_TELEMETRY_METRICS_LOG", os.environ.get(
|
||||
"CASAN_DASHBOARD_METRICS", ROOT / ".specify" / "logs" / "cost" / "metrics.jsonl")))
|
||||
ALERTS = pathlib.Path(os.environ.get(
|
||||
"CASAN_DASHBOARD_ALERTS", ROOT / ".specify" / "agentops" / "alerts.log"))
|
||||
"CASAN_TELEMETRY_ALERTS_LOG", os.environ.get(
|
||||
"CASAN_DASHBOARD_ALERTS", ROOT / ".specify" / "agentops" / "alerts.log")))
|
||||
STALE_S = int(os.environ.get("CASAN_DASHBOARD_STALE_S", "3600"))
|
||||
|
||||
|
||||
|
||||
@@ -133,9 +133,17 @@ def main():
|
||||
"telemetry_integrity": tel_text, "telemetry_ok": tel_rc == 0,
|
||||
}
|
||||
|
||||
# H6 cost telemetry
|
||||
prov = read_jsonl(os.path.join(logs, "level5", "provider-usage.jsonl"))
|
||||
metrics = read_jsonl(os.path.join(logs, "cost", "metrics.jsonl"))
|
||||
# H6 telemetry — use the same canonical sources as the Control Panel/report API.
|
||||
provider_path = os.environ.get(
|
||||
"CASAN_TELEMETRY_PROVIDER_LOG",
|
||||
os.environ.get("CASAN_CP_PROVIDER_USAGE", os.path.join(logs, "level5", "provider-usage.jsonl")),
|
||||
)
|
||||
metrics_path = os.environ.get(
|
||||
"CASAN_TELEMETRY_METRICS_LOG",
|
||||
os.environ.get("CASAN_DASHBOARD_METRICS", os.path.join(logs, "cost", "metrics.jsonl")),
|
||||
)
|
||||
prov = read_jsonl(provider_path)
|
||||
metrics = read_jsonl(metrics_path)
|
||||
cost_rc = int(os.environ.get("CASAN_EP_COST_RC", "3") or "3")
|
||||
total_tokens = sum(int(r.get("total_tokens", 0)) for r in prov if str(r.get("total_tokens", "")).isdigit())
|
||||
reports["h6-cost-telemetry.json"] = {
|
||||
|
||||
@@ -1063,7 +1063,11 @@ def audit(job: dict, status: str) -> str:
|
||||
|
||||
|
||||
def metric(job: dict, status: str, started: float, local_meta: dict, cloud_meta: dict) -> None:
|
||||
append_jsonl(os.path.join(STATE_ROOT, "logs", "cost", "metrics.jsonl"), {
|
||||
metrics_path = os.environ.get(
|
||||
"CASAN_TELEMETRY_METRICS_LOG",
|
||||
os.environ.get("CASAN_DASHBOARD_METRICS", os.path.join(STATE_ROOT, "logs", "cost", "metrics.jsonl")),
|
||||
)
|
||||
append_jsonl(metrics_path, {
|
||||
"timestamp": now(),
|
||||
"trace_id": job["id"],
|
||||
"harness": "H6-agentops",
|
||||
|
||||
@@ -14,8 +14,8 @@ fi
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$SCRIPT_DIR/casan-paths.sh"
|
||||
PROJECT_ROOT="$CASAN_APP_ROOT"
|
||||
LOG_DIR="$CASAN_STATE_ROOT/logs/level5"
|
||||
OUT="$LOG_DIR/provider-usage.jsonl"
|
||||
OUT="$CASAN_TELEMETRY_PROVIDER_LOG"
|
||||
LOG_DIR="$(dirname "$OUT")"
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
python - "$INPUT_JSON" "$OUT" <<'PY'
|
||||
|
||||
@@ -22,6 +22,85 @@ AUTH_BRIDGE_AUDIT="$AUTH_BRIDGE_DIR/model-audit.jsonl"
|
||||
AUTH_BRIDGE="$ROOT/packages/casan-control-panel/scripts/provider-auth-bridge.py"
|
||||
CMD="${1:-status}"
|
||||
|
||||
resolve_python() {
|
||||
if [[ -n "${CASAN_PYTHON_BIN:-}" ]]; then
|
||||
if command -v "$CASAN_PYTHON_BIN" >/dev/null 2>&1 && "$CASAN_PYTHON_BIN" --version >/dev/null 2>&1; then
|
||||
printf '%s\n' "$CASAN_PYTHON_BIN"
|
||||
return 0
|
||||
fi
|
||||
echo "CASAN_LOCAL_PYTHON_INVALID path=$CASAN_PYTHON_BIN" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Prefer macOS' universal system Python over stale framework installs that may
|
||||
# appear first in PATH but are terminated by Gatekeeper/Rosetta on Apple Silicon.
|
||||
local candidate
|
||||
for candidate in /usr/bin/python3 python3 python; do
|
||||
if command -v "$candidate" >/dev/null 2>&1 && "$candidate" --version >/dev/null 2>&1; then
|
||||
printf '%s\n' "$candidate"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
echo "CASAN_LOCAL_PYTHON_MISSING" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
prepare_docker_cli() {
|
||||
local original_config="${DOCKER_CONFIG:-$HOME/.docker}"
|
||||
local config_file="$original_config/config.json"
|
||||
[[ -f "$config_file" ]] || return 0
|
||||
|
||||
local python_bin credential_store helper context_name docker_host fallback_config
|
||||
python_bin="$(resolve_python)" || return 1
|
||||
credential_store="$("$python_bin" - "$config_file" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
try:
|
||||
with open(sys.argv[1], encoding="utf-8") as handle:
|
||||
print(json.load(handle).get("credsStore", ""))
|
||||
except (OSError, ValueError):
|
||||
print("")
|
||||
PY
|
||||
)"
|
||||
[[ -n "$credential_store" ]] || return 0
|
||||
|
||||
helper="$(command -v "docker-credential-$credential_store" 2>/dev/null || true)"
|
||||
if [[ -n "$helper" ]] && "$python_bin" - "$helper" <<'PY' >/dev/null 2>&1
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
raise SystemExit(subprocess.run(
|
||||
[sys.argv[1], "list"],
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
).returncode)
|
||||
PY
|
||||
then
|
||||
return 0
|
||||
fi
|
||||
|
||||
# CASAN's local compose files use public images only. If Docker Desktop's
|
||||
# credential helper is broken, isolate this process from it without changing
|
||||
# ~/.docker/config.json or touching any stored login credentials.
|
||||
context_name="$(docker context show)"
|
||||
docker_host="$(docker context inspect --format '{{(index .Endpoints "docker").Host}}' "$context_name")"
|
||||
[[ -n "$docker_host" ]] || { echo "CASAN_LOCAL_DOCKER_CONTEXT_INVALID context=$context_name" >&2; return 1; }
|
||||
fallback_config="${TMPDIR:-/tmp}/casan-docker-public-$UID"
|
||||
mkdir -p "$fallback_config"
|
||||
if [[ -d "$original_config/cli-plugins" && ! -e "$fallback_config/cli-plugins" ]]; then
|
||||
ln -s "$original_config/cli-plugins" "$fallback_config/cli-plugins"
|
||||
fi
|
||||
umask 077
|
||||
printf '%s\n' '{"auths":{"https://index.docker.io/v1/":{},"quay.io":{}}}' > "$fallback_config/config.json"
|
||||
export DOCKER_CONFIG="$fallback_config"
|
||||
export DOCKER_HOST="$docker_host"
|
||||
export DOCKER_BUILDKIT=0
|
||||
export COMPOSE_DOCKER_CLI_BUILD=0
|
||||
echo "CASAN_LOCAL_DOCKER_CREDENTIAL_FALLBACK helper=$credential_store context=$context_name builder=classic" >&2
|
||||
}
|
||||
|
||||
cp_compose() {
|
||||
if [[ -f "$AUTH_BRIDGE_TOKEN_FILE" ]]; then
|
||||
export CASAN_AUTH_BRIDGE_TOKEN
|
||||
@@ -35,6 +114,8 @@ cp_compose() {
|
||||
}
|
||||
|
||||
start_auth_bridge() {
|
||||
local python_bin
|
||||
python_bin="$(resolve_python)" || return 1
|
||||
mkdir -p "$AUTH_BRIDGE_DIR"
|
||||
if [[ ! -s "$AUTH_BRIDGE_TOKEN_FILE" ]]; then
|
||||
openssl rand -hex 32 > "$AUTH_BRIDGE_TOKEN_FILE"
|
||||
@@ -52,7 +133,7 @@ start_auth_bridge() {
|
||||
rm -f "$AUTH_BRIDGE_PID_FILE"
|
||||
fi
|
||||
[[ -f "$AUTH_BRIDGE" ]] || { echo "CASAN_AUTH_BRIDGE_MISSING" >&2; return 1; }
|
||||
nohup python3 "$AUTH_BRIDGE" --bind 0.0.0.0 --port 20130 --token-file "$AUTH_BRIDGE_TOKEN_FILE" --audit-log "$AUTH_BRIDGE_AUDIT" > "$AUTH_BRIDGE_LOG" 2>&1 &
|
||||
nohup "$python_bin" "$AUTH_BRIDGE" --bind 0.0.0.0 --port 20130 --token-file "$AUTH_BRIDGE_TOKEN_FILE" --audit-log "$AUTH_BRIDGE_AUDIT" > "$AUTH_BRIDGE_LOG" 2>&1 &
|
||||
echo "$!" > "$AUTH_BRIDGE_PID_FILE"
|
||||
chmod 600 "$AUTH_BRIDGE_PID_FILE" "$AUTH_BRIDGE_LOG" "$AUTH_BRIDGE_AUDIT" 2>/dev/null || true
|
||||
if ! wait_url "http://127.0.0.1:20130/healthz"; then
|
||||
@@ -74,6 +155,7 @@ stop_auth_bridge() {
|
||||
|
||||
need_docker() {
|
||||
command -v docker >/dev/null 2>&1 || { echo "CASAN_LOCAL_DOCKER_MISSING" >&2; exit 1; }
|
||||
prepare_docker_cli
|
||||
docker compose version >/dev/null 2>&1 || { echo "CASAN_LOCAL_COMPOSE_MISSING" >&2; exit 1; }
|
||||
}
|
||||
|
||||
|
||||
@@ -48,7 +48,10 @@ OLLAMA_HOST = "127.0.0.1:11434" # the only allowed ollama endpoint
|
||||
DOCKER_OLLAMA_HOST = "host.docker.internal:11434"
|
||||
ALLOWED_CLOUD = {"api.anthropic.com", "api.openai.com"}
|
||||
REPO_ROOT = _casan_app_root()
|
||||
PROVIDER_LOG = os.path.join(REPO_ROOT, ".specify/logs/level5/provider-usage.jsonl")
|
||||
PROVIDER_LOG = os.environ.get(
|
||||
"CASAN_TELEMETRY_PROVIDER_LOG",
|
||||
os.environ.get("CASAN_CP_PROVIDER_USAGE", os.path.join(REPO_ROOT, ".specify/logs/level5/provider-usage.jsonl")),
|
||||
)
|
||||
|
||||
# SEC-21 (ARCH-07): a 180s-per-call timeout across many pipeline steps let a hung
|
||||
# model stall a run for tens of minutes. Use a lower, configurable per-call timeout,
|
||||
|
||||
@@ -22,7 +22,7 @@ fi
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$SCRIPT_DIR/casan-paths.sh"
|
||||
PROJECT_ROOT="$CASAN_APP_ROOT"
|
||||
OUT="${2:-$CASAN_STATE_ROOT/logs/level5/provider-usage.jsonl}"
|
||||
OUT="${2:-$CASAN_TELEMETRY_PROVIDER_LOG}"
|
||||
mkdir -p "$(dirname "$OUT")"
|
||||
|
||||
# SEC-13 (M-09): SSRF guard on the fetch URL. ALWAYS reject non-http(s) schemes
|
||||
|
||||
@@ -19,9 +19,9 @@ $scriptDir = Split-Path $MyInvocation.MyCommand.Path -Parent
|
||||
$projectRoot = (Resolve-Path (Join-Path $scriptDir "../../..")).Path
|
||||
$logDir = Join-Path $projectRoot ".specify/logs"
|
||||
$traceDir = Join-Path $logDir "trace"
|
||||
$metricsDir = Join-Path $logDir "cost"
|
||||
$alertLog = Join-Path $projectRoot ".specify/agentops/alerts.log"
|
||||
$metricsLog = Join-Path $metricsDir "metrics.jsonl"
|
||||
$metricsLog = if ($env:CASAN_TELEMETRY_METRICS_LOG) { $env:CASAN_TELEMETRY_METRICS_LOG } elseif ($env:CASAN_DASHBOARD_METRICS) { $env:CASAN_DASHBOARD_METRICS } else { Join-Path $logDir "cost/metrics.jsonl" }
|
||||
$metricsDir = Split-Path $metricsLog -Parent
|
||||
$alertLog = if ($env:CASAN_TELEMETRY_ALERTS_LOG) { $env:CASAN_TELEMETRY_ALERTS_LOG } elseif ($env:CASAN_DASHBOARD_ALERTS) { $env:CASAN_DASHBOARD_ALERTS } else { Join-Path $projectRoot ".specify/agentops/alerts.log" }
|
||||
$toolAudit = Join-Path $logDir "audit/tool-calls.jsonl"
|
||||
|
||||
foreach ($d in @($traceDir, $metricsDir, (Split-Path $OutputFile -Parent), (Split-Path $alertLog -Parent), (Split-Path $toolAudit -Parent))) {
|
||||
|
||||
@@ -10,8 +10,8 @@ param(
|
||||
|
||||
$scriptDir = Split-Path $MyInvocation.MyCommand.Path -Parent
|
||||
$projectRoot = (Resolve-Path (Join-Path $scriptDir "../../..")).Path
|
||||
$metricsLog = Join-Path $projectRoot ".specify/logs/cost/metrics.jsonl"
|
||||
$alertLog = Join-Path $projectRoot ".specify/agentops/alerts.log"
|
||||
$metricsLog = if ($env:CASAN_TELEMETRY_METRICS_LOG) { $env:CASAN_TELEMETRY_METRICS_LOG } elseif ($env:CASAN_DASHBOARD_METRICS) { $env:CASAN_DASHBOARD_METRICS } else { Join-Path $projectRoot ".specify/logs/cost/metrics.jsonl" }
|
||||
$alertLog = if ($env:CASAN_TELEMETRY_ALERTS_LOG) { $env:CASAN_TELEMETRY_ALERTS_LOG } elseif ($env:CASAN_DASHBOARD_ALERTS) { $env:CASAN_DASHBOARD_ALERTS } else { Join-Path $projectRoot ".specify/agentops/alerts.log" }
|
||||
|
||||
if (!$OutputHtml) { $OutputHtml = Join-Path $projectRoot "docs/output/casan/agentops-dashboard.html" }
|
||||
if (!(Test-Path (Split-Path $OutputHtml -Parent))) { New-Item -ItemType Directory -Force -Path (Split-Path $OutputHtml -Parent) | Out-Null }
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
from datetime import datetime, timezone
|
||||
|
||||
@@ -16,10 +17,14 @@ def _app_root(start):
|
||||
|
||||
|
||||
ROOT = _app_root(__file__)
|
||||
METRICS = ROOT / ".specify" / "logs" / "cost" / "metrics.jsonl"
|
||||
METRICS = pathlib.Path(os.environ.get(
|
||||
"CASAN_TELEMETRY_METRICS_LOG", os.environ.get(
|
||||
"CASAN_DASHBOARD_METRICS", ROOT / ".specify" / "logs" / "cost" / "metrics.jsonl")))
|
||||
FALLBACK = ROOT / ".specify" / "logs" / "level5" / "fallback.jsonl"
|
||||
TOOL = ROOT / ".specify" / "logs" / "level5" / "tool-registry.jsonl"
|
||||
PROVIDER = ROOT / ".specify" / "logs" / "level5" / "provider-usage.jsonl"
|
||||
PROVIDER = pathlib.Path(os.environ.get(
|
||||
"CASAN_TELEMETRY_PROVIDER_LOG", os.environ.get(
|
||||
"CASAN_CP_PROVIDER_USAGE", ROOT / ".specify" / "logs" / "level5" / "provider-usage.jsonl")))
|
||||
PROJECT_REGISTRY = ROOT / ".specify" / "level5" / "project-registry.json"
|
||||
DASHBOARD = ROOT / "docs" / "output" / "casan" / "central-agentops-dashboard.html"
|
||||
LEGACY_DASHBOARD = ROOT / "docs" / "output" / "casan" / "agentops-dashboard.html"
|
||||
|
||||
@@ -1 +1 @@
|
||||
12941
|
||||
16827
|
||||
|
||||
@@ -1 +1 @@
|
||||
b39f21f536ddb5ce72aac871e08584044fc2578f02026a79916b92dbd8738e86
|
||||
dba1854708ce590be46221a77130fce18d2f3242804a8811ae6de815f79020b8
|
||||
|
||||
Reference in New Issue
Block a user