update report h6
This commit is contained in:
@@ -537,3 +537,6 @@
|
|||||||
{"timestamp":"2026-07-19T04:00:29Z","trace_id":"trace-1784433629-809","harness":"H4-security","mode":"output","status":"pass","action":"allow","risk_level":"low","matched_categories":[],"input_hash":"e39bb3117bcde6f297fe51ac7d86546bbfbb7b17339821001d681b95a766019b","output_hash":"665a7b0ed97ad39aeec1aab585b6a0e3779b261f7d0a6734f7f4cb07672a3354"}
|
{"timestamp":"2026-07-19T04:00:29Z","trace_id":"trace-1784433629-809","harness":"H4-security","mode":"output","status":"pass","action":"allow","risk_level":"low","matched_categories":[],"input_hash":"e39bb3117bcde6f297fe51ac7d86546bbfbb7b17339821001d681b95a766019b","output_hash":"665a7b0ed97ad39aeec1aab585b6a0e3779b261f7d0a6734f7f4cb07672a3354"}
|
||||||
{"timestamp":"2026-07-19T04:01:15Z","trace_id":"trace-1784433675-137","harness":"H4-security","mode":"input","status":"pass","action":"alert","risk_level":"medium","matched_categories":["semantic-availability"],"input_hash":"152e2c124ba6f23a03165ccb19ef9f04d6eb1155858101e8f741bb89f2245234","output_hash":"152e2c124ba6f23a03165ccb19ef9f04d6eb1155858101e8f741bb89f2245234"}
|
{"timestamp":"2026-07-19T04:01:15Z","trace_id":"trace-1784433675-137","harness":"H4-security","mode":"input","status":"pass","action":"alert","risk_level":"medium","matched_categories":["semantic-availability"],"input_hash":"152e2c124ba6f23a03165ccb19ef9f04d6eb1155858101e8f741bb89f2245234","output_hash":"152e2c124ba6f23a03165ccb19ef9f04d6eb1155858101e8f741bb89f2245234"}
|
||||||
{"timestamp":"2026-07-19T04:01:16Z","trace_id":"trace-1784433676-759","harness":"H4-security","mode":"output","status":"pass","action":"allow","risk_level":"low","matched_categories":[],"input_hash":"e39bb3117bcde6f297fe51ac7d86546bbfbb7b17339821001d681b95a766019b","output_hash":"e39bb3117bcde6f297fe51ac7d86546bbfbb7b17339821001d681b95a766019b"}
|
{"timestamp":"2026-07-19T04:01:16Z","trace_id":"trace-1784433676-759","harness":"H4-security","mode":"output","status":"pass","action":"allow","risk_level":"low","matched_categories":[],"input_hash":"e39bb3117bcde6f297fe51ac7d86546bbfbb7b17339821001d681b95a766019b","output_hash":"e39bb3117bcde6f297fe51ac7d86546bbfbb7b17339821001d681b95a766019b"}
|
||||||
|
{"timestamp":"2026-07-19T05:18:02Z","trace_id":"trace-1784438282-2705","harness":"H4-security","mode":"input","status":"pass","action":"alert","risk_level":"medium","matched_categories":["semantic-availability"],"input_hash":"3084720034103b1c979a3ae80f6f59e360833eaa6a8c88d45381820edfbe2886","output_hash":"3084720034103b1c979a3ae80f6f59e360833eaa6a8c88d45381820edfbe2886"}
|
||||||
|
{"timestamp":"2026-07-19T05:18:03Z","trace_id":"trace-1784438283-3332","harness":"H4-security","mode":"output","status":"pass","action":"allow","risk_level":"low","matched_categories":[],"input_hash":"92848c78682b10335aedc8fa6ebfb8e9b7b5f17d9709df94b7742bda7412631e","output_hash":"92848c78682b10335aedc8fa6ebfb8e9b7b5f17d9709df94b7742bda7412631e"}
|
||||||
|
{"timestamp":"2026-07-19T05:18:20Z","trace_id":"trace-1784438300-3900","harness":"H4-security","mode":"input","status":"pass","action":"allow","risk_level":"low","matched_categories":[],"input_hash":"cbd0ed3c59982438aadbf2372bf8e463b9e2a884a4e452472683bca515616767","output_hash":"14487285dc9cada36542a27770878b55e3929878a40ebb527b67feef56fd5b8e"}
|
||||||
|
|||||||
@@ -197,6 +197,26 @@ docker volume rm casan-local-prod_minio-data
|
|||||||
This is destructive and cannot be undone. Do not use it if the MinIO bucket has
|
This is destructive and cannot be undone. Do not use it if the MinIO bucket has
|
||||||
evidence you need to retain.
|
evidence you need to retain.
|
||||||
|
|
||||||
|
### Python used by the local auth bridge
|
||||||
|
|
||||||
|
`local-full.sh` validates the Python interpreter before starting the host-side
|
||||||
|
provider auth bridge. On macOS it prefers `/usr/bin/python3`, avoiding an old
|
||||||
|
Intel-only framework Python that may be first in `PATH` but terminated by macOS.
|
||||||
|
To select another working Python explicitly, set `CASAN_PYTHON_BIN`:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
CASAN_PYTHON_BIN=/opt/homebrew/bin/python3 \
|
||||||
|
bash packages/casan-harness/scripts/bash/local-full.sh start
|
||||||
|
```
|
||||||
|
|
||||||
|
The launcher also validates Docker's configured credential helper. If macOS
|
||||||
|
terminates that helper, CASAN uses an isolated, temporary anonymous Docker
|
||||||
|
configuration for this local stack's public images while preserving the active
|
||||||
|
Docker Desktop socket and Compose plugin. The same fallback uses Docker's classic
|
||||||
|
builder if the bundled BuildKit metadata helper is also terminated. It does not
|
||||||
|
edit `~/.docker/config.json` or delete stored registry credentials. Reinstall or
|
||||||
|
update Docker Desktop later to repair the helper globally.
|
||||||
|
|
||||||
## Current scope and next phase
|
## Current scope and next phase
|
||||||
|
|
||||||
This local environment includes every currently implemented CASAN component.
|
This local environment includes every currently implemented CASAN component.
|
||||||
|
|||||||
@@ -0,0 +1,90 @@
|
|||||||
|
# CASAN Plan-19 — Harness Reporting Contract
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
Turn harness evidence into reviewable, portable reports. H6 is the first implemented
|
||||||
|
report; H1–H7 share one contract so later reports do not invent incompatible envelopes,
|
||||||
|
filter semantics or verdict fields.
|
||||||
|
|
||||||
|
## Source-of-truth boundary
|
||||||
|
|
||||||
|
H6 is a read-only projection over three canonical files. The report API and the existing
|
||||||
|
Control Panel resolve these paths through the same `app-root.ts` contract; harness writers
|
||||||
|
resolve them through `casan-paths.sh`.
|
||||||
|
|
||||||
|
| Source | Canonical environment variable | Default path | Required |
|
||||||
|
|---|---|---|---|
|
||||||
|
| Runtime metrics | `CASAN_TELEMETRY_METRICS_LOG` | `.specify/logs/cost/metrics.jsonl` | yes |
|
||||||
|
| Provider usage | `CASAN_TELEMETRY_PROVIDER_LOG` | `.specify/logs/level5/provider-usage.jsonl` | no |
|
||||||
|
| Alerts | `CASAN_TELEMETRY_ALERTS_LOG` | `.specify/agentops/alerts.log` | no |
|
||||||
|
|
||||||
|
Legacy names (`CASAN_DASHBOARD_METRICS`, `CASAN_CP_PROVIDER_USAGE`,
|
||||||
|
`CASAN_PROVIDER_LOG`, `CASAN_DASHBOARD_ALERTS`, and chat-specific metrics aliases) are
|
||||||
|
compatibility fallbacks only. New deployments should set the canonical variables.
|
||||||
|
|
||||||
|
Freshness is source-specific and based on filesystem modification time. A source becomes
|
||||||
|
stale when its age exceeds `CASAN_DASHBOARD_STALE_S` (default `3600` seconds). Missing
|
||||||
|
required metrics produce a `missing` freshness status and an insufficient-data verdict.
|
||||||
|
|
||||||
|
## Common H1–H7 contract
|
||||||
|
|
||||||
|
`packages/casan-control-panel/backend/src/reports/report.contract.ts` defines schema
|
||||||
|
version `1`. Every harness report has:
|
||||||
|
|
||||||
|
- identity: `report_id`, `harness`, `title`, `generated_at`;
|
||||||
|
- scope: `project`, `from`, `to`, `run`;
|
||||||
|
- decision: `verdict`, `verdict_reasons`, and explicit `thresholds`;
|
||||||
|
- evidence: source-level freshness, record counts and findings;
|
||||||
|
- quality: `complete`, `partial` or `insufficient` with warnings;
|
||||||
|
- presentation-neutral `summary` and `details` data.
|
||||||
|
|
||||||
|
The catalog endpoint marks H6 as `implemented`; H1–H5 and H7 are `contract_ready`. This
|
||||||
|
is deliberate: report presence must not be confused with measured evidence or a passing
|
||||||
|
score.
|
||||||
|
|
||||||
|
## H6 API and UI
|
||||||
|
|
||||||
|
| Capability | Route |
|
||||||
|
|---|---|
|
||||||
|
| Report catalog | `GET /api/v1/reports` |
|
||||||
|
| H6 JSON API | `GET /api/v1/reports/h6` |
|
||||||
|
| Independent export | `GET /api/v1/reports/h6/export?format=json|html` |
|
||||||
|
| Browser report | `/reports/h6` |
|
||||||
|
|
||||||
|
Optional filters are `project`, `from`, `to`, and `run`; `limit` only controls recent-row
|
||||||
|
detail and does not change summary aggregation. Project/run identifiers are allowlisted to
|
||||||
|
safe identifier characters, dates must parse as ISO-compatible values, and inverted ranges
|
||||||
|
fail with `400`.
|
||||||
|
|
||||||
|
The H6 summary includes runtime counts and outcomes, failure rate, retries, latency
|
||||||
|
distribution, local/provider tokens, estimated/provider-reported cost, provider calls and
|
||||||
|
alert signals. Verdicts are evidence-derived:
|
||||||
|
|
||||||
|
- `no_data`: no runtime or provider record matches the scope;
|
||||||
|
- `fail`: required metrics are missing or failure rate exceeds its threshold;
|
||||||
|
- `attention`: stale telemetry, failures, degraded runs, high P95 latency or alerts exist;
|
||||||
|
- `pass`: data is present and no finding crosses a configured threshold.
|
||||||
|
|
||||||
|
Defaults are `5%` maximum failure rate and `5000 ms` maximum P95 latency, configurable
|
||||||
|
through `CASAN_H6_FAILURE_RATE_THRESHOLD_PCT` and
|
||||||
|
`CASAN_H6_P95_LATENCY_THRESHOLD_MS`.
|
||||||
|
|
||||||
|
## Export integrity
|
||||||
|
|
||||||
|
HTML and JSON exports are generated from the exact report object returned by the H6 service.
|
||||||
|
The HTML artifact is standalone and print-ready; JSON keeps schema version `1` for automated
|
||||||
|
consumers. Neither export reads a second data source or contains a hard-coded maturity
|
||||||
|
score. Source paths outside the app root are represented as `[external]/<filename>` so an
|
||||||
|
export does not disclose host directory structure.
|
||||||
|
|
||||||
|
## Verification
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npm run build -w @casan/control-panel-backend
|
||||||
|
npm run build -w @casan/control-panel-frontend
|
||||||
|
cd packages/casan-control-panel/backend
|
||||||
|
node --import tsx --test test/reports.test.ts
|
||||||
|
```
|
||||||
|
|
||||||
|
The report tests cover the common catalog, filtered aggregation, exact-run/no-data behavior,
|
||||||
|
query validation, HTML escaping and absence of hard-coded scoring.
|
||||||
@@ -107,9 +107,10 @@ services:
|
|||||||
environment:
|
environment:
|
||||||
CASAN_DASHBOARD_BIND: 0.0.0.0
|
CASAN_DASHBOARD_BIND: 0.0.0.0
|
||||||
CASAN_DASHBOARD_STALE_S: "315360000"
|
CASAN_DASHBOARD_STALE_S: "315360000"
|
||||||
CASAN_DASHBOARD_METRICS: /workspace/.specify/logs/cost/metrics.jsonl
|
CASAN_TELEMETRY_METRICS_LOG: /workspace/.specify/logs/cost/metrics.jsonl
|
||||||
CASAN_DASHBOARD_HTML: /workspace/docs/output/casan/central-agentops-dashboard.html
|
CASAN_DASHBOARD_HTML: /workspace/docs/output/casan/central-agentops-dashboard.html
|
||||||
CASAN_DASHBOARD_ALERTS: /workspace/.specify/agentops/alerts.log
|
CASAN_TELEMETRY_PROVIDER_LOG: /workspace/.specify/logs/level5/provider-usage.jsonl
|
||||||
|
CASAN_TELEMETRY_ALERTS_LOG: /workspace/.specify/agentops/alerts.log
|
||||||
volumes:
|
volumes:
|
||||||
- ../..:/workspace:ro
|
- ../..:/workspace:ro
|
||||||
expose:
|
expose:
|
||||||
|
|||||||
@@ -23,6 +23,17 @@ Open http://127.0.0.1:5174 — panels show REAL metrics from `.specify/logs/**`.
|
|||||||
`incidents` · `tools` · `traceability` · `drift` · `cost` · `GET /healthz` (200 fresh /
|
`incidents` · `tools` · `traceability` · `drift` · `cost` · `GET /healthz` (200 fresh /
|
||||||
503 stale — fail-loud, mirrors `dashboard-server.py`).
|
503 stale — fail-loud, mirrors `dashboard-server.py`).
|
||||||
|
|
||||||
|
Harness reports:
|
||||||
|
|
||||||
|
- `GET /api/v1/reports` — versioned H1–H7 report catalog and availability.
|
||||||
|
- `GET /api/v1/reports/h6?project=<id>&from=<ISO>&to=<ISO>&run=<id>` — H6 AgentOps
|
||||||
|
report from the same runtime metrics, provider usage and alert files as the Control Panel.
|
||||||
|
- `GET /api/v1/reports/h6/export?format=html|json&...` — standalone, print-ready HTML
|
||||||
|
or machine-readable JSON export. Both formats serialize the same filtered report object;
|
||||||
|
no maturity score is hard-coded.
|
||||||
|
- `/reports/h6` — UI report view with project, time-range and run filters plus source-level
|
||||||
|
freshness and data-quality warnings.
|
||||||
|
|
||||||
Metrics export: `GET /api/v1/metrics` provides Prometheus text exposition for
|
Metrics export: `GET /api/v1/metrics` provides Prometheus text exposition for
|
||||||
aggregate freshness, run/failure/cost/token and H4/H5/action/incident counters.
|
aggregate freshness, run/failure/cost/token and H4/H5/action/incident counters.
|
||||||
It intentionally contains no tenant, actor, trace, prompt or Evidence Pack
|
It intentionally contains no tenant, actor, trace, prompt or Evidence Pack
|
||||||
@@ -121,7 +132,11 @@ FinOps/SLO:
|
|||||||
|
|
||||||
Data sources + aggregation mirror `packages/casan-harness/tests/generate-agentops-dashboard.py`.
|
Data sources + aggregation mirror `packages/casan-harness/tests/generate-agentops-dashboard.py`.
|
||||||
App root + telemetry paths resolve via the same marker walk-up as `casan-paths.sh`
|
App root + telemetry paths resolve via the same marker walk-up as `casan-paths.sh`
|
||||||
(`.specify` or `packages/casan-harness`) and honor `CASAN_DASHBOARD_*` env overrides.
|
(`.specify` or `packages/casan-harness`). The canonical source variables are
|
||||||
|
`CASAN_TELEMETRY_METRICS_LOG`, `CASAN_TELEMETRY_PROVIDER_LOG`, and
|
||||||
|
`CASAN_TELEMETRY_ALERTS_LOG`; legacy dashboard/control-panel names remain accepted as
|
||||||
|
compatibility aliases. Freshness is calculated independently from each file's mtime using
|
||||||
|
`CASAN_DASHBOARD_STALE_S` (default `3600`).
|
||||||
|
|
||||||
## Security posture (MVP)
|
## Security posture (MVP)
|
||||||
Binds `127.0.0.1` by default. Refuses a non-loopback bind under `CASAN_PROFILE=prod` /
|
Binds `127.0.0.1` by default. Refuses a non-loopback bind under `CASAN_PROFILE=prod` /
|
||||||
|
|||||||
@@ -9,9 +9,10 @@ import { ProviderAuthModule } from './provider-auth/provider-auth.module.js';
|
|||||||
import { GoalsModule } from './goals/goals.module.js';
|
import { GoalsModule } from './goals/goals.module.js';
|
||||||
import { EvidenceModule } from './evidence/evidence.module.js';
|
import { EvidenceModule } from './evidence/evidence.module.js';
|
||||||
import { SessionController } from './session/session.controller.js';
|
import { SessionController } from './session/session.controller.js';
|
||||||
|
import { ReportsModule } from './reports/reports.module.js';
|
||||||
|
|
||||||
@Module({
|
@Module({
|
||||||
imports: [TelemetryModule, SettingsModule, KillSwitchModule, ApprovalsModule, ChatModule, ProviderAuthModule, GoalsModule, EvidenceModule],
|
imports: [TelemetryModule, SettingsModule, KillSwitchModule, ApprovalsModule, ChatModule, ProviderAuthModule, GoalsModule, EvidenceModule, ReportsModule],
|
||||||
controllers: [HealthController, SessionController],
|
controllers: [HealthController, SessionController],
|
||||||
})
|
})
|
||||||
export class AppModule {}
|
export class AppModule {}
|
||||||
|
|||||||
@@ -3,8 +3,8 @@
|
|||||||
// Mirrors packages/casan-harness/scripts/bash/casan-paths.sh: walk UP from a start dir
|
// Mirrors packages/casan-harness/scripts/bash/casan-paths.sh: walk UP from a start dir
|
||||||
// for a marker (`.specify` state dir OR `packages/casan-harness`), so the API reads the
|
// for a marker (`.specify` state dir OR `packages/casan-harness`), so the API reads the
|
||||||
// real harness telemetry whether launched from the repo, a workspace subdir, or a bundle.
|
// real harness telemetry whether launched from the repo, a workspace subdir, or a bundle.
|
||||||
// Every path is overridable by the same CASAN_DASHBOARD_* env vars the legacy
|
// Canonical CASAN_TELEMETRY_* variables are shared with harness writers; legacy
|
||||||
// dashboard-server.py honors, so console + dashboard read identical sources.
|
// dashboard/control-panel names remain accepted so upgrades do not split sources.
|
||||||
import { existsSync, statSync } from 'node:fs';
|
import { existsSync, statSync } from 'node:fs';
|
||||||
import { dirname, join, resolve } from 'node:path';
|
import { dirname, join, resolve } from 'node:path';
|
||||||
import { fileURLToPath } from 'node:url';
|
import { fileURLToPath } from 'node:url';
|
||||||
@@ -27,35 +27,74 @@ export const APP_ROOT = process.env.CASAN_APP_ROOT
|
|||||||
? resolve(process.env.CASAN_APP_ROOT)
|
? resolve(process.env.CASAN_APP_ROOT)
|
||||||
: findAppRoot(HERE);
|
: findAppRoot(HERE);
|
||||||
|
|
||||||
const env = (k: string, fallback: string) => (process.env[k] ? resolve(process.env[k]!) : join(APP_ROOT, fallback));
|
const env = (keys: string[], fallback: string) => {
|
||||||
|
const configured = keys.map((key) => process.env[key]).find((value): value is string => Boolean(value));
|
||||||
|
return configured ? resolve(configured) : join(APP_ROOT, fallback);
|
||||||
|
};
|
||||||
|
|
||||||
// Runtime telemetry (regenerated) + generated reports. Names match the harness layout.
|
// Runtime telemetry (regenerated) + generated reports. Names match the harness layout.
|
||||||
export const PATHS = {
|
export const PATHS = {
|
||||||
metrics: env('CASAN_DASHBOARD_METRICS', '.specify/logs/cost/metrics.jsonl'),
|
metrics: env(['CASAN_TELEMETRY_METRICS_LOG', 'CASAN_DASHBOARD_METRICS'], '.specify/logs/cost/metrics.jsonl'),
|
||||||
providerUsage: env('CASAN_CP_PROVIDER_USAGE', '.specify/logs/level5/provider-usage.jsonl'),
|
providerUsage: env(['CASAN_TELEMETRY_PROVIDER_LOG', 'CASAN_CP_PROVIDER_USAGE'], '.specify/logs/level5/provider-usage.jsonl'),
|
||||||
audit: env('CASAN_CP_AUDIT', '.specify/logs/audit/audit.jsonl'),
|
alerts: env(['CASAN_TELEMETRY_ALERTS_LOG', 'CASAN_DASHBOARD_ALERTS'], '.specify/agentops/alerts.log'),
|
||||||
auditHead: env('CASAN_CP_AUDIT_HEAD', '.specify/logs/audit/audit-head.txt'),
|
audit: env(['CASAN_CP_AUDIT'], '.specify/logs/audit/audit.jsonl'),
|
||||||
security: env('CASAN_CP_SECURITY', '.specify/logs/audit/security.jsonl'),
|
auditHead: env(['CASAN_CP_AUDIT_HEAD'], '.specify/logs/audit/audit-head.txt'),
|
||||||
toolRegistry: env('CASAN_CP_TOOL_REGISTRY', '.specify/logs/level5/tool-registry.jsonl'),
|
security: env(['CASAN_CP_SECURITY'], '.specify/logs/audit/security.jsonl'),
|
||||||
actionGate: env('CASAN_CP_ACTION_GATE', '.specify/logs/level5/action-gate.jsonl'),
|
toolRegistry: env(['CASAN_CP_TOOL_REGISTRY'], '.specify/logs/level5/tool-registry.jsonl'),
|
||||||
fallback: env('CASAN_CP_FALLBACK', '.specify/logs/level5/fallback.jsonl'),
|
actionGate: env(['CASAN_CP_ACTION_GATE'], '.specify/logs/level5/action-gate.jsonl'),
|
||||||
incidents: env('CASAN_CP_INCIDENTS', '.specify/logs/level5/incidents.jsonl'),
|
fallback: env(['CASAN_CP_FALLBACK'], '.specify/logs/level5/fallback.jsonl'),
|
||||||
alerts: env('CASAN_DASHBOARD_ALERTS', '.specify/agentops/alerts.log'),
|
incidents: env(['CASAN_CP_INCIDENTS'], '.specify/logs/level5/incidents.jsonl'),
|
||||||
traceDir: env('CASAN_CP_TRACE_DIR', '.specify/logs/trace'),
|
traceDir: env(['CASAN_CP_TRACE_DIR'], '.specify/logs/trace'),
|
||||||
traceEventDir: env('CASAN_CP_TRACE_EVENT_DIR', '.specify/logs/trace-events'),
|
traceEventDir: env(['CASAN_CP_TRACE_EVENT_DIR'], '.specify/logs/trace-events'),
|
||||||
traceability: env('CASAN_CP_TRACEABILITY', 'docs/output/casan/traceability-matrix.json'),
|
traceability: env(['CASAN_CP_TRACEABILITY'], 'docs/output/casan/traceability-matrix.json'),
|
||||||
drift: env('CASAN_CP_DRIFT', 'docs/output/casan/level5-evidence/09-drift-report.json'),
|
drift: env(['CASAN_CP_DRIFT'], 'docs/output/casan/level5-evidence/09-drift-report.json'),
|
||||||
businessKpi: env('CASAN_CP_KPI', 'docs/output/casan/level5-evidence/14-business-kpi-report.json'),
|
businessKpi: env(['CASAN_CP_KPI'], 'docs/output/casan/level5-evidence/14-business-kpi-report.json'),
|
||||||
benignFp: env('CASAN_CP_BENIGN_FP', 'docs/output/casan/benign-fp-report.json'),
|
benignFp: env(['CASAN_CP_BENIGN_FP'], 'docs/output/casan/benign-fp-report.json'),
|
||||||
scoringReport: env('CASAN_CP_SCORING_REPORT', 'docs/output/casan/phase3-real-run-scoring.md'),
|
scoringReport: env(['CASAN_CP_SCORING_REPORT'], 'docs/output/casan/phase3-real-run-scoring.md'),
|
||||||
approvalInbox: env('CASAN_CP_APPROVAL_INBOX', '.specify/level5/approval-inbox.json'),
|
approvalInbox: env(['CASAN_CP_APPROVAL_INBOX'], '.specify/level5/approval-inbox.json'),
|
||||||
chatAudit: env('CASAN_CP_CHAT_AUDIT', '.specify/logs/chat/chat-turns.jsonl'),
|
chatAudit: env(['CASAN_CP_CHAT_AUDIT'], '.specify/logs/chat/chat-turns.jsonl'),
|
||||||
delegationPolicy: env('CASAN_CP_DELEGATION_POLICY', 'packages/casan-harness/config/delegation-policy.yaml'),
|
delegationPolicy: env(['CASAN_CP_DELEGATION_POLICY'], 'packages/casan-harness/config/delegation-policy.yaml'),
|
||||||
selfImprove: env('CASAN_CP_SELF_IMPROVE', 'packages/casan-harness/scripts/bash/self-improve.py'),
|
selfImprove: env(['CASAN_CP_SELF_IMPROVE'], 'packages/casan-harness/scripts/bash/self-improve.py'),
|
||||||
evidencePacks: env('CASAN_CP_EVIDENCE_PACKS', 'docs/output/casan/evidence-packs'),
|
evidencePacks: env(['CASAN_CP_EVIDENCE_PACKS'], 'docs/output/casan/evidence-packs'),
|
||||||
};
|
};
|
||||||
|
|
||||||
export const STALE_AFTER_S = Number(process.env.CASAN_DASHBOARD_STALE_S ?? 3600);
|
const configuredStaleAfter = Number(process.env.CASAN_DASHBOARD_STALE_S ?? 3600);
|
||||||
|
export const STALE_AFTER_S = Number.isFinite(configuredStaleAfter) && configuredStaleAfter >= 0
|
||||||
|
? configuredStaleAfter
|
||||||
|
: 3600;
|
||||||
|
|
||||||
|
export interface SourceFreshness {
|
||||||
|
source: 'metrics' | 'provider' | 'alerts';
|
||||||
|
path: string;
|
||||||
|
present: boolean;
|
||||||
|
updated_at: string | null;
|
||||||
|
age_s: number | null;
|
||||||
|
stale: boolean;
|
||||||
|
required: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function sourceFreshness(
|
||||||
|
source: SourceFreshness['source'],
|
||||||
|
path: string,
|
||||||
|
required: boolean,
|
||||||
|
nowMs = Date.now(),
|
||||||
|
): SourceFreshness {
|
||||||
|
try {
|
||||||
|
const mtime = statSync(path).mtime;
|
||||||
|
const age = Math.max(0, Math.floor((nowMs - mtime.getTime()) / 1000));
|
||||||
|
return { source, path, present: true, updated_at: mtime.toISOString(), age_s: age, stale: age > STALE_AFTER_S, required };
|
||||||
|
} catch {
|
||||||
|
return { source, path, present: false, updated_at: null, age_s: null, stale: required, required };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function telemetryFreshness(nowMs = Date.now()): SourceFreshness[] {
|
||||||
|
return [
|
||||||
|
sourceFreshness('metrics', PATHS.metrics, true, nowMs),
|
||||||
|
sourceFreshness('provider', PATHS.providerUsage, false, nowMs),
|
||||||
|
sourceFreshness('alerts', PATHS.alerts, false, nowMs),
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
// Freshness of the primary metrics feed (mtime), used by /healthz + `stale` flags.
|
// Freshness of the primary metrics feed (mtime), used by /healthz + `stale` flags.
|
||||||
export function metricsAgeSeconds(): number | null {
|
export function metricsAgeSeconds(): number | null {
|
||||||
|
|||||||
@@ -0,0 +1,61 @@
|
|||||||
|
import type { H6Breakdown, H6Report } from './h6-report.js';
|
||||||
|
|
||||||
|
const escapeHtml = (value: unknown): string => String(value ?? '')
|
||||||
|
.replaceAll('&', '&')
|
||||||
|
.replaceAll('<', '<')
|
||||||
|
.replaceAll('>', '>')
|
||||||
|
.replaceAll('"', '"')
|
||||||
|
.replaceAll("'", ''');
|
||||||
|
|
||||||
|
const number = (value: number): string => new Intl.NumberFormat('en-US').format(value);
|
||||||
|
const money = (value: number): string => `$${value.toFixed(6)}`;
|
||||||
|
|
||||||
|
function breakdownRows(rows: H6Breakdown[]): string {
|
||||||
|
if (rows.length === 0) return '<tr><td colspan="6" class="empty">No records in the selected scope.</td></tr>';
|
||||||
|
return rows.map((row) => `<tr>
|
||||||
|
<td>${escapeHtml(row.key)}</td><td>${number(row.runs)}</td><td>${number(row.failures)}</td>
|
||||||
|
<td>${number(row.latency_avg_ms)} ms</td><td>${number(row.tokens)}</td><td>${money(row.cost_usd)}</td>
|
||||||
|
</tr>`).join('');
|
||||||
|
}
|
||||||
|
|
||||||
|
export function renderH6ReportHtml(report: H6Report): string {
|
||||||
|
const verdictClass = report.verdict === 'pass' ? 'pass' : report.verdict === 'fail' ? 'fail' : report.verdict === 'attention' ? 'attention' : 'neutral';
|
||||||
|
const scope = [
|
||||||
|
report.scope.project ? `Project: ${report.scope.project}` : 'All projects',
|
||||||
|
report.scope.run ? `Run: ${report.scope.run}` : 'All runs',
|
||||||
|
report.scope.from ? `From: ${report.scope.from}` : null,
|
||||||
|
report.scope.to ? `To: ${report.scope.to}` : null,
|
||||||
|
].filter((value): value is string => Boolean(value));
|
||||||
|
const findings = report.findings.length
|
||||||
|
? report.findings.map((finding) => `<li class="${finding.severity}"><strong>${escapeHtml(finding.code)}</strong><span>${escapeHtml(finding.message)}</span></li>`).join('')
|
||||||
|
: '<li class="info"><strong>NO_FINDINGS</strong><span>No threshold breach was detected in this scope.</span></li>';
|
||||||
|
const sourceRows = report.evidence_sources.map((source) => `<tr>
|
||||||
|
<td>${escapeHtml(source.source)}</td><td>${source.present ? 'present' : 'missing'}</td><td>${source.stale ? 'stale' : 'fresh'}</td>
|
||||||
|
<td>${source.age_s === null ? '—' : `${number(source.age_s)} s`}</td><td>${number(source.records)}</td><td><code>${escapeHtml(source.path)}</code></td>
|
||||||
|
</tr>`).join('');
|
||||||
|
const warnings = report.data_quality.warnings.length
|
||||||
|
? `<ul>${report.data_quality.warnings.map((warning) => `<li>${escapeHtml(warning)}</li>`).join('')}</ul>`
|
||||||
|
: '<p>No data-quality warning.</p>';
|
||||||
|
|
||||||
|
return `<!doctype html>
|
||||||
|
<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
||||||
|
<title>${escapeHtml(report.title)} · ${escapeHtml(report.report_id)}</title>
|
||||||
|
<style>
|
||||||
|
:root{color-scheme:light;--ink:#172033;--muted:#64748b;--line:#dbe2ec;--paper:#fff;--wash:#f4f6fa;--indigo:#4f46e5;--teal:#0f766e}
|
||||||
|
*{box-sizing:border-box}body{margin:0;background:var(--wash);color:var(--ink);font-family:Inter,ui-sans-serif,system-ui,-apple-system,"Segoe UI",sans-serif;line-height:1.45}
|
||||||
|
main{max-width:1180px;margin:0 auto;padding:42px 28px 72px}.hero{position:relative;overflow:hidden;border:1px solid #cbd5e1;border-radius:20px;background:#111827;color:#f8fafc;padding:30px;box-shadow:0 24px 50px rgba(15,23,42,.14)}
|
||||||
|
.hero:after{content:"";position:absolute;right:-90px;top:-110px;width:300px;height:300px;border-radius:999px;background:radial-gradient(circle,#6366f1 0,transparent 68%);opacity:.5}.eyebrow{font-size:11px;text-transform:uppercase;letter-spacing:.18em;color:#a5b4fc;font-weight:800}h1{font-size:34px;line-height:1.1;margin:8px 0}.scope{display:flex;gap:8px;flex-wrap:wrap;color:#cbd5e1;font-size:12px}.scope span{border:1px solid #334155;border-radius:999px;padding:6px 10px;background:#0f172a}.verdict{position:relative;z-index:1;display:inline-flex;margin-top:22px;border-radius:999px;padding:9px 14px;font-size:12px;text-transform:uppercase;letter-spacing:.12em;font-weight:900}.verdict.pass{background:#d1fae5;color:#065f46}.verdict.fail{background:#fee2e2;color:#991b1b}.verdict.attention{background:#fef3c7;color:#92400e}.verdict.neutral{background:#e2e8f0;color:#334155}
|
||||||
|
.meta{margin-top:10px;color:#94a3b8;font-size:12px}.grid{display:grid;grid-template-columns:repeat(4,minmax(0,1fr));gap:12px;margin-top:18px}.card,.section{background:var(--paper);border:1px solid var(--line);border-radius:16px;box-shadow:0 5px 14px rgba(15,23,42,.04)}.card{padding:17px}.card span{display:block;color:var(--muted);font-size:11px;text-transform:uppercase;letter-spacing:.1em;font-weight:700}.card strong{display:block;margin-top:6px;font-size:26px}.section{padding:22px;margin-top:18px}h2{font-size:16px;margin:0 0 14px}table{border-collapse:collapse;width:100%;font-size:12px}th{text-align:left;color:var(--muted);font-size:10px;text-transform:uppercase;letter-spacing:.1em;padding:9px;border-bottom:1px solid var(--line)}td{padding:10px 9px;border-bottom:1px solid #edf1f5;vertical-align:top}.empty{color:var(--muted);text-align:center;padding:24px}code{font-size:11px;word-break:break-all}.findings{list-style:none;padding:0;margin:0;display:grid;gap:9px}.findings li{display:grid;grid-template-columns:190px 1fr;gap:12px;border-left:4px solid #94a3b8;background:#f8fafc;padding:11px 13px;border-radius:8px}.findings li.warning{border-color:#f59e0b}.findings li.critical{border-color:#ef4444}.findings span{color:#475569}.quality{border-left:4px solid var(--indigo);background:#eef2ff;padding:14px 16px;border-radius:10px;font-size:13px}.quality p,.quality ul{margin:0}.quality ul{padding-left:18px}.footer{margin-top:24px;color:var(--muted);font-size:11px;text-align:center}
|
||||||
|
@media(max-width:800px){main{padding:20px 14px 50px}.grid{grid-template-columns:repeat(2,minmax(0,1fr))}.hero{padding:22px}h1{font-size:28px}.table-wrap{overflow:auto}.findings li{grid-template-columns:1fr}}
|
||||||
|
@media print{body{background:#fff}main{max-width:none;padding:0}.hero,.card,.section{box-shadow:none}.hero{background:#111827!important;-webkit-print-color-adjust:exact;print-color-adjust:exact}.section{break-inside:avoid}.footer{margin-top:10px}}
|
||||||
|
</style></head><body><main>
|
||||||
|
<section class="hero"><div class="eyebrow">CASAN assurance dossier · contract v${report.schema_version}</div><h1>${escapeHtml(report.title)}</h1><div class="scope">${scope.map((item) => `<span>${escapeHtml(item)}</span>`).join('')}</div><div class="verdict ${verdictClass}">${escapeHtml(report.verdict)}</div><div class="meta">Generated ${escapeHtml(report.generated_at)} · ${escapeHtml(report.report_id)} · Freshness ${escapeHtml(report.freshness.status)}</div></section>
|
||||||
|
<section class="grid"><div class="card"><span>Governed runs</span><strong>${number(report.summary.runs)}</strong></div><div class="card"><span>Failure rate</span><strong>${report.summary.failure_rate_pct}%</strong></div><div class="card"><span>P95 latency</span><strong>${number(report.summary.latency_ms.p95)} ms</strong></div><div class="card"><span>Provider tokens</span><strong>${number(report.summary.tokens.provider_total)}</strong></div><div class="card"><span>Actual provider cost</span><strong>${money(report.summary.cost_usd.provider_actual)}</strong></div><div class="card"><span>Estimated cost</span><strong>${money(report.summary.cost_usd.estimated)}</strong></div><div class="card"><span>Alerts</span><strong>${number(report.summary.alerts)}</strong></div><div class="card"><span>Retries</span><strong>${number(report.summary.retries)}</strong></div></section>
|
||||||
|
<section class="section"><h2>Verdict findings</h2><ul class="findings">${findings}</ul></section>
|
||||||
|
<section class="section"><h2>Evidence freshness</h2><div class="table-wrap"><table><thead><tr><th>Source</th><th>Presence</th><th>State</th><th>Age</th><th>Records</th><th>Path</th></tr></thead><tbody>${sourceRows}</tbody></table></div></section>
|
||||||
|
<section class="section"><h2>Step breakdown</h2><div class="table-wrap"><table><thead><tr><th>Step</th><th>Runs</th><th>Failures</th><th>Avg latency</th><th>Tokens</th><th>Cost</th></tr></thead><tbody>${breakdownRows(report.details.by_step)}</tbody></table></div></section>
|
||||||
|
<section class="section"><h2>Provider/model breakdown</h2><div class="table-wrap"><table><thead><tr><th>Provider · model</th><th>Calls</th><th>Failures</th><th>Avg latency</th><th>Tokens</th><th>Cost</th></tr></thead><tbody>${breakdownRows(report.details.by_provider)}</tbody></table></div></section>
|
||||||
|
<section class="section"><h2>Data quality · ${escapeHtml(report.data_quality.status)}</h2><div class="quality">${warnings}</div></section>
|
||||||
|
<div class="footer">Generated from CASAN runtime evidence. No maturity score or telemetry value is hard-coded in this document.</div>
|
||||||
|
</main></body></html>`;
|
||||||
|
}
|
||||||
@@ -0,0 +1,307 @@
|
|||||||
|
import { BadRequestException } from '@nestjs/common';
|
||||||
|
import { basename, relative } from 'node:path';
|
||||||
|
import { APP_ROOT, PATHS, STALE_AFTER_S, telemetryFreshness, type SourceFreshness } from '../common/app-root.js';
|
||||||
|
import type { HarnessReport, HarnessReportEvidenceSource, HarnessReportFinding, HarnessReportScope } from './report.contract.js';
|
||||||
|
|
||||||
|
type TelemetryRow = Record<string, unknown>;
|
||||||
|
|
||||||
|
export interface H6ReportQuery {
|
||||||
|
project: string | null;
|
||||||
|
from: string | null;
|
||||||
|
to: string | null;
|
||||||
|
run: string | null;
|
||||||
|
limit: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface H6ReportQueryParams {
|
||||||
|
project?: string;
|
||||||
|
from?: string;
|
||||||
|
to?: string;
|
||||||
|
run?: string;
|
||||||
|
limit?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface H6ReportInput {
|
||||||
|
metrics: TelemetryRow[];
|
||||||
|
provider: TelemetryRow[];
|
||||||
|
alerts: TelemetryRow[];
|
||||||
|
sourceFreshness: SourceFreshness[];
|
||||||
|
now: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface H6ReportSummary {
|
||||||
|
runs: number;
|
||||||
|
success: number;
|
||||||
|
failed: number;
|
||||||
|
degraded: number;
|
||||||
|
failure_rate_pct: number;
|
||||||
|
retries: number;
|
||||||
|
latency_ms: { average: number; p50: number; p95: number; p99: number; max: number };
|
||||||
|
tokens: { input: number; output: number; total: number; provider_total: number };
|
||||||
|
cost_usd: { provider_actual: number; estimated: number };
|
||||||
|
provider_calls: number;
|
||||||
|
alerts: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface H6Breakdown {
|
||||||
|
key: string;
|
||||||
|
runs: number;
|
||||||
|
failures: number;
|
||||||
|
latency_avg_ms: number;
|
||||||
|
tokens: number;
|
||||||
|
cost_usd: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface H6ReportDetails {
|
||||||
|
by_status: Array<{ status: string; count: number }>;
|
||||||
|
by_step: H6Breakdown[];
|
||||||
|
by_provider: H6Breakdown[];
|
||||||
|
by_cost_source: Array<{ source: string; records: number; cost_usd: number }>;
|
||||||
|
by_alert: Array<{ alert: string; count: number }>;
|
||||||
|
recent_runs: TelemetryRow[];
|
||||||
|
recent_alerts: TelemetryRow[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export type H6Report = HarnessReport<H6ReportSummary, H6ReportDetails>;
|
||||||
|
|
||||||
|
const safeFilter = /^[a-zA-Z0-9][a-zA-Z0-9._:-]{0,127}$/;
|
||||||
|
const numberValue = (value: unknown): number => typeof value === 'number' && Number.isFinite(value) ? value : 0;
|
||||||
|
const stringValue = (value: unknown, fallback: string): string => typeof value === 'string' && value.trim() ? value.trim() : fallback;
|
||||||
|
const round = (value: number, digits = 2): number => Number(value.toFixed(digits));
|
||||||
|
const configuredThreshold = (name: string, fallback: number): number => {
|
||||||
|
const parsed = Number(process.env[name] ?? fallback);
|
||||||
|
return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback;
|
||||||
|
};
|
||||||
|
const rowProject = (row: TelemetryRow): string => stringValue(row.project ?? row.project_id, 'default');
|
||||||
|
const rowRun = (row: TelemetryRow): string => stringValue(row.run_id ?? row.trace_id, 'unattributed');
|
||||||
|
const rowTime = (row: TelemetryRow): number | null => {
|
||||||
|
const timestamp = typeof row.timestamp === 'string' ? Date.parse(row.timestamp) : Number.NaN;
|
||||||
|
return Number.isFinite(timestamp) ? timestamp : null;
|
||||||
|
};
|
||||||
|
|
||||||
|
function parseDate(value: string | undefined, label: 'from' | 'to'): string | null {
|
||||||
|
if (!value) return null;
|
||||||
|
const dateOnly = /^\d{4}-\d{2}-\d{2}$/.test(value);
|
||||||
|
const normalized = dateOnly ? `${value}T${label === 'from' ? '00:00:00.000' : '23:59:59.999'}Z` : value;
|
||||||
|
const parsed = Date.parse(normalized);
|
||||||
|
if (!Number.isFinite(parsed)) throw new BadRequestException(`H6_REPORT_INVALID_${label.toUpperCase()}`);
|
||||||
|
return new Date(parsed).toISOString();
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseFilter(value: string | undefined, label: 'project' | 'run'): string | null {
|
||||||
|
if (!value) return null;
|
||||||
|
if (!safeFilter.test(value)) throw new BadRequestException(`H6_REPORT_INVALID_${label.toUpperCase()}`);
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function parseH6ReportQuery(raw: H6ReportQueryParams): H6ReportQuery {
|
||||||
|
const from = parseDate(raw.from, 'from');
|
||||||
|
const to = parseDate(raw.to, 'to');
|
||||||
|
if (from && to && Date.parse(from) > Date.parse(to)) throw new BadRequestException('H6_REPORT_INVALID_TIME_RANGE');
|
||||||
|
const parsedLimit = Number(raw.limit ?? 50);
|
||||||
|
if (!Number.isInteger(parsedLimit) || parsedLimit < 1 || parsedLimit > 200) {
|
||||||
|
throw new BadRequestException('H6_REPORT_INVALID_LIMIT');
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
project: parseFilter(raw.project, 'project'),
|
||||||
|
from,
|
||||||
|
to,
|
||||||
|
run: parseFilter(raw.run, 'run'),
|
||||||
|
limit: parsedLimit,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function filtered(rows: TelemetryRow[], query: H6ReportQuery): TelemetryRow[] {
|
||||||
|
const fromMs = query.from ? Date.parse(query.from) : null;
|
||||||
|
const toMs = query.to ? Date.parse(query.to) : null;
|
||||||
|
return rows.filter((row) => {
|
||||||
|
if (query.project && rowProject(row) !== query.project) return false;
|
||||||
|
if (query.run && rowRun(row) !== query.run) return false;
|
||||||
|
const timestamp = rowTime(row);
|
||||||
|
if (fromMs !== null && (timestamp === null || timestamp < fromMs)) return false;
|
||||||
|
if (toMs !== null && (timestamp === null || timestamp > toMs)) return false;
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function percentile(values: number[], percentileValue: number): number {
|
||||||
|
if (values.length === 0) return 0;
|
||||||
|
const sorted = [...values].sort((left, right) => left - right);
|
||||||
|
const index = Math.max(0, Math.ceil((percentileValue / 100) * sorted.length) - 1);
|
||||||
|
return sorted[index];
|
||||||
|
}
|
||||||
|
|
||||||
|
function sum(rows: TelemetryRow[], key: string): number {
|
||||||
|
return rows.reduce((total, row) => total + numberValue(row[key]), 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
function grouped(rows: TelemetryRow[], keyOf: (row: TelemetryRow) => string): H6Breakdown[] {
|
||||||
|
const groups = new Map<string, TelemetryRow[]>();
|
||||||
|
for (const row of rows) {
|
||||||
|
const key = keyOf(row);
|
||||||
|
groups.set(key, [...(groups.get(key) ?? []), row]);
|
||||||
|
}
|
||||||
|
return [...groups.entries()].map(([key, records]) => {
|
||||||
|
const latencies = records.map((row) => numberValue(row.latency_ms)).filter((value) => value > 0);
|
||||||
|
return {
|
||||||
|
key,
|
||||||
|
runs: records.length,
|
||||||
|
failures: records.filter((row) => stringValue(row.status, 'unknown') === 'failed').length,
|
||||||
|
latency_avg_ms: latencies.length ? Math.round(latencies.reduce((total, value) => total + value, 0) / latencies.length) : 0,
|
||||||
|
tokens: sum(records, 'total_tokens'),
|
||||||
|
cost_usd: round(sum(records, records.some((row) => row.cost_usd !== undefined) ? 'cost_usd' : 'cost_estimate'), 6),
|
||||||
|
};
|
||||||
|
}).sort((left, right) => right.runs - left.runs || left.key.localeCompare(right.key));
|
||||||
|
}
|
||||||
|
|
||||||
|
function counted(rows: TelemetryRow[], keyOf: (row: TelemetryRow) => string): Array<{ key: string; count: number }> {
|
||||||
|
const counts = new Map<string, number>();
|
||||||
|
for (const row of rows) {
|
||||||
|
const key = keyOf(row);
|
||||||
|
counts.set(key, (counts.get(key) ?? 0) + 1);
|
||||||
|
}
|
||||||
|
return [...counts.entries()].map(([key, countValue]) => ({ key, count: countValue }))
|
||||||
|
.sort((left, right) => right.count - left.count || left.key.localeCompare(right.key));
|
||||||
|
}
|
||||||
|
|
||||||
|
function alertTypes(row: TelemetryRow): string[] {
|
||||||
|
const structured = row.body && typeof row.body === 'object' ? (row.body as Record<string, unknown>)['alert.type'] : undefined;
|
||||||
|
if (typeof structured === 'string') return [structured];
|
||||||
|
if (Array.isArray(row.alerts)) return row.alerts.filter((value): value is string => typeof value === 'string');
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
function evidenceSources(input: H6ReportInput): HarnessReportEvidenceSource[] {
|
||||||
|
const counts: Record<SourceFreshness['source'], number> = {
|
||||||
|
metrics: input.metrics.length,
|
||||||
|
provider: input.provider.length,
|
||||||
|
alerts: input.alerts.length,
|
||||||
|
};
|
||||||
|
return input.sourceFreshness.map((source) => {
|
||||||
|
const appRelativePath = relative(APP_ROOT, source.path) || '.';
|
||||||
|
return {
|
||||||
|
...source,
|
||||||
|
path: appRelativePath.startsWith('..') ? `[external]/${basename(source.path)}` : appRelativePath,
|
||||||
|
records: counts[source.source],
|
||||||
|
};
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildH6Report(input: H6ReportInput, query: H6ReportQuery): H6Report {
|
||||||
|
const metrics = filtered(input.metrics, query);
|
||||||
|
const provider = filtered(input.provider, query);
|
||||||
|
const alerts = filtered(input.alerts, query);
|
||||||
|
const latencies = metrics.map((row) => numberValue(row.latency_ms)).filter((value) => value > 0);
|
||||||
|
const failed = metrics.filter((row) => stringValue(row.status, 'unknown') === 'failed').length;
|
||||||
|
const degraded = metrics.filter((row) => stringValue(row.status, 'unknown') === 'degraded').length;
|
||||||
|
const success = metrics.filter((row) => ['success', 'pass', 'passed'].includes(stringValue(row.status, 'unknown'))).length;
|
||||||
|
const sourceEvidence = evidenceSources(input);
|
||||||
|
const primary = sourceEvidence.find((source) => source.source === 'metrics');
|
||||||
|
const failureRate = metrics.length ? round((failed / metrics.length) * 100, 1) : 0;
|
||||||
|
const p95 = percentile(latencies, 95);
|
||||||
|
const failureThreshold = configuredThreshold('CASAN_H6_FAILURE_RATE_THRESHOLD_PCT', 5);
|
||||||
|
const p95Threshold = configuredThreshold('CASAN_H6_P95_LATENCY_THRESHOLD_MS', 5000);
|
||||||
|
const findings: HarnessReportFinding[] = [];
|
||||||
|
const warnings: string[] = [];
|
||||||
|
|
||||||
|
if (!primary?.present) findings.push({ severity: 'critical', code: 'METRICS_MISSING', message: 'The required runtime metrics source is missing.' });
|
||||||
|
else if (primary.stale) findings.push({ severity: 'warning', code: 'METRICS_STALE', message: 'The primary runtime metrics source is stale.', metric: 'age_s', value: primary.age_s ?? 'unknown', threshold: STALE_AFTER_S });
|
||||||
|
if (failureRate > failureThreshold) findings.push({ severity: 'critical', code: 'FAILURE_RATE_BREACH', message: 'Failure rate exceeds the configured H6 threshold.', metric: 'failure_rate_pct', value: failureRate, threshold: failureThreshold });
|
||||||
|
else if (failed > 0) findings.push({ severity: 'warning', code: 'FAILURES_PRESENT', message: `${failed} failed run(s) are present in the selected scope.`, metric: 'failed', value: failed });
|
||||||
|
if (p95 > p95Threshold) findings.push({ severity: 'warning', code: 'P95_LATENCY_BREACH', message: 'P95 latency exceeds the configured H6 threshold.', metric: 'p95_latency_ms', value: p95, threshold: p95Threshold });
|
||||||
|
const alertFingerprints = new Map<string, Set<string>>();
|
||||||
|
for (const row of [...metrics, ...alerts]) {
|
||||||
|
for (const alert of alertTypes(row)) {
|
||||||
|
const run = rowRun(row);
|
||||||
|
const fingerprint = `${run === 'unattributed' ? stringValue(row.timestamp, 'unknown-time') : run}:${alert}`;
|
||||||
|
const known = alertFingerprints.get(alert) ?? new Set<string>();
|
||||||
|
known.add(fingerprint);
|
||||||
|
alertFingerprints.set(alert, known);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const alertCounts = [...alertFingerprints.entries()].map(([alert, fingerprints]) => ({ alert, count: fingerprints.size }));
|
||||||
|
const alertCount = alertCounts.reduce((total, entry) => total + entry.count, 0);
|
||||||
|
if (alertCount > 0) findings.push({ severity: 'warning', code: 'ALERTS_PRESENT', message: `${alertCount} alert signal(s) require review.`, metric: 'alerts', value: alertCount });
|
||||||
|
if (degraded > 0) findings.push({ severity: 'warning', code: 'DEGRADED_RUNS_PRESENT', message: `${degraded} degraded run(s) are present in the selected scope.`, metric: 'degraded', value: degraded });
|
||||||
|
if (provider.length === 0) warnings.push('No provider usage records matched the selected scope; token and actual-cost breakdown may be incomplete.');
|
||||||
|
for (const source of sourceEvidence) {
|
||||||
|
if (source.source === 'metrics') continue;
|
||||||
|
if (!source.present) warnings.push(`Optional ${source.source} telemetry source is missing; its breakdown is unavailable.`);
|
||||||
|
else if (source.stale) warnings.push(`Optional ${source.source} telemetry source is stale; its breakdown may not reflect recent activity.`);
|
||||||
|
}
|
||||||
|
const estimatedRecords = metrics.filter((row) => stringValue(row.cost_source, 'unknown').includes('estimate')).length;
|
||||||
|
if (estimatedRecords > 0) warnings.push(`${estimatedRecords} metric record(s) use estimated rather than provider-reported cost.`);
|
||||||
|
const unattributed = [...input.metrics, ...input.provider].filter((row) => row.project === undefined && row.project_id === undefined).length;
|
||||||
|
if (unattributed > 0) warnings.push(`${unattributed} source record(s) have no project attribute and are classified as project "default".`);
|
||||||
|
|
||||||
|
const critical = findings.some((finding) => finding.severity === 'critical');
|
||||||
|
const warning = findings.some((finding) => finding.severity === 'warning');
|
||||||
|
const verdict = metrics.length === 0 && provider.length === 0 ? 'no_data' : critical ? 'fail' : warning ? 'attention' : 'pass';
|
||||||
|
const freshnessStatus = !primary?.present ? 'missing' : primary.stale ? 'stale' : 'live';
|
||||||
|
const availableProjects = [...new Set([...input.metrics, ...input.provider, ...input.alerts].map(rowProject))].sort();
|
||||||
|
const availableRuns = [...new Set([...input.metrics, ...input.provider, ...input.alerts].map(rowRun).filter((run) => run !== 'unattributed'))].sort();
|
||||||
|
const costSources = grouped(metrics, (row) => stringValue(row.cost_source, 'unknown')).map((entry) => ({
|
||||||
|
source: entry.key,
|
||||||
|
records: entry.runs,
|
||||||
|
cost_usd: entry.cost_usd,
|
||||||
|
}));
|
||||||
|
const scope: HarnessReportScope = { project: query.project, from: query.from, to: query.to, run: query.run };
|
||||||
|
|
||||||
|
return {
|
||||||
|
schema_version: 1,
|
||||||
|
report_id: `H6-${input.now.toISOString().replace(/[-:.TZ]/g, '').slice(0, 14)}`,
|
||||||
|
harness: 'H6',
|
||||||
|
title: 'H6 · AgentOps Report',
|
||||||
|
description: 'Evidence-backed runtime, token, cost, failure, retry and alert telemetry.',
|
||||||
|
generated_at: input.now.toISOString(),
|
||||||
|
scope,
|
||||||
|
verdict,
|
||||||
|
verdict_reasons: findings.map((finding) => finding.code),
|
||||||
|
freshness: { status: freshnessStatus, stale_after_s: STALE_AFTER_S, primary_age_s: primary?.age_s ?? null, sources: sourceEvidence },
|
||||||
|
summary: {
|
||||||
|
runs: metrics.length,
|
||||||
|
success,
|
||||||
|
failed,
|
||||||
|
degraded,
|
||||||
|
failure_rate_pct: failureRate,
|
||||||
|
retries: sum(metrics, 'retry_count'),
|
||||||
|
latency_ms: {
|
||||||
|
average: latencies.length ? Math.round(latencies.reduce((total, value) => total + value, 0) / latencies.length) : 0,
|
||||||
|
p50: percentile(latencies, 50),
|
||||||
|
p95,
|
||||||
|
p99: percentile(latencies, 99),
|
||||||
|
max: latencies.length ? Math.max(...latencies) : 0,
|
||||||
|
},
|
||||||
|
tokens: { input: sum(metrics, 'input_tokens'), output: sum(metrics, 'output_tokens'), total: sum(metrics, 'total_tokens'), provider_total: sum(provider, 'total_tokens') },
|
||||||
|
cost_usd: { provider_actual: round(sum(provider, 'cost_usd'), 6), estimated: round(sum(metrics, 'cost_estimate'), 6) },
|
||||||
|
provider_calls: provider.length,
|
||||||
|
alerts: alertCount,
|
||||||
|
},
|
||||||
|
thresholds: { failure_rate_pct: failureThreshold, p95_latency_ms: p95Threshold, freshness_age_s: STALE_AFTER_S },
|
||||||
|
findings,
|
||||||
|
evidence_sources: sourceEvidence,
|
||||||
|
data_quality: { status: !primary?.present ? 'insufficient' : warnings.length ? 'partial' : 'complete', warnings },
|
||||||
|
available_filters: { projects: availableProjects, runs: availableRuns },
|
||||||
|
details: {
|
||||||
|
by_status: counted(metrics, (row) => stringValue(row.status, 'unknown')).map(({ key, count: countValue }) => ({ status: key, count: countValue })),
|
||||||
|
by_step: grouped(metrics, (row) => stringValue(row.step, 'unknown-step')),
|
||||||
|
by_provider: grouped(provider, (row) => `${stringValue(row.provider, 'unknown-provider')} · ${stringValue(row.model, 'unknown-model')}`),
|
||||||
|
by_cost_source: costSources,
|
||||||
|
by_alert: alertCounts.sort((left, right) => right.count - left.count || left.alert.localeCompare(right.alert)),
|
||||||
|
recent_runs: [...metrics].slice(-query.limit).reverse(),
|
||||||
|
recent_alerts: [...alerts].slice(-query.limit).reverse(),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function defaultH6ReportInput(now = new Date()): H6ReportInput {
|
||||||
|
return {
|
||||||
|
metrics: [],
|
||||||
|
provider: [],
|
||||||
|
alerts: [],
|
||||||
|
sourceFreshness: telemetryFreshness(now.getTime()),
|
||||||
|
now,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export const H6_REPORT_PATHS = PATHS;
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
import type { SourceFreshness } from '../common/app-root.js';
|
||||||
|
|
||||||
|
export const HARNESS_REPORT_DEFINITIONS = [
|
||||||
|
{ id: 'H1', slug: 'context', title: 'H1 · Context', description: 'Context selection, prompt contract and risk classification' },
|
||||||
|
{ id: 'H2', slug: 'tool', title: 'H2 · Tool', description: 'Allowlisted tools, actions and execution boundaries' },
|
||||||
|
{ id: 'H3', slug: 'evaluation', title: 'H3 · Evaluation', description: 'Quality evaluation, traceability and acceptance evidence' },
|
||||||
|
{ id: 'H4', slug: 'security', title: 'H4 · Security', description: 'Input, output and artifact security controls' },
|
||||||
|
{ id: 'H5', slug: 'governance', title: 'H5 · Governance', description: 'Policy decisions, approvals and audit integrity' },
|
||||||
|
{ id: 'H6', slug: 'agentops', title: 'H6 · AgentOps', description: 'Runtime, token, cost, failure and alert telemetry' },
|
||||||
|
{ id: 'H7', slug: 'orchestration', title: 'H7 · Orchestration', description: 'Final outcome, certification and rollback evidence' },
|
||||||
|
] as const;
|
||||||
|
|
||||||
|
export type HarnessReportId = typeof HARNESS_REPORT_DEFINITIONS[number]['id'];
|
||||||
|
export type HarnessReportVerdict = 'pass' | 'attention' | 'fail' | 'no_data';
|
||||||
|
export type ReportExportFormat = 'json' | 'html';
|
||||||
|
|
||||||
|
export interface HarnessReportScope {
|
||||||
|
project: string | null;
|
||||||
|
from: string | null;
|
||||||
|
to: string | null;
|
||||||
|
run: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface HarnessReportFinding {
|
||||||
|
severity: 'info' | 'warning' | 'critical';
|
||||||
|
code: string;
|
||||||
|
message: string;
|
||||||
|
metric?: string;
|
||||||
|
value?: number | string;
|
||||||
|
threshold?: number | string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface HarnessReportEvidenceSource extends SourceFreshness {
|
||||||
|
records: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface HarnessReport<TSummary, TDetails> {
|
||||||
|
schema_version: 1;
|
||||||
|
report_id: string;
|
||||||
|
harness: HarnessReportId;
|
||||||
|
title: string;
|
||||||
|
description: string;
|
||||||
|
generated_at: string;
|
||||||
|
scope: HarnessReportScope;
|
||||||
|
verdict: HarnessReportVerdict;
|
||||||
|
verdict_reasons: string[];
|
||||||
|
freshness: {
|
||||||
|
status: 'live' | 'stale' | 'missing';
|
||||||
|
stale_after_s: number;
|
||||||
|
primary_age_s: number | null;
|
||||||
|
sources: HarnessReportEvidenceSource[];
|
||||||
|
};
|
||||||
|
summary: TSummary;
|
||||||
|
thresholds: Record<string, number | string | boolean>;
|
||||||
|
findings: HarnessReportFinding[];
|
||||||
|
evidence_sources: HarnessReportEvidenceSource[];
|
||||||
|
data_quality: {
|
||||||
|
status: 'complete' | 'partial' | 'insufficient';
|
||||||
|
warnings: string[];
|
||||||
|
};
|
||||||
|
available_filters: {
|
||||||
|
projects: string[];
|
||||||
|
runs: string[];
|
||||||
|
};
|
||||||
|
details: TDetails;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface HarnessReportCatalogEntry {
|
||||||
|
id: HarnessReportId;
|
||||||
|
slug: string;
|
||||||
|
title: string;
|
||||||
|
description: string;
|
||||||
|
contract_version: 1;
|
||||||
|
endpoint: string;
|
||||||
|
availability: 'implemented' | 'contract_ready';
|
||||||
|
}
|
||||||
|
|
||||||
|
export const HARNESS_REPORT_CATALOG: HarnessReportCatalogEntry[] = HARNESS_REPORT_DEFINITIONS.map((definition) => ({
|
||||||
|
...definition,
|
||||||
|
contract_version: 1,
|
||||||
|
endpoint: `/api/v1/reports/${definition.id.toLowerCase()}`,
|
||||||
|
availability: definition.id === 'H6' ? 'implemented' : 'contract_ready',
|
||||||
|
}));
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
import { BadRequestException, Controller, Get, Header, Inject, Query, Res } from '@nestjs/common';
|
||||||
|
import type { Response } from 'express';
|
||||||
|
import { ok } from '../common/api-response.js';
|
||||||
|
import { parseH6ReportQuery, type H6ReportQueryParams } from './h6-report.js';
|
||||||
|
import { ReportsService } from './reports.service.js';
|
||||||
|
|
||||||
|
interface H6QueryParams extends H6ReportQueryParams {
|
||||||
|
format?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Controller('api/v1/reports')
|
||||||
|
export class ReportsController {
|
||||||
|
constructor(@Inject(ReportsService) private readonly reports: ReportsService) {}
|
||||||
|
|
||||||
|
@Get()
|
||||||
|
catalog() {
|
||||||
|
return ok(this.reports.catalog());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Get('h6')
|
||||||
|
h6(@Query() raw: H6QueryParams) {
|
||||||
|
return ok(this.reports.h6(parseH6ReportQuery(raw)));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Get('h6/export')
|
||||||
|
@Header('Cache-Control', 'no-store')
|
||||||
|
exportH6(@Query() raw: H6QueryParams, @Res() response: Response) {
|
||||||
|
const format = raw.format ?? 'json';
|
||||||
|
if (format !== 'json' && format !== 'html') throw new BadRequestException('H6_REPORT_INVALID_FORMAT');
|
||||||
|
const report = this.reports.h6(parseH6ReportQuery(raw));
|
||||||
|
const body = this.reports.serializeH6(report, format);
|
||||||
|
const stamp = report.generated_at.slice(0, 10);
|
||||||
|
response.type(format === 'html' ? 'text/html; charset=utf-8' : 'application/json; charset=utf-8');
|
||||||
|
response.setHeader('Content-Disposition', `attachment; filename="casan-h6-report-${stamp}.${format}"`);
|
||||||
|
response.send(body);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
import { Module } from '@nestjs/common';
|
||||||
|
import { ReportsController } from './reports.controller.js';
|
||||||
|
import { ReportsService } from './reports.service.js';
|
||||||
|
|
||||||
|
@Module({
|
||||||
|
controllers: [ReportsController],
|
||||||
|
providers: [ReportsService],
|
||||||
|
exports: [ReportsService],
|
||||||
|
})
|
||||||
|
export class ReportsModule {}
|
||||||
@@ -0,0 +1,28 @@
|
|||||||
|
import { Injectable } from '@nestjs/common';
|
||||||
|
import { PATHS, telemetryFreshness } from '../common/app-root.js';
|
||||||
|
import { readJsonl } from '../telemetry/telemetry.reader.js';
|
||||||
|
import { buildH6Report, type H6Report, type H6ReportQuery } from './h6-report.js';
|
||||||
|
import { renderH6ReportHtml } from './h6-report.html.js';
|
||||||
|
import { HARNESS_REPORT_CATALOG } from './report.contract.js';
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class ReportsService {
|
||||||
|
catalog() {
|
||||||
|
return { schema_version: 1, reports: HARNESS_REPORT_CATALOG };
|
||||||
|
}
|
||||||
|
|
||||||
|
h6(query: H6ReportQuery): H6Report {
|
||||||
|
const now = new Date();
|
||||||
|
return buildH6Report({
|
||||||
|
metrics: readJsonl(PATHS.metrics),
|
||||||
|
provider: readJsonl(PATHS.providerUsage),
|
||||||
|
alerts: readJsonl(PATHS.alerts),
|
||||||
|
sourceFreshness: telemetryFreshness(now.getTime()),
|
||||||
|
now,
|
||||||
|
}, query);
|
||||||
|
}
|
||||||
|
|
||||||
|
serializeH6(report: H6Report, format: 'json' | 'html'): string {
|
||||||
|
return format === 'html' ? renderH6ReportHtml(report) : `${JSON.stringify(report, null, 2)}\n`;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -5,7 +5,7 @@ import { existsSync, readFileSync, statSync } from 'node:fs';
|
|||||||
import { execFileSync } from 'node:child_process';
|
import { execFileSync } from 'node:child_process';
|
||||||
import { join, relative } from 'node:path';
|
import { join, relative } from 'node:path';
|
||||||
import { Injectable } from '@nestjs/common';
|
import { Injectable } from '@nestjs/common';
|
||||||
import { APP_ROOT, PATHS, isStale, metricsAgeSeconds, STALE_AFTER_S } from '../common/app-root.js';
|
import { APP_ROOT, PATHS, isStale, metricsAgeSeconds, STALE_AFTER_S, telemetryFreshness } from '../common/app-root.js';
|
||||||
import { readJsonl, readJson, readHead, readTrace } from './telemetry.reader.js';
|
import { readJsonl, readJson, readHead, readTrace } from './telemetry.reader.js';
|
||||||
|
|
||||||
type Row = Record<string, any>;
|
type Row = Record<string, any>;
|
||||||
@@ -217,7 +217,7 @@ export function buildChatLoopWidget(
|
|||||||
export class TelemetryService {
|
export class TelemetryService {
|
||||||
private freshness() {
|
private freshness() {
|
||||||
const age = metricsAgeSeconds();
|
const age = metricsAgeSeconds();
|
||||||
return { stale: isStale(), age_s: age, stale_after_s: STALE_AFTER_S };
|
return { stale: isStale(), age_s: age, stale_after_s: STALE_AFTER_S, sources: telemetryFreshness() };
|
||||||
}
|
}
|
||||||
|
|
||||||
overview() {
|
overview() {
|
||||||
|
|||||||
@@ -0,0 +1,119 @@
|
|||||||
|
import { test } from 'node:test';
|
||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import { BadRequestException } from '@nestjs/common';
|
||||||
|
import { mkdtempSync, utimesSync, writeFileSync } from 'node:fs';
|
||||||
|
import { tmpdir } from 'node:os';
|
||||||
|
import { join } from 'node:path';
|
||||||
|
import { sourceFreshness } from '../src/common/app-root.js';
|
||||||
|
import { buildH6Report, parseH6ReportQuery, type H6ReportInput } from '../src/reports/h6-report.js';
|
||||||
|
import { renderH6ReportHtml } from '../src/reports/h6-report.html.js';
|
||||||
|
import { HARNESS_REPORT_CATALOG } from '../src/reports/report.contract.js';
|
||||||
|
|
||||||
|
const NOW = new Date('2026-07-20T12:00:00.000Z');
|
||||||
|
|
||||||
|
function fixture(): H6ReportInput {
|
||||||
|
return {
|
||||||
|
now: NOW,
|
||||||
|
metrics: [
|
||||||
|
{ timestamp: '2026-07-19T08:00:00Z', trace_id: 'run-a', project: 'okr', status: 'success', step: 'generate', latency_ms: 100, input_tokens: 10, output_tokens: 20, total_tokens: 30, cost_estimate: 0.001, cost_source: 'word_count_estimate', retry_count: 0, alerts: [] },
|
||||||
|
{ timestamp: '2026-07-19T09:00:00Z', trace_id: 'run-b', project: 'okr', status: 'failed', step: 'test', latency_ms: 9000, input_tokens: 20, output_tokens: 5, total_tokens: 25, cost_estimate: 0.002, cost_source: 'provider_telemetry', retry_count: 2, alerts: ['high-latency', 'execution-failed'] },
|
||||||
|
{ timestamp: '2026-07-18T09:00:00Z', trace_id: 'run-c', project: 'desk', status: 'success', step: 'test', latency_ms: 250, total_tokens: 10, cost_estimate: 0.0001, cost_source: 'word_count_estimate' },
|
||||||
|
],
|
||||||
|
provider: [
|
||||||
|
{ timestamp: '2026-07-19T08:00:00Z', run_id: 'run-a', project: 'okr', provider: 'ollama', model: 'ornith:9b', status: 'success', step: 'generate', latency_ms: 80, total_tokens: 28, cost_usd: 0 },
|
||||||
|
{ timestamp: '2026-07-18T09:00:00Z', run_id: 'run-c', project: 'desk', provider: 'openai', model: 'gpt-test', status: 'success', step: 'test', latency_ms: 200, total_tokens: 10, cost_usd: 0.01 },
|
||||||
|
],
|
||||||
|
alerts: [
|
||||||
|
{ timestamp: '2026-07-19T09:00:00Z', trace_id: 'run-b', project: 'okr', body: { 'alert.type': 'execution-failed' } },
|
||||||
|
],
|
||||||
|
sourceFreshness: [
|
||||||
|
{ source: 'metrics', path: '/workspace/.specify/logs/cost/metrics.jsonl', present: true, updated_at: '2026-07-20T11:59:00Z', age_s: 60, stale: false, required: true },
|
||||||
|
{ source: 'provider', path: '/workspace/.specify/logs/level5/provider-usage.jsonl', present: true, updated_at: '2026-07-20T11:58:00Z', age_s: 120, stale: false, required: false },
|
||||||
|
{ source: 'alerts', path: '/workspace/.specify/agentops/alerts.log', present: true, updated_at: '2026-07-20T11:57:00Z', age_s: 180, stale: false, required: false },
|
||||||
|
],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
test('H1-H7 share one versioned report catalog contract', () => {
|
||||||
|
assert.deepEqual(HARNESS_REPORT_CATALOG.map((entry) => entry.id), ['H1', 'H2', 'H3', 'H4', 'H5', 'H6', 'H7']);
|
||||||
|
assert.ok(HARNESS_REPORT_CATALOG.every((entry) => entry.contract_version === 1));
|
||||||
|
assert.equal(HARNESS_REPORT_CATALOG.find((entry) => entry.id === 'H6')?.availability, 'implemented');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('telemetry freshness is source-specific and missing required data fails loud', () => {
|
||||||
|
const directory = mkdtempSync(join(tmpdir(), 'casan-h6-'));
|
||||||
|
const metricsPath = join(directory, 'metrics.jsonl');
|
||||||
|
writeFileSync(metricsPath, '{}\n');
|
||||||
|
const oldTime = new Date(NOW.getTime() - 7200 * 1000);
|
||||||
|
utimesSync(metricsPath, oldTime, oldTime);
|
||||||
|
const staleMetrics = sourceFreshness('metrics', metricsPath, true, NOW.getTime());
|
||||||
|
const missingMetrics = sourceFreshness('metrics', join(directory, 'missing.jsonl'), true, NOW.getTime());
|
||||||
|
assert.equal(staleMetrics.present, true);
|
||||||
|
assert.equal(staleMetrics.age_s, 7200);
|
||||||
|
assert.equal(staleMetrics.stale, true);
|
||||||
|
assert.deepEqual(
|
||||||
|
{ present: missingMetrics.present, stale: missingMetrics.stale, required: missingMetrics.required },
|
||||||
|
{ present: false, stale: true, required: true },
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('H6 report filters project/time/run and aggregates measured evidence', () => {
|
||||||
|
const query = parseH6ReportQuery({ project: 'okr', from: '2026-07-19', to: '2026-07-19' });
|
||||||
|
const report = buildH6Report(fixture(), query);
|
||||||
|
assert.equal(report.schema_version, 1);
|
||||||
|
assert.equal(report.harness, 'H6');
|
||||||
|
assert.equal(report.summary.runs, 2);
|
||||||
|
assert.equal(report.summary.failed, 1);
|
||||||
|
assert.equal(report.summary.retries, 2);
|
||||||
|
assert.equal(report.summary.latency_ms.p50, 100);
|
||||||
|
assert.equal(report.summary.latency_ms.p95, 9000);
|
||||||
|
assert.equal(report.summary.tokens.provider_total, 28);
|
||||||
|
assert.equal(report.summary.alerts, 2);
|
||||||
|
assert.deepEqual(report.details.by_alert, [
|
||||||
|
{ alert: 'execution-failed', count: 1 },
|
||||||
|
{ alert: 'high-latency', count: 1 },
|
||||||
|
]);
|
||||||
|
assert.equal(report.available_filters.projects.length, 2);
|
||||||
|
assert.equal(report.freshness.status, 'live');
|
||||||
|
assert.equal(report.verdict, 'fail');
|
||||||
|
assert.ok(report.findings.some((finding) => finding.code === 'FAILURE_RATE_BREACH'));
|
||||||
|
});
|
||||||
|
|
||||||
|
test('H6 report supports exact run filtering and truthful no-data verdict', () => {
|
||||||
|
const selected = buildH6Report(fixture(), parseH6ReportQuery({ run: 'run-a' }));
|
||||||
|
assert.equal(selected.summary.runs, 1);
|
||||||
|
assert.equal(selected.summary.provider_calls, 1);
|
||||||
|
assert.equal(selected.verdict, 'pass');
|
||||||
|
const empty = buildH6Report(fixture(), parseH6ReportQuery({ run: 'not-found' }));
|
||||||
|
assert.equal(empty.verdict, 'no_data');
|
||||||
|
assert.equal(empty.summary.runs, 0);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('H6 query validation rejects traversal, inverted dates and invalid limits', () => {
|
||||||
|
assert.throws(() => parseH6ReportQuery({ project: '../secret' }), BadRequestException);
|
||||||
|
assert.throws(() => parseH6ReportQuery({ from: '2026-07-20', to: '2026-07-19' }), BadRequestException);
|
||||||
|
assert.throws(() => parseH6ReportQuery({ limit: '5000' }), BadRequestException);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('H6 report makes stale optional sources explicit in data quality', () => {
|
||||||
|
const input = fixture();
|
||||||
|
input.sourceFreshness[1] = { ...input.sourceFreshness[1], stale: true, age_s: 7200 };
|
||||||
|
input.sourceFreshness[2] = { ...input.sourceFreshness[2], present: false, updated_at: null, age_s: null, stale: false };
|
||||||
|
const report = buildH6Report(input, parseH6ReportQuery({ run: 'run-a' }));
|
||||||
|
assert.equal(report.data_quality.status, 'partial');
|
||||||
|
assert.ok(report.data_quality.warnings.some((warning) => warning.includes('provider telemetry source is stale')));
|
||||||
|
assert.ok(report.data_quality.warnings.some((warning) => warning.includes('alerts telemetry source is missing')));
|
||||||
|
});
|
||||||
|
|
||||||
|
test('HTML export is standalone, escaped and contains no hard-coded maturity score', () => {
|
||||||
|
const input = fixture();
|
||||||
|
input.metrics[0].step = '<script>alert(1)</script>';
|
||||||
|
const report = buildH6Report(input, parseH6ReportQuery({ project: 'okr' }));
|
||||||
|
const html = renderH6ReportHtml(report);
|
||||||
|
assert.match(html, /^<!doctype html>/);
|
||||||
|
assert.match(html, /H6 · AgentOps Report/);
|
||||||
|
assert.match(html, /<script>alert\(1\)<\/script>/);
|
||||||
|
assert.doesNotMatch(html, /<script>alert\(1\)<\/script>/);
|
||||||
|
assert.doesNotMatch(html, /Average\s+\d|\/100|218 core tests/i);
|
||||||
|
assert.match(html, /No maturity score or telemetry value is hard-coded/);
|
||||||
|
});
|
||||||
@@ -13,6 +13,7 @@ import { CommandCenter } from './pages/CommandCenter';
|
|||||||
import { Chat } from './pages/Chat';
|
import { Chat } from './pages/Chat';
|
||||||
import { Goals } from './pages/Goals';
|
import { Goals } from './pages/Goals';
|
||||||
import { EvidencePacks } from './pages/EvidencePacks';
|
import { EvidencePacks } from './pages/EvidencePacks';
|
||||||
|
import { H6ReportPage } from './pages/H6Report';
|
||||||
|
|
||||||
export default function App() {
|
export default function App() {
|
||||||
return (
|
return (
|
||||||
@@ -31,6 +32,7 @@ export default function App() {
|
|||||||
<Route path="/chat" element={<Chat />} />
|
<Route path="/chat" element={<Chat />} />
|
||||||
<Route path="/goals" element={<Goals />} />
|
<Route path="/goals" element={<Goals />} />
|
||||||
<Route path="/evidence-packs" element={<EvidencePacks />} />
|
<Route path="/evidence-packs" element={<EvidencePacks />} />
|
||||||
|
<Route path="/reports/h6" element={<H6ReportPage />} />
|
||||||
<Route path="*" element={<Navigate to="/" replace />} />
|
<Route path="*" element={<Navigate to="/" replace />} />
|
||||||
</Routes>
|
</Routes>
|
||||||
</AppLayout>
|
</AppLayout>
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ const PAGE_COPY: Record<string, { title: string; eyebrow: string }> = {
|
|||||||
'/incidents': { title: 'Incident response', eyebrow: 'Containment' },
|
'/incidents': { title: 'Incident response', eyebrow: 'Containment' },
|
||||||
'/traceability': { title: 'Traceability', eyebrow: 'Evidence graph' },
|
'/traceability': { title: 'Traceability', eyebrow: 'Evidence graph' },
|
||||||
'/finops': { title: 'FinOps & SLO', eyebrow: 'Model economics' },
|
'/finops': { title: 'FinOps & SLO', eyebrow: 'Model economics' },
|
||||||
|
'/reports/h6': { title: 'H6 AgentOps report', eyebrow: 'Assurance dossier' },
|
||||||
'/approvals': { title: 'Approval inbox', eyebrow: 'Human-in-the-loop' },
|
'/approvals': { title: 'Approval inbox', eyebrow: 'Human-in-the-loop' },
|
||||||
'/settings': { title: 'Governed settings', eyebrow: 'Control plane' },
|
'/settings': { title: 'Governed settings', eyebrow: 'Control plane' },
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
import type { ReactNode } from 'react';
|
import type { ReactNode } from 'react';
|
||||||
import { NavLink } from 'react-router-dom';
|
import { NavLink } from 'react-router-dom';
|
||||||
|
|
||||||
type IconName = 'grid' | 'command' | 'chat' | 'goal' | 'runs' | 'shield' | 'governance' | 'incident' | 'trace' | 'evidence' | 'coins' | 'approval' | 'settings';
|
type IconName = 'grid' | 'command' | 'chat' | 'goal' | 'runs' | 'shield' | 'governance' | 'incident' | 'trace' | 'evidence' | 'report' | 'coins' | 'approval' | 'settings';
|
||||||
|
|
||||||
interface NavItem { to: string; label: string; icon: IconName; }
|
interface NavItem { to: string; label: string; icon: IconName; }
|
||||||
|
|
||||||
@@ -19,6 +19,7 @@ const NAVIGATION: Array<{ label: string; items: NavItem[] }> = [
|
|||||||
{ to: '/incidents', label: 'Incidents', icon: 'incident' },
|
{ to: '/incidents', label: 'Incidents', icon: 'incident' },
|
||||||
{ to: '/traceability', label: 'Traceability', icon: 'trace' },
|
{ to: '/traceability', label: 'Traceability', icon: 'trace' },
|
||||||
{ to: '/evidence-packs', label: 'Evidence packs', icon: 'evidence' },
|
{ to: '/evidence-packs', label: 'Evidence packs', icon: 'evidence' },
|
||||||
|
{ to: '/reports/h6', label: 'H6 AgentOps report', icon: 'report' },
|
||||||
] },
|
] },
|
||||||
{ label: 'Control', items: [
|
{ label: 'Control', items: [
|
||||||
{ to: '/finops', label: 'FinOps & SLO', icon: 'coins' },
|
{ to: '/finops', label: 'FinOps & SLO', icon: 'coins' },
|
||||||
@@ -41,6 +42,7 @@ function Icon({ name }: { name: IconName }) {
|
|||||||
incident: <><path d="M10.3 3.3 2.7 17a2 2 0 0 0 1.7 3h15.2a2 2 0 0 0 1.7-3L13.7 3.3a2 2 0 0 0-3.4 0Z" /><path d="M12 9v4M12 17h.01" /></>,
|
incident: <><path d="M10.3 3.3 2.7 17a2 2 0 0 0 1.7 3h15.2a2 2 0 0 0 1.7-3L13.7 3.3a2 2 0 0 0-3.4 0Z" /><path d="M12 9v4M12 17h.01" /></>,
|
||||||
trace: <><circle cx="6" cy="6" r="3" /><circle cx="18" cy="18" r="3" /><circle cx="18" cy="6" r="3" /><path d="m8.6 7.5 6.8 3M9 6h6" /></>,
|
trace: <><circle cx="6" cy="6" r="3" /><circle cx="18" cy="18" r="3" /><circle cx="18" cy="6" r="3" /><path d="m8.6 7.5 6.8 3M9 6h6" /></>,
|
||||||
evidence: <><path d="M7 3h7l3 3v15H7a2 2 0 0 1-2-2V5a2 2 0 0 1 2-2Z" /><path d="M14 3v4h4M8.5 12h7M8.5 16h5" /><path d="m9 8.5 1 1 2-2" /></>,
|
evidence: <><path d="M7 3h7l3 3v15H7a2 2 0 0 1-2-2V5a2 2 0 0 1 2-2Z" /><path d="M14 3v4h4M8.5 12h7M8.5 16h5" /><path d="m9 8.5 1 1 2-2" /></>,
|
||||||
|
report: <><path d="M4 20V10M10 20V4M16 20v-7M22 20H2" /><path d="M3 10h2M9 4h2M15 13h2" /><path d="m16 7 2 2 4-5" /></>,
|
||||||
coins: <><ellipse cx="12" cy="5" rx="7" ry="3" /><path d="M5 5v7c0 1.7 3.1 3 7 3s7-1.3 7-3V5M5 12v7c0 1.7 3.1 3 7 3s7-1.3 7-3v-7" /></>,
|
coins: <><ellipse cx="12" cy="5" rx="7" ry="3" /><path d="M5 5v7c0 1.7 3.1 3 7 3s7-1.3 7-3V5M5 12v7c0 1.7 3.1 3 7 3s7-1.3 7-3v-7" /></>,
|
||||||
approval: <><path d="M9 11 11 13l4-4" /><path d="M12 22c5-2.1 8-5.3 8-10V5l-8-3-8 3v7c0 4.7 3 7.9 8 10Z" /></>,
|
approval: <><path d="M9 11 11 13l4-4" /><path d="M12 22c5-2.1 8-5.3 8-10V5l-8-3-8 3v7c0 4.7 3 7.9 8 10Z" /></>,
|
||||||
settings: <><circle cx="12" cy="12" r="3" /><path d="M19.4 15a1.7 1.7 0 0 0 .3 1.9l.1.1-2 2-.1-.1a1.7 1.7 0 0 0-1.9-.3 1.7 1.7 0 0 0-1 1.5v.2h-2.8v-.2a1.7 1.7 0 0 0-1-1.5 1.7 1.7 0 0 0-1.9.3l-.1.1-2-2 .1-.1A1.7 1.7 0 0 0 7.4 15a1.7 1.7 0 0 0-1.5-1H5.7v-2.8h.2a1.7 1.7 0 0 0 1.5-1 1.7 1.7 0 0 0-.3-1.9L7 8.2l2-2 .1.1a1.7 1.7 0 0 0 1.9.3 1.7 1.7 0 0 0 1-1.5v-.2h2.8v.2a1.7 1.7 0 0 0 1 1.5 1.7 1.7 0 0 0 1.9-.3l.1-.1 2 2-.1.1a1.7 1.7 0 0 0-.3 1.9 1.7 1.7 0 0 0 1.5 1h.2V14h-.2a1.7 1.7 0 0 0-1.5 1Z" /></>,
|
settings: <><circle cx="12" cy="12" r="3" /><path d="M19.4 15a1.7 1.7 0 0 0 .3 1.9l.1.1-2 2-.1-.1a1.7 1.7 0 0 0-1.9-.3 1.7 1.7 0 0 0-1 1.5v.2h-2.8v-.2a1.7 1.7 0 0 0-1-1.5 1.7 1.7 0 0 0-1.9.3l-.1.1-2-2 .1-.1A1.7 1.7 0 0 0 7.4 15a1.7 1.7 0 0 0-1.5-1H5.7v-2.8h.2a1.7 1.7 0 0 0 1.5-1 1.7 1.7 0 0 0-.3-1.9L7 8.2l2-2 .1.1a1.7 1.7 0 0 0 1.9.3 1.7 1.7 0 0 0 1-1.5v-.2h2.8v.2a1.7 1.7 0 0 0 1 1.5 1.7 1.7 0 0 0 1.9-.3l.1-.1 2 2-.1.1a1.7 1.7 0 0 0-.3 1.9 1.7 1.7 0 0 0 1.5 1h.2V14h-.2a1.7 1.7 0 0 0-1.5 1Z" /></>,
|
||||||
|
|||||||
@@ -21,7 +21,92 @@ async function post<T>(path: string, body: unknown, headers: Record<string, stri
|
|||||||
return res.data.data as T;
|
return res.data.data as T;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface Freshness { stale: boolean; age_s: number | null; stale_after_s: number }
|
export interface TelemetrySourceFreshness {
|
||||||
|
source: 'metrics' | 'provider' | 'alerts';
|
||||||
|
path: string;
|
||||||
|
present: boolean;
|
||||||
|
updated_at: string | null;
|
||||||
|
age_s: number | null;
|
||||||
|
stale: boolean;
|
||||||
|
required: boolean;
|
||||||
|
records?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface Freshness { stale: boolean; age_s: number | null; stale_after_s: number; sources?: TelemetrySourceFreshness[] }
|
||||||
|
|
||||||
|
export type HarnessReportId = 'H1' | 'H2' | 'H3' | 'H4' | 'H5' | 'H6' | 'H7';
|
||||||
|
export type HarnessReportVerdict = 'pass' | 'attention' | 'fail' | 'no_data';
|
||||||
|
|
||||||
|
export interface H6ReportQuery {
|
||||||
|
project?: string;
|
||||||
|
from?: string;
|
||||||
|
to?: string;
|
||||||
|
run?: string;
|
||||||
|
limit?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface H6ReportFinding {
|
||||||
|
severity: 'info' | 'warning' | 'critical';
|
||||||
|
code: string;
|
||||||
|
message: string;
|
||||||
|
metric?: string;
|
||||||
|
value?: number | string;
|
||||||
|
threshold?: number | string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface H6Breakdown {
|
||||||
|
key: string;
|
||||||
|
runs: number;
|
||||||
|
failures: number;
|
||||||
|
latency_avg_ms: number;
|
||||||
|
tokens: number;
|
||||||
|
cost_usd: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface H6Report {
|
||||||
|
schema_version: 1;
|
||||||
|
report_id: string;
|
||||||
|
harness: 'H6';
|
||||||
|
title: string;
|
||||||
|
description: string;
|
||||||
|
generated_at: string;
|
||||||
|
scope: { project: string | null; from: string | null; to: string | null; run: string | null };
|
||||||
|
verdict: HarnessReportVerdict;
|
||||||
|
verdict_reasons: string[];
|
||||||
|
freshness: {
|
||||||
|
status: 'live' | 'stale' | 'missing';
|
||||||
|
stale_after_s: number;
|
||||||
|
primary_age_s: number | null;
|
||||||
|
sources: Array<TelemetrySourceFreshness & { records: number }>;
|
||||||
|
};
|
||||||
|
summary: {
|
||||||
|
runs: number;
|
||||||
|
success: number;
|
||||||
|
failed: number;
|
||||||
|
degraded: number;
|
||||||
|
failure_rate_pct: number;
|
||||||
|
retries: number;
|
||||||
|
latency_ms: { average: number; p50: number; p95: number; p99: number; max: number };
|
||||||
|
tokens: { input: number; output: number; total: number; provider_total: number };
|
||||||
|
cost_usd: { provider_actual: number; estimated: number };
|
||||||
|
provider_calls: number;
|
||||||
|
alerts: number;
|
||||||
|
};
|
||||||
|
thresholds: Record<string, number | string | boolean>;
|
||||||
|
findings: H6ReportFinding[];
|
||||||
|
evidence_sources: Array<TelemetrySourceFreshness & { records: number }>;
|
||||||
|
data_quality: { status: 'complete' | 'partial' | 'insufficient'; warnings: string[] };
|
||||||
|
available_filters: { projects: string[]; runs: string[] };
|
||||||
|
details: {
|
||||||
|
by_status: Array<{ status: string; count: number }>;
|
||||||
|
by_step: H6Breakdown[];
|
||||||
|
by_provider: H6Breakdown[];
|
||||||
|
by_cost_source: Array<{ source: string; records: number; cost_usd: number }>;
|
||||||
|
by_alert: Array<{ alert: string; count: number }>;
|
||||||
|
recent_runs: HarnessRunRecord[];
|
||||||
|
recent_alerts: Array<Record<string, unknown>>;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export interface HarnessRunRecord {
|
export interface HarnessRunRecord {
|
||||||
timestamp?: string;
|
timestamp?: string;
|
||||||
@@ -478,6 +563,24 @@ function actorHeaders(actor: SettingsActor): Record<string, string> {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function reportQuery(query: H6ReportQuery): string {
|
||||||
|
const params = new URLSearchParams();
|
||||||
|
if (query.project) params.set('project', query.project);
|
||||||
|
if (query.from) params.set('from', query.from);
|
||||||
|
if (query.to) params.set('to', query.to);
|
||||||
|
if (query.run) params.set('run', query.run);
|
||||||
|
if (query.limit) params.set('limit', String(query.limit));
|
||||||
|
const serialized = params.toString();
|
||||||
|
return serialized ? `?${serialized}` : '';
|
||||||
|
}
|
||||||
|
|
||||||
|
export function h6ReportExportUrl(query: H6ReportQuery, format: 'json' | 'html'): string {
|
||||||
|
const base = import.meta.env.VITE_API_BASE_URL ?? '/api/v1';
|
||||||
|
const params = new URLSearchParams(reportQuery(query).replace(/^\?/, ''));
|
||||||
|
params.set('format', format);
|
||||||
|
return `${base}/reports/h6/export?${params.toString()}`;
|
||||||
|
}
|
||||||
|
|
||||||
export const api = {
|
export const api = {
|
||||||
session: () => get<SettingsActor>('session'),
|
session: () => get<SettingsActor>('session'),
|
||||||
overview: () => get<Overview>('overview'),
|
overview: () => get<Overview>('overview'),
|
||||||
@@ -502,6 +605,7 @@ export const api = {
|
|||||||
incidents: () => get<Freshness & { total: number; kill_switch_scopes: string[]; incidents: any[] }>('incidents'),
|
incidents: () => get<Freshness & { total: number; kill_switch_scopes: string[]; incidents: any[] }>('incidents'),
|
||||||
traceability: () => get<Freshness & { matrix: any }>('traceability'),
|
traceability: () => get<Freshness & { matrix: any }>('traceability'),
|
||||||
cost: () => get<Freshness & { provider_tokens: number; provider_cost: number; by_provider: any[]; business_kpi: any }>('cost'),
|
cost: () => get<Freshness & { provider_tokens: number; provider_cost: number; by_provider: any[]; business_kpi: any }>('cost'),
|
||||||
|
h6Report: (query: H6ReportQuery = {}) => get<H6Report>(`reports/h6${reportQuery(query)}`),
|
||||||
command: () => get<CommandCenterState>('command'),
|
command: () => get<CommandCenterState>('command'),
|
||||||
evidencePacks: () => get<{ root: string; packs: EvidencePackSummary[] }>('evidence-packs'),
|
evidencePacks: () => get<{ root: string; packs: EvidencePackSummary[] }>('evidence-packs'),
|
||||||
evidencePack: (id: string) => get<EvidencePackDetail>(`evidence-packs/${encodeURIComponent(id)}`),
|
evidencePack: (id: string) => get<EvidencePackDetail>(`evidence-packs/${encodeURIComponent(id)}`),
|
||||||
|
|||||||
@@ -0,0 +1,139 @@
|
|||||||
|
import { useQuery } from '@tanstack/react-query';
|
||||||
|
import { useMemo, useState, type FormEvent, type ReactNode } from 'react';
|
||||||
|
import { useSearchParams } from 'react-router-dom';
|
||||||
|
import { api, h6ReportExportUrl, type H6Breakdown, type H6ReportQuery, type HarnessReportVerdict } from '../lib/api';
|
||||||
|
import { Card, StatusBadge } from '../components/ui/Card';
|
||||||
|
|
||||||
|
const money = (value: number): string => `$${value.toFixed(6)}`;
|
||||||
|
const integer = (value: number): string => new Intl.NumberFormat('en-US').format(value);
|
||||||
|
const verdictTone: Record<HarnessReportVerdict, string> = {
|
||||||
|
pass: 'border-emerald-300 bg-emerald-50 text-emerald-900',
|
||||||
|
attention: 'border-amber-300 bg-amber-50 text-amber-950',
|
||||||
|
fail: 'border-rose-300 bg-rose-50 text-rose-950',
|
||||||
|
no_data: 'border-slate-300 bg-slate-50 text-slate-800',
|
||||||
|
};
|
||||||
|
|
||||||
|
function queryFromSearch(search: URLSearchParams): H6ReportQuery {
|
||||||
|
return {
|
||||||
|
project: search.get('project') || undefined,
|
||||||
|
from: search.get('from') || undefined,
|
||||||
|
to: search.get('to') || undefined,
|
||||||
|
run: search.get('run') || undefined,
|
||||||
|
limit: 50,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function FilterField({ label, children }: { label: string; children: ReactNode }) {
|
||||||
|
return <label className="block"><span className="mb-1.5 block text-[10px] font-bold uppercase tracking-[0.13em] text-slate-500">{label}</span>{children}</label>;
|
||||||
|
}
|
||||||
|
|
||||||
|
function MetricCard({ label, value, sub, accent = false }: { label: string; value: string; sub?: string; accent?: boolean }) {
|
||||||
|
return <div className={`relative overflow-hidden rounded-2xl border p-4 ${accent ? 'border-indigo-200 bg-indigo-50/70' : 'border-slate-200 bg-white'}`}>
|
||||||
|
{accent && <span className="absolute -right-5 -top-5 h-16 w-16 rounded-full bg-indigo-200/50 blur-xl" />}
|
||||||
|
<div className="relative text-[10px] font-bold uppercase tracking-[0.12em] text-slate-500">{label}</div>
|
||||||
|
<div className="relative mt-2 text-2xl font-semibold tracking-tight text-slate-900">{value}</div>
|
||||||
|
{sub && <div className="relative mt-1 text-xs text-slate-500">{sub}</div>}
|
||||||
|
</div>;
|
||||||
|
}
|
||||||
|
|
||||||
|
function BreakdownTable({ rows, subject }: { rows: H6Breakdown[]; subject: string }) {
|
||||||
|
return <div className="overflow-x-auto"><table className="w-full min-w-[680px] text-sm">
|
||||||
|
<thead><tr className="border-b border-slate-200 text-left text-[10px] font-bold uppercase tracking-[0.1em] text-slate-400">
|
||||||
|
<th className="py-2.5 pr-3">{subject}</th><th>runs</th><th>failures</th><th>avg latency</th><th>tokens</th><th>cost</th>
|
||||||
|
</tr></thead>
|
||||||
|
<tbody>{rows.map((row) => <tr key={row.key} className="border-b border-slate-100 transition-colors hover:bg-slate-50/80">
|
||||||
|
<td className="max-w-[300px] py-3 pr-3 font-medium text-slate-800">{row.key}</td><td>{integer(row.runs)}</td><td>{integer(row.failures)}</td><td>{integer(row.latency_avg_ms)} ms</td><td>{integer(row.tokens)}</td><td>{money(row.cost_usd)}</td>
|
||||||
|
</tr>)}{rows.length === 0 && <tr><td colSpan={6} className="py-8 text-center text-slate-500">No matching {subject.toLowerCase()} records.</td></tr>}</tbody>
|
||||||
|
</table></div>;
|
||||||
|
}
|
||||||
|
|
||||||
|
function LoadingReport() {
|
||||||
|
return <div className="space-y-4" aria-label="Loading H6 report"><div className="h-44 animate-pulse rounded-3xl bg-slate-200" /><div className="grid grid-cols-2 gap-3 lg:grid-cols-4">{Array.from({ length: 8 }, (_, index) => <div key={index} className="h-28 animate-pulse rounded-2xl bg-slate-200" />)}</div></div>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function H6ReportPage() {
|
||||||
|
const [searchParams, setSearchParams] = useSearchParams();
|
||||||
|
const query = useMemo(() => queryFromSearch(searchParams), [searchParams]);
|
||||||
|
const [draft, setDraft] = useState(() => ({
|
||||||
|
project: searchParams.get('project') ?? '',
|
||||||
|
from: searchParams.get('from') ?? '',
|
||||||
|
to: searchParams.get('to') ?? '',
|
||||||
|
run: searchParams.get('run') ?? '',
|
||||||
|
}));
|
||||||
|
const report = useQuery({ queryKey: ['reports', 'h6', query], queryFn: () => api.h6Report(query), retry: false });
|
||||||
|
|
||||||
|
const applyFilters = (event: FormEvent<HTMLFormElement>) => {
|
||||||
|
event.preventDefault();
|
||||||
|
const next = new URLSearchParams();
|
||||||
|
if (draft.project) next.set('project', draft.project);
|
||||||
|
if (draft.from) next.set('from', draft.from);
|
||||||
|
if (draft.to) next.set('to', draft.to);
|
||||||
|
if (draft.run) next.set('run', draft.run);
|
||||||
|
setSearchParams(next);
|
||||||
|
};
|
||||||
|
|
||||||
|
const clearFilters = () => {
|
||||||
|
setDraft({ project: '', from: '', to: '', run: '' });
|
||||||
|
setSearchParams({});
|
||||||
|
};
|
||||||
|
|
||||||
|
const inputClass = 'w-full rounded-xl border border-slate-300 bg-white px-3 py-2.5 text-sm text-slate-800 outline-none transition focus:border-indigo-400 focus:ring-2 focus:ring-indigo-100';
|
||||||
|
|
||||||
|
return <div className="space-y-5">
|
||||||
|
<section className="relative overflow-hidden rounded-3xl border border-slate-800 bg-[#111827] px-5 py-6 text-white shadow-[0_22px_55px_rgba(15,23,42,0.16)] sm:px-7">
|
||||||
|
<div className="absolute right-[-60px] top-[-90px] h-64 w-64 rounded-full bg-indigo-500/25 blur-3xl" />
|
||||||
|
<div className="absolute bottom-[-120px] left-1/3 h-56 w-56 rounded-full bg-teal-500/10 blur-3xl" />
|
||||||
|
<div className="relative flex flex-col justify-between gap-5 lg:flex-row lg:items-end">
|
||||||
|
<div><div className="text-[10px] font-bold uppercase tracking-[0.2em] text-indigo-300">H6 · evidence-backed operations</div><h2 className="mt-2 text-2xl font-semibold tracking-tight sm:text-3xl">AgentOps assurance dossier</h2><p className="mt-2 max-w-2xl text-sm leading-6 text-slate-300">One report surface for runtime health, provider usage, cost provenance, failures, retries, alerts and source freshness.</p></div>
|
||||||
|
{report.data && <div className="flex flex-wrap items-center gap-2"><span className={`rounded-full border px-3 py-2 text-xs font-black uppercase tracking-[0.12em] ${verdictTone[report.data.verdict]}`}>{report.data.verdict}</span><span className="rounded-full border border-slate-700 bg-slate-900/70 px-3 py-2 text-xs text-slate-300">{report.data.freshness.status} · {report.data.report_id}</span></div>}
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<Card title="Report scope" right={<span className="text-xs font-normal normal-case tracking-normal text-slate-400">Filters apply to API and both exports</span>}>
|
||||||
|
<form onSubmit={applyFilters} className="grid gap-3 md:grid-cols-2 xl:grid-cols-[1fr_1fr_1fr_1.4fr_auto] xl:items-end">
|
||||||
|
<FilterField label="Project"><select value={draft.project} onChange={(event) => setDraft((current) => ({ ...current, project: event.target.value }))} className={inputClass}><option value="">All projects</option>{report.data?.available_filters.projects.map((project) => <option key={project} value={project}>{project}</option>)}</select></FilterField>
|
||||||
|
<FilterField label="From"><input type="date" value={draft.from} onChange={(event) => setDraft((current) => ({ ...current, from: event.target.value }))} className={inputClass} /></FilterField>
|
||||||
|
<FilterField label="To"><input type="date" value={draft.to} onChange={(event) => setDraft((current) => ({ ...current, to: event.target.value }))} className={inputClass} /></FilterField>
|
||||||
|
<FilterField label="Run / trace"><input list="h6-run-options" value={draft.run} onChange={(event) => setDraft((current) => ({ ...current, run: event.target.value }))} placeholder="All runs" className={inputClass} /><datalist id="h6-run-options">{report.data?.available_filters.runs.map((run) => <option key={run} value={run} />)}</datalist></FilterField>
|
||||||
|
<div className="flex gap-2"><button type="submit" className="rounded-xl bg-indigo-600 px-4 py-2.5 text-sm font-semibold text-white transition hover:bg-indigo-700 focus:outline-none focus:ring-2 focus:ring-indigo-300">Apply</button><button type="button" onClick={clearFilters} className="rounded-xl border border-slate-300 bg-white px-3 py-2.5 text-sm font-semibold text-slate-600 transition hover:bg-slate-50">Clear</button></div>
|
||||||
|
</form>
|
||||||
|
</Card>
|
||||||
|
|
||||||
|
{report.isLoading && <LoadingReport />}
|
||||||
|
{report.isError && <div role="alert" className="rounded-2xl border border-rose-200 bg-rose-50 p-5 text-sm text-rose-800"><div className="font-semibold">The H6 report could not be generated.</div><p className="mt-1 text-rose-700">Check the selected time range and confirm the Control Panel API can read the canonical telemetry sources.</p></div>}
|
||||||
|
|
||||||
|
{report.data && <>
|
||||||
|
<section className="grid grid-cols-2 gap-3 lg:grid-cols-4 xl:grid-cols-8">
|
||||||
|
<MetricCard label="Runs" value={integer(report.data.summary.runs)} sub={`${report.data.summary.success} successful`} accent />
|
||||||
|
<MetricCard label="Failures" value={integer(report.data.summary.failed)} sub={`${report.data.summary.failure_rate_pct}% rate`} />
|
||||||
|
<MetricCard label="Degraded" value={integer(report.data.summary.degraded)} />
|
||||||
|
<MetricCard label="P95 latency" value={`${integer(report.data.summary.latency_ms.p95)}ms`} sub={`P50 ${integer(report.data.summary.latency_ms.p50)}ms`} />
|
||||||
|
<MetricCard label="Provider tokens" value={integer(report.data.summary.tokens.provider_total)} sub={`${report.data.summary.provider_calls} calls`} accent />
|
||||||
|
<MetricCard label="Actual cost" value={money(report.data.summary.cost_usd.provider_actual)} sub="provider reported" />
|
||||||
|
<MetricCard label="Estimated cost" value={money(report.data.summary.cost_usd.estimated)} sub="runtime fallback" />
|
||||||
|
<MetricCard label="Alerts" value={integer(report.data.summary.alerts)} sub={`${report.data.summary.retries} retries`} />
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<div className="grid gap-5 xl:grid-cols-[1.15fr_0.85fr]">
|
||||||
|
<Card title="Verdict findings" right={<StatusBadge value={report.data.verdict} />}>
|
||||||
|
<div className="space-y-2.5">{report.data.findings.map((finding) => <div key={finding.code} className={`rounded-xl border-l-4 p-3.5 ${finding.severity === 'critical' ? 'border-rose-500 bg-rose-50' : finding.severity === 'warning' ? 'border-amber-500 bg-amber-50' : 'border-indigo-500 bg-indigo-50'}`}><div className="flex flex-wrap items-center justify-between gap-2"><span className="font-mono text-xs font-bold text-slate-800">{finding.code}</span>{finding.metric && <span className="text-[11px] text-slate-500">{finding.metric}: {String(finding.value)}{finding.threshold !== undefined ? ` · threshold ${String(finding.threshold)}` : ''}</span>}</div><p className="mt-1 text-sm leading-5 text-slate-600">{finding.message}</p></div>)}{report.data.findings.length === 0 && <div className="rounded-xl border border-emerald-200 bg-emerald-50 p-4 text-sm text-emerald-800">No threshold breach was detected in this scope.</div>}</div>
|
||||||
|
</Card>
|
||||||
|
<Card title="Evidence freshness" right={<StatusBadge value={report.data.freshness.status} />}>
|
||||||
|
<div className="space-y-3">{report.data.evidence_sources.map((source) => <div key={source.source} className="rounded-xl border border-slate-200 bg-slate-50/70 p-3"><div className="flex items-center justify-between gap-3"><span className="text-sm font-semibold capitalize text-slate-800">{source.source}</span><StatusBadge value={!source.present ? 'missing' : source.stale ? 'stale' : 'fresh'} /></div><div className="mt-2 flex justify-between text-xs text-slate-500"><span>{source.records} records</span><span>{source.age_s === null ? 'no timestamp' : `${integer(source.age_s)}s old`}</span></div><div className="mt-2 break-all font-mono text-[10px] leading-4 text-slate-400">{source.path}</div></div>)}</div>
|
||||||
|
</Card>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="grid gap-5 xl:grid-cols-2"><Card title="Runtime by step"><BreakdownTable rows={report.data.details.by_step} subject="Step" /></Card><Card title="Provider usage"><BreakdownTable rows={report.data.details.by_provider} subject="Provider · model" /></Card></div>
|
||||||
|
|
||||||
|
<div className="grid gap-5 xl:grid-cols-[0.8fr_1.2fr]">
|
||||||
|
<Card title={`Data quality · ${report.data.data_quality.status}`}>
|
||||||
|
{report.data.data_quality.warnings.length > 0 ? <ul className="space-y-2 text-sm leading-5 text-slate-600">{report.data.data_quality.warnings.map((warning) => <li key={warning} className="flex gap-2"><span className="mt-1 h-2 w-2 shrink-0 rounded-full bg-amber-400" />{warning}</li>)}</ul> : <div className="text-sm text-emerald-700">All required sources are present and no estimation warning was detected.</div>}
|
||||||
|
</Card>
|
||||||
|
<Card title="Independent export" right={<span className="text-xs font-normal normal-case tracking-normal text-slate-400">Generated on demand · no hard-coded score</span>}>
|
||||||
|
<p className="text-sm leading-6 text-slate-600">Both exports use the same contract, filters, verdict rules and canonical sources as this screen. JSON is machine-auditable; HTML is print-ready for team review.</p>
|
||||||
|
<div className="mt-4 flex flex-wrap gap-3"><a href={h6ReportExportUrl(query, 'json')} className="rounded-xl bg-slate-900 px-4 py-2.5 text-sm font-semibold text-white transition hover:bg-slate-700">Download JSON</a><a href={h6ReportExportUrl(query, 'html')} className="rounded-xl border border-indigo-300 bg-indigo-50 px-4 py-2.5 text-sm font-semibold text-indigo-700 transition hover:bg-indigo-100">Download HTML</a></div>
|
||||||
|
</Card>
|
||||||
|
</div>
|
||||||
|
</>}
|
||||||
|
</div>;
|
||||||
|
}
|
||||||
@@ -27,9 +27,9 @@ LOG_DIR="$CASAN_STATE_ROOT/logs"
|
|||||||
TRACE_DIR="$LOG_DIR/trace"
|
TRACE_DIR="$LOG_DIR/trace"
|
||||||
# SEC-23 (MT-01): telemetry dir is tenant-scoped when CASAN_METRICS_DIR is set
|
# SEC-23 (MT-01): telemetry dir is tenant-scoped when CASAN_METRICS_DIR is set
|
||||||
# (tenant-paths.sh exports it per tenant); default is the shared path.
|
# (tenant-paths.sh exports it per tenant); default is the shared path.
|
||||||
METRICS_DIR="${CASAN_METRICS_DIR:-$LOG_DIR/cost}"
|
METRICS_DIR="${CASAN_METRICS_DIR:-$(dirname "$CASAN_TELEMETRY_METRICS_LOG")}"
|
||||||
ALERT_LOG="$CASAN_HARNESS_ROOT/agentops/alerts.log"
|
ALERT_LOG="$CASAN_TELEMETRY_ALERTS_LOG"
|
||||||
METRICS_LOG="$METRICS_DIR/metrics.jsonl"
|
METRICS_LOG="${CASAN_METRICS_LOG:-$CASAN_TELEMETRY_METRICS_LOG}"
|
||||||
mkdir -p "$TRACE_DIR" "$METRICS_DIR" "$(dirname "$OUTPUT_FILE")" "$(dirname "$ALERT_LOG")"
|
mkdir -p "$TRACE_DIR" "$METRICS_DIR" "$(dirname "$OUTPUT_FILE")" "$(dirname "$ALERT_LOG")"
|
||||||
|
|
||||||
# shellcheck source=tool-audit-lib.sh
|
# shellcheck source=tool-audit-lib.sh
|
||||||
@@ -127,7 +127,7 @@ COST_SOURCE="word_count_estimate"
|
|||||||
|
|
||||||
# Prefer real provider usage when telemetry has been imported; the word-count
|
# Prefer real provider usage when telemetry has been imported; the word-count
|
||||||
# figure above is an explicit fallback, not presented as a real billed cost.
|
# figure above is an explicit fallback, not presented as a real billed cost.
|
||||||
PROVIDER_LOG="$CASAN_STATE_ROOT/logs/level5/provider-usage.jsonl"
|
PROVIDER_LOG="$CASAN_TELEMETRY_PROVIDER_LOG"
|
||||||
if [[ -f "$PROVIDER_LOG" ]] && command -v python >/dev/null 2>&1; then
|
if [[ -f "$PROVIDER_LOG" ]] && command -v python >/dev/null 2>&1; then
|
||||||
# Use real provider telemetry ONLY when a record genuinely matches this step.
|
# Use real provider telemetry ONLY when a record genuinely matches this step.
|
||||||
# Do NOT fall back to an arbitrary record (that would reuse one sample's cost
|
# Do NOT fall back to an arbitrary record (that would reuse one sample's cost
|
||||||
|
|||||||
@@ -61,6 +61,20 @@ if [[ -z "${CASAN_STATE_ROOT:-}" ]]; then
|
|||||||
CASAN_STATE_ROOT="$CASAN_APP_ROOT/.specify"
|
CASAN_STATE_ROOT="$CASAN_APP_ROOT/.specify"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# Canonical H6 telemetry files. Writers and readers must use these three names so
|
||||||
|
# runtime, provider usage and alerts cannot silently drift into different trees.
|
||||||
|
# The older dashboard/control-panel variables remain accepted as compatibility
|
||||||
|
# aliases while downstream deployments move to the CASAN_TELEMETRY_* contract.
|
||||||
|
if [[ -z "${CASAN_TELEMETRY_METRICS_LOG:-}" ]]; then
|
||||||
|
CASAN_TELEMETRY_METRICS_LOG="${CASAN_DASHBOARD_METRICS:-$CASAN_STATE_ROOT/logs/cost/metrics.jsonl}"
|
||||||
|
fi
|
||||||
|
if [[ -z "${CASAN_TELEMETRY_PROVIDER_LOG:-}" ]]; then
|
||||||
|
CASAN_TELEMETRY_PROVIDER_LOG="${CASAN_CP_PROVIDER_USAGE:-${CASAN_PROVIDER_LOG:-$CASAN_STATE_ROOT/logs/level5/provider-usage.jsonl}}"
|
||||||
|
fi
|
||||||
|
if [[ -z "${CASAN_TELEMETRY_ALERTS_LOG:-}" ]]; then
|
||||||
|
CASAN_TELEMETRY_ALERTS_LOG="${CASAN_DASHBOARD_ALERTS:-$CASAN_STATE_ROOT/agentops/alerts.log}"
|
||||||
|
fi
|
||||||
|
|
||||||
# Governance root: central-governance mixes harness pub-keys/registries, runtime
|
# Governance root: central-governance mixes harness pub-keys/registries, runtime
|
||||||
# policy-manifest state, and a private key. Rooted under STATE (not HARNESS) so the
|
# policy-manifest state, and a private key. Rooted under STATE (not HARNESS) so the
|
||||||
# runtime-regenerated policy-manifest.{json,sig} and the private key never land inside
|
# runtime-regenerated policy-manifest.{json,sig} and the private key never land inside
|
||||||
|
|||||||
@@ -75,7 +75,8 @@ def head_path() -> str:
|
|||||||
|
|
||||||
|
|
||||||
def metrics_path() -> str:
|
def metrics_path() -> str:
|
||||||
return guarded_override(os.environ["CASAN_CHAT_METRICS_LOG"]) if os.environ.get("CASAN_CHAT_METRICS_LOG") else tenant_path("telemetry/cost/metrics.jsonl", "logs/cost/metrics.jsonl")
|
override = os.environ.get("CASAN_TELEMETRY_METRICS_LOG") or os.environ.get("CASAN_CHAT_METRICS_LOG")
|
||||||
|
return guarded_override(override) if override else tenant_path("telemetry/cost/metrics.jsonl", "logs/cost/metrics.jsonl")
|
||||||
|
|
||||||
|
|
||||||
def artifact_dir() -> str:
|
def artifact_dir() -> str:
|
||||||
|
|||||||
@@ -90,7 +90,8 @@ def head_path() -> str:
|
|||||||
|
|
||||||
|
|
||||||
def metrics_path() -> str:
|
def metrics_path() -> str:
|
||||||
return guarded_override(os.environ["CASAN_CHAT_METRICS_LOG"]) if os.environ.get("CASAN_CHAT_METRICS_LOG") else tenant_path("telemetry/cost/metrics.jsonl", "logs/cost/metrics.jsonl")
|
override = os.environ.get("CASAN_TELEMETRY_METRICS_LOG") or os.environ.get("CASAN_CHAT_METRICS_LOG")
|
||||||
|
return guarded_override(override) if override else tenant_path("telemetry/cost/metrics.jsonl", "logs/cost/metrics.jsonl")
|
||||||
|
|
||||||
|
|
||||||
def trace_events_path(trace_id: str) -> str:
|
def trace_events_path(trace_id: str) -> str:
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ set -uo pipefail
|
|||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
source "$SCRIPT_DIR/casan-paths.sh"
|
source "$SCRIPT_DIR/casan-paths.sh"
|
||||||
ROOT="$CASAN_APP_ROOT"
|
ROOT="$CASAN_APP_ROOT"
|
||||||
PROVIDER_LOG="${CASAN_PROVIDER_LOG:-$CASAN_STATE_ROOT/logs/level5/provider-usage.jsonl}"
|
PROVIDER_LOG="$CASAN_TELEMETRY_PROVIDER_LOG"
|
||||||
CIRCUIT_BREAKER_THRESHOLD="${CIRCUIT_BREAKER_THRESHOLD:-5}"
|
CIRCUIT_BREAKER_THRESHOLD="${CIRCUIT_BREAKER_THRESHOLD:-5}"
|
||||||
CIRCUIT_WINDOW="${CIRCUIT_WINDOW:-10}"
|
CIRCUIT_WINDOW="${CIRCUIT_WINDOW:-10}"
|
||||||
CIRCUIT_WINDOW_FAIL_PCT="${CIRCUIT_WINDOW_FAIL_PCT:-50}"
|
CIRCUIT_WINDOW_FAIL_PCT="${CIRCUIT_WINDOW_FAIL_PCT:-50}"
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ elif [[ -n "${CASAN_TENANT_ID:-}" ]]; then
|
|||||||
LOG="$(bash "$SCRIPT_DIR/tenant-store.sh" resolve telemetry/provider-usage.jsonl 2>/dev/null)" \
|
LOG="$(bash "$SCRIPT_DIR/tenant-store.sh" resolve telemetry/provider-usage.jsonl 2>/dev/null)" \
|
||||||
|| { echo "COST_SPIKE_TENANT_DENIED" >&2; exit 3; }
|
|| { echo "COST_SPIKE_TENANT_DENIED" >&2; exit 3; }
|
||||||
else
|
else
|
||||||
LOG="$CASAN_STATE_ROOT/logs/level5/provider-usage.jsonl"
|
LOG="$CASAN_TELEMETRY_PROVIDER_LOG"
|
||||||
fi
|
fi
|
||||||
MULT="${2:-3.0}"
|
MULT="${2:-3.0}"
|
||||||
|
|
||||||
|
|||||||
@@ -35,9 +35,11 @@ PORT = int(sys.argv[1]) if len(sys.argv) > 1 else 8787
|
|||||||
DASH = pathlib.Path(os.environ.get(
|
DASH = pathlib.Path(os.environ.get(
|
||||||
"CASAN_DASHBOARD_HTML", ROOT / "docs" / "output" / "casan" / "central-agentops-dashboard.html"))
|
"CASAN_DASHBOARD_HTML", ROOT / "docs" / "output" / "casan" / "central-agentops-dashboard.html"))
|
||||||
METRICS = pathlib.Path(os.environ.get(
|
METRICS = pathlib.Path(os.environ.get(
|
||||||
"CASAN_DASHBOARD_METRICS", ROOT / ".specify" / "logs" / "cost" / "metrics.jsonl"))
|
"CASAN_TELEMETRY_METRICS_LOG", os.environ.get(
|
||||||
|
"CASAN_DASHBOARD_METRICS", ROOT / ".specify" / "logs" / "cost" / "metrics.jsonl")))
|
||||||
ALERTS = pathlib.Path(os.environ.get(
|
ALERTS = pathlib.Path(os.environ.get(
|
||||||
"CASAN_DASHBOARD_ALERTS", ROOT / ".specify" / "agentops" / "alerts.log"))
|
"CASAN_TELEMETRY_ALERTS_LOG", os.environ.get(
|
||||||
|
"CASAN_DASHBOARD_ALERTS", ROOT / ".specify" / "agentops" / "alerts.log")))
|
||||||
STALE_S = int(os.environ.get("CASAN_DASHBOARD_STALE_S", "3600"))
|
STALE_S = int(os.environ.get("CASAN_DASHBOARD_STALE_S", "3600"))
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -133,9 +133,17 @@ def main():
|
|||||||
"telemetry_integrity": tel_text, "telemetry_ok": tel_rc == 0,
|
"telemetry_integrity": tel_text, "telemetry_ok": tel_rc == 0,
|
||||||
}
|
}
|
||||||
|
|
||||||
# H6 cost telemetry
|
# H6 telemetry — use the same canonical sources as the Control Panel/report API.
|
||||||
prov = read_jsonl(os.path.join(logs, "level5", "provider-usage.jsonl"))
|
provider_path = os.environ.get(
|
||||||
metrics = read_jsonl(os.path.join(logs, "cost", "metrics.jsonl"))
|
"CASAN_TELEMETRY_PROVIDER_LOG",
|
||||||
|
os.environ.get("CASAN_CP_PROVIDER_USAGE", os.path.join(logs, "level5", "provider-usage.jsonl")),
|
||||||
|
)
|
||||||
|
metrics_path = os.environ.get(
|
||||||
|
"CASAN_TELEMETRY_METRICS_LOG",
|
||||||
|
os.environ.get("CASAN_DASHBOARD_METRICS", os.path.join(logs, "cost", "metrics.jsonl")),
|
||||||
|
)
|
||||||
|
prov = read_jsonl(provider_path)
|
||||||
|
metrics = read_jsonl(metrics_path)
|
||||||
cost_rc = int(os.environ.get("CASAN_EP_COST_RC", "3") or "3")
|
cost_rc = int(os.environ.get("CASAN_EP_COST_RC", "3") or "3")
|
||||||
total_tokens = sum(int(r.get("total_tokens", 0)) for r in prov if str(r.get("total_tokens", "")).isdigit())
|
total_tokens = sum(int(r.get("total_tokens", 0)) for r in prov if str(r.get("total_tokens", "")).isdigit())
|
||||||
reports["h6-cost-telemetry.json"] = {
|
reports["h6-cost-telemetry.json"] = {
|
||||||
|
|||||||
@@ -1063,7 +1063,11 @@ def audit(job: dict, status: str) -> str:
|
|||||||
|
|
||||||
|
|
||||||
def metric(job: dict, status: str, started: float, local_meta: dict, cloud_meta: dict) -> None:
|
def metric(job: dict, status: str, started: float, local_meta: dict, cloud_meta: dict) -> None:
|
||||||
append_jsonl(os.path.join(STATE_ROOT, "logs", "cost", "metrics.jsonl"), {
|
metrics_path = os.environ.get(
|
||||||
|
"CASAN_TELEMETRY_METRICS_LOG",
|
||||||
|
os.environ.get("CASAN_DASHBOARD_METRICS", os.path.join(STATE_ROOT, "logs", "cost", "metrics.jsonl")),
|
||||||
|
)
|
||||||
|
append_jsonl(metrics_path, {
|
||||||
"timestamp": now(),
|
"timestamp": now(),
|
||||||
"trace_id": job["id"],
|
"trace_id": job["id"],
|
||||||
"harness": "H6-agentops",
|
"harness": "H6-agentops",
|
||||||
|
|||||||
@@ -14,8 +14,8 @@ fi
|
|||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
source "$SCRIPT_DIR/casan-paths.sh"
|
source "$SCRIPT_DIR/casan-paths.sh"
|
||||||
PROJECT_ROOT="$CASAN_APP_ROOT"
|
PROJECT_ROOT="$CASAN_APP_ROOT"
|
||||||
LOG_DIR="$CASAN_STATE_ROOT/logs/level5"
|
OUT="$CASAN_TELEMETRY_PROVIDER_LOG"
|
||||||
OUT="$LOG_DIR/provider-usage.jsonl"
|
LOG_DIR="$(dirname "$OUT")"
|
||||||
mkdir -p "$LOG_DIR"
|
mkdir -p "$LOG_DIR"
|
||||||
|
|
||||||
python - "$INPUT_JSON" "$OUT" <<'PY'
|
python - "$INPUT_JSON" "$OUT" <<'PY'
|
||||||
|
|||||||
@@ -22,6 +22,85 @@ AUTH_BRIDGE_AUDIT="$AUTH_BRIDGE_DIR/model-audit.jsonl"
|
|||||||
AUTH_BRIDGE="$ROOT/packages/casan-control-panel/scripts/provider-auth-bridge.py"
|
AUTH_BRIDGE="$ROOT/packages/casan-control-panel/scripts/provider-auth-bridge.py"
|
||||||
CMD="${1:-status}"
|
CMD="${1:-status}"
|
||||||
|
|
||||||
|
resolve_python() {
|
||||||
|
if [[ -n "${CASAN_PYTHON_BIN:-}" ]]; then
|
||||||
|
if command -v "$CASAN_PYTHON_BIN" >/dev/null 2>&1 && "$CASAN_PYTHON_BIN" --version >/dev/null 2>&1; then
|
||||||
|
printf '%s\n' "$CASAN_PYTHON_BIN"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
echo "CASAN_LOCAL_PYTHON_INVALID path=$CASAN_PYTHON_BIN" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Prefer macOS' universal system Python over stale framework installs that may
|
||||||
|
# appear first in PATH but are terminated by Gatekeeper/Rosetta on Apple Silicon.
|
||||||
|
local candidate
|
||||||
|
for candidate in /usr/bin/python3 python3 python; do
|
||||||
|
if command -v "$candidate" >/dev/null 2>&1 && "$candidate" --version >/dev/null 2>&1; then
|
||||||
|
printf '%s\n' "$candidate"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
echo "CASAN_LOCAL_PYTHON_MISSING" >&2
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
prepare_docker_cli() {
|
||||||
|
local original_config="${DOCKER_CONFIG:-$HOME/.docker}"
|
||||||
|
local config_file="$original_config/config.json"
|
||||||
|
[[ -f "$config_file" ]] || return 0
|
||||||
|
|
||||||
|
local python_bin credential_store helper context_name docker_host fallback_config
|
||||||
|
python_bin="$(resolve_python)" || return 1
|
||||||
|
credential_store="$("$python_bin" - "$config_file" <<'PY'
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
|
||||||
|
try:
|
||||||
|
with open(sys.argv[1], encoding="utf-8") as handle:
|
||||||
|
print(json.load(handle).get("credsStore", ""))
|
||||||
|
except (OSError, ValueError):
|
||||||
|
print("")
|
||||||
|
PY
|
||||||
|
)"
|
||||||
|
[[ -n "$credential_store" ]] || return 0
|
||||||
|
|
||||||
|
helper="$(command -v "docker-credential-$credential_store" 2>/dev/null || true)"
|
||||||
|
if [[ -n "$helper" ]] && "$python_bin" - "$helper" <<'PY' >/dev/null 2>&1
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
|
||||||
|
raise SystemExit(subprocess.run(
|
||||||
|
[sys.argv[1], "list"],
|
||||||
|
stdin=subprocess.DEVNULL,
|
||||||
|
stdout=subprocess.DEVNULL,
|
||||||
|
stderr=subprocess.DEVNULL,
|
||||||
|
).returncode)
|
||||||
|
PY
|
||||||
|
then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# CASAN's local compose files use public images only. If Docker Desktop's
|
||||||
|
# credential helper is broken, isolate this process from it without changing
|
||||||
|
# ~/.docker/config.json or touching any stored login credentials.
|
||||||
|
context_name="$(docker context show)"
|
||||||
|
docker_host="$(docker context inspect --format '{{(index .Endpoints "docker").Host}}' "$context_name")"
|
||||||
|
[[ -n "$docker_host" ]] || { echo "CASAN_LOCAL_DOCKER_CONTEXT_INVALID context=$context_name" >&2; return 1; }
|
||||||
|
fallback_config="${TMPDIR:-/tmp}/casan-docker-public-$UID"
|
||||||
|
mkdir -p "$fallback_config"
|
||||||
|
if [[ -d "$original_config/cli-plugins" && ! -e "$fallback_config/cli-plugins" ]]; then
|
||||||
|
ln -s "$original_config/cli-plugins" "$fallback_config/cli-plugins"
|
||||||
|
fi
|
||||||
|
umask 077
|
||||||
|
printf '%s\n' '{"auths":{"https://index.docker.io/v1/":{},"quay.io":{}}}' > "$fallback_config/config.json"
|
||||||
|
export DOCKER_CONFIG="$fallback_config"
|
||||||
|
export DOCKER_HOST="$docker_host"
|
||||||
|
export DOCKER_BUILDKIT=0
|
||||||
|
export COMPOSE_DOCKER_CLI_BUILD=0
|
||||||
|
echo "CASAN_LOCAL_DOCKER_CREDENTIAL_FALLBACK helper=$credential_store context=$context_name builder=classic" >&2
|
||||||
|
}
|
||||||
|
|
||||||
cp_compose() {
|
cp_compose() {
|
||||||
if [[ -f "$AUTH_BRIDGE_TOKEN_FILE" ]]; then
|
if [[ -f "$AUTH_BRIDGE_TOKEN_FILE" ]]; then
|
||||||
export CASAN_AUTH_BRIDGE_TOKEN
|
export CASAN_AUTH_BRIDGE_TOKEN
|
||||||
@@ -35,6 +114,8 @@ cp_compose() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
start_auth_bridge() {
|
start_auth_bridge() {
|
||||||
|
local python_bin
|
||||||
|
python_bin="$(resolve_python)" || return 1
|
||||||
mkdir -p "$AUTH_BRIDGE_DIR"
|
mkdir -p "$AUTH_BRIDGE_DIR"
|
||||||
if [[ ! -s "$AUTH_BRIDGE_TOKEN_FILE" ]]; then
|
if [[ ! -s "$AUTH_BRIDGE_TOKEN_FILE" ]]; then
|
||||||
openssl rand -hex 32 > "$AUTH_BRIDGE_TOKEN_FILE"
|
openssl rand -hex 32 > "$AUTH_BRIDGE_TOKEN_FILE"
|
||||||
@@ -52,7 +133,7 @@ start_auth_bridge() {
|
|||||||
rm -f "$AUTH_BRIDGE_PID_FILE"
|
rm -f "$AUTH_BRIDGE_PID_FILE"
|
||||||
fi
|
fi
|
||||||
[[ -f "$AUTH_BRIDGE" ]] || { echo "CASAN_AUTH_BRIDGE_MISSING" >&2; return 1; }
|
[[ -f "$AUTH_BRIDGE" ]] || { echo "CASAN_AUTH_BRIDGE_MISSING" >&2; return 1; }
|
||||||
nohup python3 "$AUTH_BRIDGE" --bind 0.0.0.0 --port 20130 --token-file "$AUTH_BRIDGE_TOKEN_FILE" --audit-log "$AUTH_BRIDGE_AUDIT" > "$AUTH_BRIDGE_LOG" 2>&1 &
|
nohup "$python_bin" "$AUTH_BRIDGE" --bind 0.0.0.0 --port 20130 --token-file "$AUTH_BRIDGE_TOKEN_FILE" --audit-log "$AUTH_BRIDGE_AUDIT" > "$AUTH_BRIDGE_LOG" 2>&1 &
|
||||||
echo "$!" > "$AUTH_BRIDGE_PID_FILE"
|
echo "$!" > "$AUTH_BRIDGE_PID_FILE"
|
||||||
chmod 600 "$AUTH_BRIDGE_PID_FILE" "$AUTH_BRIDGE_LOG" "$AUTH_BRIDGE_AUDIT" 2>/dev/null || true
|
chmod 600 "$AUTH_BRIDGE_PID_FILE" "$AUTH_BRIDGE_LOG" "$AUTH_BRIDGE_AUDIT" 2>/dev/null || true
|
||||||
if ! wait_url "http://127.0.0.1:20130/healthz"; then
|
if ! wait_url "http://127.0.0.1:20130/healthz"; then
|
||||||
@@ -74,6 +155,7 @@ stop_auth_bridge() {
|
|||||||
|
|
||||||
need_docker() {
|
need_docker() {
|
||||||
command -v docker >/dev/null 2>&1 || { echo "CASAN_LOCAL_DOCKER_MISSING" >&2; exit 1; }
|
command -v docker >/dev/null 2>&1 || { echo "CASAN_LOCAL_DOCKER_MISSING" >&2; exit 1; }
|
||||||
|
prepare_docker_cli
|
||||||
docker compose version >/dev/null 2>&1 || { echo "CASAN_LOCAL_COMPOSE_MISSING" >&2; exit 1; }
|
docker compose version >/dev/null 2>&1 || { echo "CASAN_LOCAL_COMPOSE_MISSING" >&2; exit 1; }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -48,7 +48,10 @@ OLLAMA_HOST = "127.0.0.1:11434" # the only allowed ollama endpoint
|
|||||||
DOCKER_OLLAMA_HOST = "host.docker.internal:11434"
|
DOCKER_OLLAMA_HOST = "host.docker.internal:11434"
|
||||||
ALLOWED_CLOUD = {"api.anthropic.com", "api.openai.com"}
|
ALLOWED_CLOUD = {"api.anthropic.com", "api.openai.com"}
|
||||||
REPO_ROOT = _casan_app_root()
|
REPO_ROOT = _casan_app_root()
|
||||||
PROVIDER_LOG = os.path.join(REPO_ROOT, ".specify/logs/level5/provider-usage.jsonl")
|
PROVIDER_LOG = os.environ.get(
|
||||||
|
"CASAN_TELEMETRY_PROVIDER_LOG",
|
||||||
|
os.environ.get("CASAN_CP_PROVIDER_USAGE", os.path.join(REPO_ROOT, ".specify/logs/level5/provider-usage.jsonl")),
|
||||||
|
)
|
||||||
|
|
||||||
# SEC-21 (ARCH-07): a 180s-per-call timeout across many pipeline steps let a hung
|
# SEC-21 (ARCH-07): a 180s-per-call timeout across many pipeline steps let a hung
|
||||||
# model stall a run for tens of minutes. Use a lower, configurable per-call timeout,
|
# model stall a run for tens of minutes. Use a lower, configurable per-call timeout,
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ fi
|
|||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
source "$SCRIPT_DIR/casan-paths.sh"
|
source "$SCRIPT_DIR/casan-paths.sh"
|
||||||
PROJECT_ROOT="$CASAN_APP_ROOT"
|
PROJECT_ROOT="$CASAN_APP_ROOT"
|
||||||
OUT="${2:-$CASAN_STATE_ROOT/logs/level5/provider-usage.jsonl}"
|
OUT="${2:-$CASAN_TELEMETRY_PROVIDER_LOG}"
|
||||||
mkdir -p "$(dirname "$OUT")"
|
mkdir -p "$(dirname "$OUT")"
|
||||||
|
|
||||||
# SEC-13 (M-09): SSRF guard on the fetch URL. ALWAYS reject non-http(s) schemes
|
# SEC-13 (M-09): SSRF guard on the fetch URL. ALWAYS reject non-http(s) schemes
|
||||||
|
|||||||
@@ -19,9 +19,9 @@ $scriptDir = Split-Path $MyInvocation.MyCommand.Path -Parent
|
|||||||
$projectRoot = (Resolve-Path (Join-Path $scriptDir "../../..")).Path
|
$projectRoot = (Resolve-Path (Join-Path $scriptDir "../../..")).Path
|
||||||
$logDir = Join-Path $projectRoot ".specify/logs"
|
$logDir = Join-Path $projectRoot ".specify/logs"
|
||||||
$traceDir = Join-Path $logDir "trace"
|
$traceDir = Join-Path $logDir "trace"
|
||||||
$metricsDir = Join-Path $logDir "cost"
|
$metricsLog = if ($env:CASAN_TELEMETRY_METRICS_LOG) { $env:CASAN_TELEMETRY_METRICS_LOG } elseif ($env:CASAN_DASHBOARD_METRICS) { $env:CASAN_DASHBOARD_METRICS } else { Join-Path $logDir "cost/metrics.jsonl" }
|
||||||
$alertLog = Join-Path $projectRoot ".specify/agentops/alerts.log"
|
$metricsDir = Split-Path $metricsLog -Parent
|
||||||
$metricsLog = Join-Path $metricsDir "metrics.jsonl"
|
$alertLog = if ($env:CASAN_TELEMETRY_ALERTS_LOG) { $env:CASAN_TELEMETRY_ALERTS_LOG } elseif ($env:CASAN_DASHBOARD_ALERTS) { $env:CASAN_DASHBOARD_ALERTS } else { Join-Path $projectRoot ".specify/agentops/alerts.log" }
|
||||||
$toolAudit = Join-Path $logDir "audit/tool-calls.jsonl"
|
$toolAudit = Join-Path $logDir "audit/tool-calls.jsonl"
|
||||||
|
|
||||||
foreach ($d in @($traceDir, $metricsDir, (Split-Path $OutputFile -Parent), (Split-Path $alertLog -Parent), (Split-Path $toolAudit -Parent))) {
|
foreach ($d in @($traceDir, $metricsDir, (Split-Path $OutputFile -Parent), (Split-Path $alertLog -Parent), (Split-Path $toolAudit -Parent))) {
|
||||||
|
|||||||
@@ -10,8 +10,8 @@ param(
|
|||||||
|
|
||||||
$scriptDir = Split-Path $MyInvocation.MyCommand.Path -Parent
|
$scriptDir = Split-Path $MyInvocation.MyCommand.Path -Parent
|
||||||
$projectRoot = (Resolve-Path (Join-Path $scriptDir "../../..")).Path
|
$projectRoot = (Resolve-Path (Join-Path $scriptDir "../../..")).Path
|
||||||
$metricsLog = Join-Path $projectRoot ".specify/logs/cost/metrics.jsonl"
|
$metricsLog = if ($env:CASAN_TELEMETRY_METRICS_LOG) { $env:CASAN_TELEMETRY_METRICS_LOG } elseif ($env:CASAN_DASHBOARD_METRICS) { $env:CASAN_DASHBOARD_METRICS } else { Join-Path $projectRoot ".specify/logs/cost/metrics.jsonl" }
|
||||||
$alertLog = Join-Path $projectRoot ".specify/agentops/alerts.log"
|
$alertLog = if ($env:CASAN_TELEMETRY_ALERTS_LOG) { $env:CASAN_TELEMETRY_ALERTS_LOG } elseif ($env:CASAN_DASHBOARD_ALERTS) { $env:CASAN_DASHBOARD_ALERTS } else { Join-Path $projectRoot ".specify/agentops/alerts.log" }
|
||||||
|
|
||||||
if (!$OutputHtml) { $OutputHtml = Join-Path $projectRoot "docs/output/casan/agentops-dashboard.html" }
|
if (!$OutputHtml) { $OutputHtml = Join-Path $projectRoot "docs/output/casan/agentops-dashboard.html" }
|
||||||
if (!(Test-Path (Split-Path $OutputHtml -Parent))) { New-Item -ItemType Directory -Force -Path (Split-Path $OutputHtml -Parent) | Out-Null }
|
if (!(Test-Path (Split-Path $OutputHtml -Parent))) { New-Item -ItemType Directory -Force -Path (Split-Path $OutputHtml -Parent) | Out-Null }
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
import os
|
||||||
import pathlib
|
import pathlib
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
@@ -16,10 +17,14 @@ def _app_root(start):
|
|||||||
|
|
||||||
|
|
||||||
ROOT = _app_root(__file__)
|
ROOT = _app_root(__file__)
|
||||||
METRICS = ROOT / ".specify" / "logs" / "cost" / "metrics.jsonl"
|
METRICS = pathlib.Path(os.environ.get(
|
||||||
|
"CASAN_TELEMETRY_METRICS_LOG", os.environ.get(
|
||||||
|
"CASAN_DASHBOARD_METRICS", ROOT / ".specify" / "logs" / "cost" / "metrics.jsonl")))
|
||||||
FALLBACK = ROOT / ".specify" / "logs" / "level5" / "fallback.jsonl"
|
FALLBACK = ROOT / ".specify" / "logs" / "level5" / "fallback.jsonl"
|
||||||
TOOL = ROOT / ".specify" / "logs" / "level5" / "tool-registry.jsonl"
|
TOOL = ROOT / ".specify" / "logs" / "level5" / "tool-registry.jsonl"
|
||||||
PROVIDER = ROOT / ".specify" / "logs" / "level5" / "provider-usage.jsonl"
|
PROVIDER = pathlib.Path(os.environ.get(
|
||||||
|
"CASAN_TELEMETRY_PROVIDER_LOG", os.environ.get(
|
||||||
|
"CASAN_CP_PROVIDER_USAGE", ROOT / ".specify" / "logs" / "level5" / "provider-usage.jsonl")))
|
||||||
PROJECT_REGISTRY = ROOT / ".specify" / "level5" / "project-registry.json"
|
PROJECT_REGISTRY = ROOT / ".specify" / "level5" / "project-registry.json"
|
||||||
DASHBOARD = ROOT / "docs" / "output" / "casan" / "central-agentops-dashboard.html"
|
DASHBOARD = ROOT / "docs" / "output" / "casan" / "central-agentops-dashboard.html"
|
||||||
LEGACY_DASHBOARD = ROOT / "docs" / "output" / "casan" / "agentops-dashboard.html"
|
LEGACY_DASHBOARD = ROOT / "docs" / "output" / "casan" / "agentops-dashboard.html"
|
||||||
|
|||||||
@@ -1 +1 @@
|
|||||||
12941
|
16827
|
||||||
|
|||||||
@@ -1 +1 @@
|
|||||||
b39f21f536ddb5ce72aac871e08584044fc2578f02026a79916b92dbd8738e86
|
dba1854708ce590be46221a77130fce18d2f3242804a8811ae6de815f79020b8
|
||||||
|
|||||||
Reference in New Issue
Block a user