feat: export aggregate Prometheus metrics

2026-07-18 09:25:56 +07:00
parent 083a1de875
commit 9785e13c82
5 changed files with 70 additions and 2 deletions
@@ -39,7 +39,7 @@
 | T2.1 | KMS mặc định + HSM | 🟡 production fail-closed + Docker lab ✅ | `infra/local-prod` dựng Vault dev @ `:18200`; `infra-lab verify` chứng minh sign+verify. `sign-audit-head.sh` nay từ chối local-key fallback khi `CASAN_PROFILE=prod` (SEC-02 test). Còn: Vault/AWS-KMS/CloudHSM thật + token ngắn hạn qua IdP và Transit encryption cho dữ liệu tenant. |
 | T2.2 | WORM store thật (S3 Object Lock) | 🟡 Docker lab ✅ (MinIO Object Lock) | MinIO bucket `casan-worm` bật Object Lock COMPLIANCE 1d. Prod: S3 Object Lock/QLDB + trusted timestamp; cần AWS creds + bucket Object-Lock. |
 | T2.3 | Dashboard deploy + auth | 🟡 Docker lab ✅ (nginx basic auth + /healthz) | Dashboard container + nginx reverse-proxy @ `:18080`, user/pass lab `casan/casan`; `/healthz` exposed. Prod: host/TLS/OIDC or enterprise auth. |
-| T2.4 | Kênh alert managed + on-call | 🟡 Docker lab ✅ (webhook emulator) | `alert-webhook` @ `:19092` nhận live POST. Prod: trỏ `CASAN_ALERT_WEBHOOK` tới Slack/PagerDuty + rota/on-call thật. |
+| T2.4 | Kênh alert managed + on-call | 🟡 Docker lab + metrics export ✅ | `alert-webhook` @ `:19092` nhận live POST; Control Panel exports aggregate-only Prometheus text at `/api/v1/metrics` (no trace/tenant/prompt labels). Prod: restrict scrape endpoint at the reverse proxy, trỏ `CASAN_ALERT_WEBHOOK` tới Slack/PagerDuty và thiết lập rota/on-call thật. |
 | T2.5 | Billing-API telemetry thật | 🟡 Docker lab ✅ (billing API mock) | `billing-api` @ `:19093/usage`; `provider-usage-fetch.sh` import được provider telemetry. Prod: OpenAI/Anthropic usage API thật + key. |
 | T2.6 | Sandbox: rootless/nsjail + base image CI | 🟡 hardened Docker runner + test | `sandbox-container.sh` forces a non-root workload user and production refuses a rootful daemon; network, root filesystem, capabilities and quotas remain locked down. Remaining: provision rootless Docker on the customer runner, image allowlist/signature policy, and optional nsjail/bubblewrap profile for daemonless Linux CI. |
 | T2.7 | Backup/restore + restore drill | 🟡 automated + test | `state-backup.sh` creates manifest/hash-bound state snapshots, requires encryption key in prod, restores only to an empty explicit directory; `phase-state-backup` is in CI. Remaining: customer object-store replication, retention/RPO/RTO, and a retained production restore-drill record. |
@@ -23,6 +23,12 @@ Open http://127.0.0.1:5174 — panels show REAL metrics from `.specify/logs/**`.
 `incidents` · `tools` · `traceability` · `drift` · `cost` · `GET /healthz` (200 fresh /
 503 stale — fail-loud, mirrors `dashboard-server.py`).

+Metrics export: `GET /api/v1/metrics` provides Prometheus text exposition for
+aggregate freshness, run/failure/cost/token and H4/H5/action/incident counters.
+It intentionally contains no tenant, actor, trace, prompt or Evidence Pack
+labels. In production restrict the path at the authenticated reverse proxy to
+the monitoring network or service account.
+
 Command Center:

 - `GET /api/v1/command` — Plan-13 §8.6 read-only executive surface. Returns eight
@@ -1,6 +1,6 @@
 // Read-only Ops Console API. Every handler returns the standard ok() envelope. No writes,
 // no auth (loopback-bound, "Đọc ≠ Ghi"); management/RBAC is Plan-13 Track 2/3 (future).
-import { Controller, Get, Inject, Param, Query, Req, Res } from '@nestjs/common';
+import { Controller, Get, Header, Inject, Param, Query, Req, Res } from '@nestjs/common';
 import type { Request, Response } from 'express';
 import { ok } from '../common/api-response.js';
 import { TelemetryService } from './telemetry.service.js';
@@ -16,6 +16,14 @@ export class TelemetryController {
    return ok(this.svc.overview());
  }

+  // Production reverse proxy policy must restrict this aggregate-only endpoint
+  // to the monitoring network / service account.
+  @Get('metrics')
+  @Header('Content-Type', 'text/plain; version=0.0.4; charset=utf-8')
+  metrics() {
+    return this.svc.prometheusMetrics();
+  }
+
  @Get('runs')
  runs(@Query('limit') limit?: string) {
    const n = Math.min(Math.max(Number(limit) || 50, 1), 500);
@@ -263,6 +263,51 @@ export class TelemetryService {
    };
  }

+  // Prometheus exposition is deliberately aggregate-only. Do not add trace IDs,
+  // actor/tenant identifiers, prompts, filenames or evidence content as labels:
+  // those would turn an operations endpoint into a data-exfiltration path.
+  prometheusMetrics() {
+    const overview = this.overview();
+    const totals = overview.totals;
+    const lines = [
+      '# HELP casan_telemetry_stale Whether the primary telemetry feed is stale (1=true).',
+      '# TYPE casan_telemetry_stale gauge',
+      `casan_telemetry_stale ${overview.stale ? 1 : 0}`,
+      '# HELP casan_telemetry_age_seconds Age of the primary telemetry feed in seconds.',
+      '# TYPE casan_telemetry_age_seconds gauge',
+      `casan_telemetry_age_seconds ${overview.age_s ?? -1}`,
+      '# HELP casan_runs_total Number of telemetry records retained by CASAN.',
+      '# TYPE casan_runs_total gauge',
+      `casan_runs_total ${totals.runs}`,
+      '# HELP casan_failures_total Number of retained failed telemetry records.',
+      '# TYPE casan_failures_total gauge',
+      `casan_failures_total ${totals.failures}`,
+      '# HELP casan_cost_usd_total Aggregate estimated CASAN execution cost in USD.',
+      '# TYPE casan_cost_usd_total gauge',
+      `casan_cost_usd_total ${totals.total_cost}`,
+      '# HELP casan_provider_tokens_total Aggregate provider token usage.',
+      '# TYPE casan_provider_tokens_total gauge',
+      `casan_provider_tokens_total ${totals.provider_tokens}`,
+      '# HELP casan_security_blocks_total H4 security verdicts blocked.',
+      '# TYPE casan_security_blocks_total gauge',
+      `casan_security_blocks_total ${overview.harness_signals['H4-security'].blocked}`,
+      '# HELP casan_governance_denials_total H5 governance decisions denied.',
+      '# TYPE casan_governance_denials_total gauge',
+      `casan_governance_denials_total ${overview.harness_signals['H5-governance'].denied}`,
+      '# HELP casan_tool_denials_total Tool registry decisions denied.',
+      '# TYPE casan_tool_denials_total gauge',
+      `casan_tool_denials_total ${totals.tool_denies}`,
+      '# HELP casan_action_blocks_total Governed action gate block outcomes.',
+      '# TYPE casan_action_blocks_total gauge',
+      `casan_action_blocks_total ${totals.action_blocks}`,
+      '# HELP casan_critical_incidents_total Retained critical incident records.',
+      '# TYPE casan_critical_incidents_total gauge',
+      `casan_critical_incidents_total ${overview.harness_signals.incidents.critical}`,
+      '',
+    ];
+    return lines.join('\n');
+  }
+
  runs(limit = 50) {
    const metrics = readJsonl(PATHS.metrics);
    return { ...this.freshness(), count: metrics.length, runs: recent(metrics, limit) };
@@ -40,6 +40,15 @@ test('overview() returns real aggregated shape, never throws on the repo state',
  assert.ok(o.audit_chain.records >= 0);
 });

+test('prometheusMetrics() exports aggregate-safe operational metrics only', () => {
+  const metrics = new TelemetryService().prometheusMetrics();
+  assert.match(metrics, /^# HELP casan_telemetry_stale/m);
+  assert.match(metrics, /^casan_runs_total \d+/m);
+  assert.match(metrics, /^casan_security_blocks_total \d+/m);
+  assert.match(metrics, /^casan_governance_denials_total \d+/m);
+  assert.doesNotMatch(metrics, /trace_id|tenant|prompt|evidence/i);
+});
+
 test('security()/governance()/cost() return objects with expected keys', () => {
  const svc = new TelemetryService();
  assert.ok(typeof (svc.security() as any).by_status === 'object');