feat(h6): AgentOps hardening — live alerting, provider-API reconcile, hosted dashboard, window breaker (79→80)
Close the three gaps the scoring report itself flagged for H6 plus V15, each as a real MVP + fail-able adversarial test (same pattern that lifted H5): - D1 alert-dispatch.sh: alerts POST to a real HTTP webhook (severity routing, dedup window, retry) + dead-letter queue with redelivery; fail-loud in strict. Wired into agent-metrics.sh so a failing step pages live end-to-end. - D2 provider-usage-fetch.sh + telemetry-reconcile.sh: pull usage from a provider usage HTTP API (all-or-nothing schema gate, fail-loud) + reconcile local vs provider ground truth — token under-reporting/hidden runs => TELEMETRY_DISCREPANCY. - D3 dashboard-serve.sh + dashboard-server.py: serve the dashboard over HTTP with a stale-aware /healthz probe (fresh=200 ok, telemetry silent-death=503 stale). - D4 circuit-breaker-check.sh: sliding-window failure-rate breaker (V15) — interleaved successes no longer evade the consecutive-failure breaker (CIRCUIT_OPEN_WINDOW). New suite phase-h6-agentops-tests.sh: 20/20, all live against local HTTP endpoints (webhook sink, mock provider API, dashboard server) — deterministic, no model needed. Also fix sign-policy-bundle.sh key-sync invariant: the local-fallback branch only exported policy-public.pem when generating a NEW key, so a Vault-DOWN run after a Vault-signed run verified a local-key signature against the Vault pubkey (RSA padding error, run-casan4 died mid-suite). Now always re-exports the pubkey before signing — same fix class as tool-audit-lib.sh / governance-check.sh. Full battery re-run sequentially: 175/175 PASS, 0 FAIL across 8 suites (KMS SKIP this run — Vault down; validated live 2026-07-04). Docs synced: scoring-run-report (H6 79→80, no harness below 80, 155→175), CASAN_HARDENING_STATUS (Phase 5 D1–D4), Plan-07, submission README, and run-hardening.sh (H6+ scenes HO1–HO4). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
thanhnv
co-authored by
Claude Fable 5
Claude Fable 5
parent
2af67ef6a5
commit
da66a36f97
@@ -67,7 +67,7 @@ say " • Track A : đánh bại obfuscation nâng cao + fail-closed + toàn v
|
||||
say " • Track C-MVP: chặn HÀNH ĐỘNG nguy hiểm (không chỉ tên tool) + supply-chain + chống rò rỉ + sandbox"
|
||||
say " • Evidence Pack: gói bằng chứng ký số, đổi 1 byte là CHỨNG NHẬN VÔ HIỆU"
|
||||
echo
|
||||
say "Mỗi control có test đối kháng riêng (fail-able): gỡ control → test đỏ. Tổng: baseline 79 + 61 mới = ${B}140 checks${R}."
|
||||
say "Mỗi control có test đối kháng riêng (fail-able): gỡ control → test đỏ. Tổng: baseline 79 + 96 mới = ${B}175 checks${R}."
|
||||
pause
|
||||
|
||||
# ============================================================================
|
||||
@@ -237,7 +237,7 @@ pause
|
||||
|
||||
# ============================================================================
|
||||
set_step H5-HARDENING
|
||||
banner "⭐ H5+ — GOVERNANCE HARDENING (nâng harness thấp nhất: H5 76 → ~85)"
|
||||
banner "⭐ H5+ — GOVERNANCE HARDENING (nâng harness thấp nhất: H5 76 → 80)"
|
||||
say "Ba điểm hở lớn nhất của H5 — duyệt tin env-var · khoá ký local · audit xoá được — nay vá thật."
|
||||
|
||||
REG="$ROOT/.specify/level5/central-governance/reviewers.registry"
|
||||
@@ -298,6 +298,114 @@ verdict $rc
|
||||
expect "AUDIT_GAP_DETECTED · exit=1 — ledger ngoài vẫn giữ HEAD-2, nên xoá log local là lộ ngay."
|
||||
pause
|
||||
|
||||
# ============================================================================
|
||||
set_step H6-HARDENING
|
||||
banner "⭐ H6+ — AGENTOPS HARDENING (harness thấp nhất cuối cùng: H6 79 → 80)"
|
||||
say "Ba gap của H6 — alert chỉ ghi file · telemetry import tay · dashboard tĩnh — nay chạy LIVE qua HTTP."
|
||||
|
||||
# webhook sink LIVE (đóng vai Slack/PagerDuty local)
|
||||
SINKF="$W/sink-received.jsonl"; : > "$SINKF"
|
||||
cat > "$W/sink.py" <<'PYEOF'
|
||||
import sys
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
port, out = int(sys.argv[1]), sys.argv[2]
|
||||
class H(BaseHTTPRequestHandler):
|
||||
def do_POST(self):
|
||||
n = int(self.headers.get("Content-Length", 0))
|
||||
with open(out, "ab") as f: f.write(self.rfile.read(n) + b"\n")
|
||||
self.send_response(200); self.end_headers(); self.wfile.write(b'{"ok":true}')
|
||||
def log_message(self, *a): pass
|
||||
HTTPServer(("127.0.0.1", port), H).serve_forever()
|
||||
PYEOF
|
||||
SINK_PORT=18686
|
||||
python "$W/sink.py" "$SINK_PORT" "$SINKF" & SINK_PID=$!
|
||||
sleep 1
|
||||
|
||||
card "HO1" "Alerting LIVE: step fail → webhook nhận được page 🔥" "D1 · dedup + dead-letter"
|
||||
attack "Trước đây alert chỉ append vào alerts.log — 3h sáng không ai bị đánh thức."
|
||||
guard "alert-dispatch: POST tới webhook thật + dedup window (không page đôi) + dead-letter khi kênh chết (strict → fail-loud)."
|
||||
printf 'demo input\n' > "$W/h6in.txt"
|
||||
cmd "CASAN_ALERT_WEBHOOK=http://127.0.0.1:$SINK_PORT/hook agent-metrics.sh ... -- bash -c 'exit 3'"
|
||||
env CASAN_ALERT_WEBHOOK="http://127.0.0.1:$SINK_PORT/hook" CASAN_AGENTOPS_DIR="$W/aops" \
|
||||
CASAN_AGENT_NAME="video.demo" CASAN_STEP_NAME="deploy_step" \
|
||||
bash "$S/agent-metrics.sh" "$W/h6in.txt" "$W/h6out.txt" -- bash -c 'exit 3' >/dev/null 2>&1
|
||||
grep -o '"alert_type": "execution-failed"' "$SINKF" | head -1
|
||||
verdict 0
|
||||
say "→ webhook CHẾT + strict: alert không mất — vào dead-letter, kênh sống lại thì flush redeliver:"
|
||||
cmd "CASAN_ALERT_WEBHOOK=http://127.0.0.1:1/hook CASAN_ALERT_STRICT=1 alert-dispatch.sh <alert>"
|
||||
printf '{"body":{"alert.type":"cost-spike","step.name":"plan"},"resource":{"service.name":"video.demo"}}\n' > "$W/al.json"
|
||||
env CASAN_ALERT_WEBHOOK="http://127.0.0.1:1/hook" CASAN_ALERT_STRICT=1 CASAN_AGENTOPS_DIR="$W/aops" \
|
||||
bash "$S/alert-dispatch.sh" "$W/al.json" 2>&1 | grep -o "ALERT_DELIVERY_FAILED.*"; rc=${PIPESTATUS[0]}
|
||||
verdict $rc
|
||||
env CASAN_ALERT_WEBHOOK="http://127.0.0.1:$SINK_PORT/hook" CASAN_AGENTOPS_DIR="$W/aops" \
|
||||
bash "$S/alert-dispatch.sh" --flush-deadletter | grep -o "ALERT_DEADLETTER_FLUSHED.*"
|
||||
expect "Step fail → page LIVE; kênh chết → fail-loud + dead-letter; kênh hồi → redelivered=1 remaining=0."
|
||||
pause
|
||||
|
||||
card "HO2" "Provider-telemetry API + đối soát: giấu chi phí là lộ 🔥" "D2 · anti under-reporting"
|
||||
attack "Kẻ tấn công cắt bớt metrics local để một step exfil/runaway trông 'rẻ' — không ai truy."
|
||||
guard "provider-usage-fetch: kéo usage từ API provider (ground truth) · telemetry-reconcile: local khai thiếu token → TELEMETRY_DISCREPANCY."
|
||||
printf '[{"provider":"ollama","model":"ornith:9b","run_id":"v1","step":"plan","input_tokens":200,"output_tokens":300,"total_tokens":500,"cost_usd":0.0,"latency_ms":900,"status":"success"}]\n' > "$W/usage.json"
|
||||
cat > "$W/api.py" <<'PYEOF'
|
||||
import sys
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
port, src = int(sys.argv[1]), sys.argv[2]
|
||||
class H(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
data = open(src, "rb").read()
|
||||
self.send_response(200); self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(data))); self.end_headers(); self.wfile.write(data)
|
||||
def log_message(self, *a): pass
|
||||
HTTPServer(("127.0.0.1", port), H).serve_forever()
|
||||
PYEOF
|
||||
API_PORT=18687
|
||||
python "$W/api.py" "$API_PORT" "$W/usage.json" & API_PID=$!
|
||||
sleep 1
|
||||
cmd "bash \$S/provider-usage-fetch.sh http://127.0.0.1:$API_PORT/usage <out.jsonl>"
|
||||
bash "$S/provider-usage-fetch.sh" "http://127.0.0.1:$API_PORT/usage" "$W/prov.jsonl" | grep -o "PROVIDER_TELEMETRY_FETCHED.*"
|
||||
say "→ local khai 500 token (khớp) rồi khai 50 token (giấu 90% chi phí):"
|
||||
printf '{"step":"plan","total_tokens":500}\n' > "$W/loc-ok.jsonl"
|
||||
printf '{"step":"plan","total_tokens":50}\n' > "$W/loc-under.jsonl"
|
||||
cmd "bash \$S/telemetry-reconcile.sh <local> <provider> 10"
|
||||
bash "$S/telemetry-reconcile.sh" "$W/loc-ok.jsonl" "$W/prov.jsonl" 10 | grep -o "TELEMETRY_RECONCILED.*"
|
||||
bash "$S/telemetry-reconcile.sh" "$W/loc-under.jsonl" "$W/prov.jsonl" 10 2>&1 | grep -o "TELEMETRY_DISCREPANCY.*" | head -1; rc=${PIPESTATUS[0]}
|
||||
verdict $rc
|
||||
kill "$API_PID" 2>/dev/null; wait "$API_PID" 2>/dev/null
|
||||
expect "Khớp → RECONCILED; khai thiếu → TELEMETRY_DISCREPANCY · exit=1 — provider API là ground truth, local không tự khai được."
|
||||
pause
|
||||
|
||||
card "HO3" "Dashboard HOSTED: /healthz biết cả khi telemetry chết lặng" "D3 · stale-aware"
|
||||
attack "Dashboard tĩnh đẹp nhưng nếu pipeline ngừng ghi metrics — không ai biết nó đã chết từ tuần trước."
|
||||
guard "dashboard-serve: serve qua HTTP thật; /healthz 200 khi metrics tươi, 503 STALE khi metrics già → uptime-monitor page được."
|
||||
DASH_PORT=18688
|
||||
cmd "bash \$S/dashboard-serve.sh start $DASH_PORT ; curl /healthz"
|
||||
CASAN_AGENTOPS_DIR="$W/aops" bash "$S/dashboard-serve.sh" start "$DASH_PORT" | grep -o "DASHBOARD_HOSTED.*"
|
||||
curl -sS -m 3 "http://127.0.0.1:$DASH_PORT/healthz"; echo
|
||||
CASAN_AGENTOPS_DIR="$W/aops" bash "$S/dashboard-serve.sh" stop >/dev/null 2>&1
|
||||
say "→ metrics 6 tháng không ai ghi (touch về 2026-01-01):"
|
||||
printf '{"step":"s","total_tokens":5}\n' > "$W/stale.jsonl"; touch -t 202601010000 "$W/stale.jsonl"
|
||||
env CASAN_DASHBOARD_METRICS="$W/stale.jsonl" CASAN_AGENTOPS_DIR="$W/aops" bash "$S/dashboard-serve.sh" start "$DASH_PORT" >/dev/null 2>&1
|
||||
cmd "curl -w '%{http_code}' http://127.0.0.1:$DASH_PORT/healthz # metrics stale"
|
||||
curl -sS -m 3 -w ' http=%{http_code}\n' "http://127.0.0.1:$DASH_PORT/healthz"
|
||||
CASAN_AGENTOPS_DIR="$W/aops" bash "$S/dashboard-serve.sh" stop >/dev/null 2>&1
|
||||
verdict 0
|
||||
expect "Tươi → 200 ok; chết lặng → 503 stale — 'silent death' của telemetry giờ page được như downtime."
|
||||
pause
|
||||
|
||||
card "HO4" "Circuit-breaker window: xen kẽ thành công hết né được" "D4/V15"
|
||||
attack "Provider hỏng chập chờn: fail-success-fail-success... — counter 'N fail LIÊN TIẾP' không bao giờ trip."
|
||||
guard "Sliding window: tỷ lệ fail ≥50% trên 10 call gần nhất → CIRCUIT_OPEN_WINDOW, kệ xen kẽ."
|
||||
ALTF="$W/alt.jsonl"; : > "$ALTF"
|
||||
for i in 1 2 3 4 5; do
|
||||
printf '{"step":"s%s","status":"error","total_tokens":10}\n{"step":"s%s","status":"success","total_tokens":10}\n' "$i" "$i" >> "$ALTF"
|
||||
done
|
||||
cmd "CASAN_PROVIDER_LOG=<xen kẽ 5 fail/5 success> circuit-breaker-check.sh --breaker-only"
|
||||
env CASAN_PROVIDER_LOG="$ALTF" bash "$S/circuit-breaker-check.sh" --breaker-only 2>&1 | grep -o "CIRCUIT_OPEN_WINDOW.*" | head -1; rc=${PIPESTATUS[0]}
|
||||
verdict $rc
|
||||
expect "consecutive=1 (né được) NHƯNG rate 50%/10 call → CIRCUIT_OPEN_WINDOW · exit=1."
|
||||
kill "$SINK_PID" 2>/dev/null; wait "$SINK_PID" 2>/dev/null
|
||||
pause
|
||||
|
||||
# ============================================================================
|
||||
set_step EVIDENCE-PACK
|
||||
banner "🏆 PLAN-09 EVIDENCE PACK — 'VÌ SAO TIN OUTPUT NÀY?' (money-shot)"
|
||||
@@ -361,10 +469,11 @@ set_step HARDEN-DONE
|
||||
banner "CHỐT PART 2 — TRƯỞNG THÀNH PRODUCTION (trung thực)"
|
||||
echo "${B}${GR}✔ Track A${R}: homoglyph/zero-width/base64 chặn · strict fail-closed · telemetry bất biến · trần chi phí tuyệt đối+tích luỹ · FP=0%."
|
||||
echo "${B}${GR}✔ Track C-MVP${R}: tool-authz theo hành động · supply-chain (typosquat/postinstall) · data-exfil (secret→cloud, PII mask) · sandbox scaffold."
|
||||
echo "${B}${GR}✔ H5+ hardening${R}: approval ký-danh-tính (hết env-var) · khoá ký qua KMS (rotate + non-exportable) · WORM audit ngoài (chống xoá log) → H5 76→~85."
|
||||
echo "${B}${GR}✔ H5+ hardening${R}: approval ký-danh-tính (hết env-var) · khoá ký qua KMS (rotate + non-exportable) · WORM audit ngoài (chống xoá log) → H5 76→80."
|
||||
echo "${B}${GR}✔ H6+ hardening${R}: alerting LIVE (webhook + dead-letter) · provider-API + đối soát (bắt giấu chi phí) · dashboard hosted (/healthz stale-aware) · window breaker (V15) → H6 79→80."
|
||||
echo "${B}${GR}✔ Evidence Pack${R}: gói bằng chứng ký số, tamper 1 byte → vô hiệu; certified chỉ khi đủ cổng."
|
||||
echo
|
||||
echo "${B}${CY}Tổng test: baseline 79 (giữ nguyên) + 76 hardening = 155 checks, 0 fail.${R}"
|
||||
echo "${B}${CY}Tổng test: baseline 79 (giữ nguyên) + 96 hardening = 175 checks, 0 fail.${R}"
|
||||
echo "${DIM}Trung thực: sandbox là scaffold (chưa cô lập kernel); Track B + C-Governance/Ops là roadmap sau thi. Chi tiết: casan-next-plans/CASAN_HARDENING_STATUS.md${R}"
|
||||
rule
|
||||
set_step DONE
|
||||
|
||||
Reference in New Issue
Block a user