From 7e998f67c276c1bbf235fbe6b3b374ae41130e8c Mon Sep 17 00:00:00 2001 From: thanhnv Date: Fri, 3 Jul 2026 22:25:00 +0900 Subject: [PATCH] =?UTF-8?q?feat(h5-h6-hardening):=20Plan-07=20Track=20A=20?= =?UTF-8?q?=E2=80=94=20A4=20telemetry=20integrity,=20A5=20cost=20controls?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A4 (V9): telemetry-integrity.sh binds provider-usage.jsonl + cost/metrics.jsonl to a signed manifest head. Tampering a token flips the head (MISMATCH); an attacker who rewrites the head cannot re-sign it (SIGNATURE_INVALID) without the off-repo key. sign-audit-head.sh now also signs telemetry (best-effort). Supports CASAN_AUDIT_PRIV/PUB overrides for self-contained verification. A5 (V12/V13/V14): cost-spike-detect.sh adds an absolute per-call cap (CASAN_COST_ABSOLUTE_MAX_TOKENS, enforced from record #1 → catches slow-boil and cold-start) and a cumulative budget (CASAN_COST_CUMULATIVE_BUDGET_TOKENS → catches under-threshold spray), keeping the existing median×mult spike test. Backward compatible: <3 records with no caps still exits 3. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../scripts/bash/cost-spike-detect.sh | 75 +++++++++--- .../.specify/scripts/bash/sign-audit-head.sh | 5 + .../scripts/bash/telemetry-integrity.sh | 107 ++++++++++++++++++ 3 files changed, 169 insertions(+), 18 deletions(-) create mode 100755 AINative_OKR_CASAN5/.specify/scripts/bash/telemetry-integrity.sh diff --git a/AINative_OKR_CASAN5/.specify/scripts/bash/cost-spike-detect.sh b/AINative_OKR_CASAN5/.specify/scripts/bash/cost-spike-detect.sh index 6d8f142..14b790e 100755 --- a/AINative_OKR_CASAN5/.specify/scripts/bash/cost-spike-detect.sh +++ b/AINative_OKR_CASAN5/.specify/scripts/bash/cost-spike-detect.sh @@ -1,15 +1,22 @@ #!/usr/bin/env bash set -uo pipefail -# CASAN H6 cost-spike detector (WP-C). +# CASAN H6 cost detector (WP-C + Track A cost controls A5 / V12,V13,V14). # Answers the H6 key question: "if a step suddenly costs 3x the tokens, does -# anyone know?". Reads real per-step usage from provider-usage.jsonl, computes -# the median total_tokens across steps, and flags any step exceeding -# MULTIPLIER x median. Exits non-zero if a spike is found (so a pipeline/CI gate -# goes red). +# anyone know?". Reads real per-step usage from provider-usage.jsonl and applies +# three complementary controls: +# 1. Relative spike — any step > MULTIPLIER x median (needs >=3 records). +# 2. Absolute cap — any single step > CASAN_COST_ABSOLUTE_MAX_TOKENS. Works +# from the very first record, so a slow-boil that drags the median up +# (V12) and a cold-start with no history (V14) are BOTH still caught. +# 3. Cumulative budget — sum of tokens > CASAN_COST_CUMULATIVE_BUDGET_TOKENS, +# catching a spray of many under-threshold calls (V13). # # Usage: cost-spike-detect.sh [provider-usage.jsonl] [multiplier] -# Exit: 0 no spike, 2 spike detected, 64 usage, 3 not enough data. +# Env: CASAN_COST_ABSOLUTE_MAX_TOKENS per-call hard cap (0/unset = off) +# CASAN_COST_CUMULATIVE_BUDGET_TOKENS run/session budget (0/unset = off) +# Exit: 0 no violation, 2 violation (spike/absolute/cumulative), 64 usage, +# 3 not enough data for the RELATIVE test and no absolute/cumulative cap set. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" @@ -18,9 +25,14 @@ MULT="${2:-3.0}" [[ -f "$LOG" ]] || { echo "COST_SPIKE_NO_DATA file=$LOG" >&2; exit 3; } +CASAN_COST_ABSOLUTE_MAX_TOKENS="${CASAN_COST_ABSOLUTE_MAX_TOKENS:-0}" \ +CASAN_COST_CUMULATIVE_BUDGET_TOKENS="${CASAN_COST_CUMULATIVE_BUDGET_TOKENS:-0}" \ python - "$LOG" "$MULT" <<'PY' -import json, sys, statistics +import json, os, sys, statistics path, mult = sys.argv[1], float(sys.argv[2]) +abs_max = int(os.environ.get("CASAN_COST_ABSOLUTE_MAX_TOKENS", "0") or "0") +cum_budget = int(os.environ.get("CASAN_COST_CUMULATIVE_BUDGET_TOKENS", "0") or "0") + rows = [] for line in open(path, encoding="utf-8"): line = line.strip() @@ -32,18 +44,45 @@ for line in open(path, encoding="utf-8"): continue if "total_tokens" in r: rows.append((r.get("step", "?"), int(r["total_tokens"]))) -if len(rows) < 3: - sys.stderr.write(f"COST_SPIKE_NO_DATA records={len(rows)} (need >=3)\n") - raise SystemExit(3) + tokens = [t for _, t in rows] -median = statistics.median(tokens) -threshold = median * mult -spikes = [(s, t) for s, t in rows if t > threshold] -print(f"records={len(rows)} median_tokens={median} threshold={threshold:.0f} (x{mult})") -for s, t in spikes: - print(f"SPIKE step={s} tokens={t} (> {threshold:.0f})") -if spikes: - sys.stderr.write(f"COST_SPIKE_DETECTED count={len(spikes)}\n") +total = sum(tokens) +violations = [] + +# 2. Absolute per-call cap — enforced from the first record (cold-start safe). +if abs_max > 0: + over = [(s, t) for s, t in rows if t > abs_max] + for s, t in over: + violations.append(f"ABSOLUTE step={s} tokens={t} (> cap {abs_max})") + +# 3. Cumulative budget over the whole run/session. +if cum_budget > 0 and total > cum_budget: + violations.append(f"CUMULATIVE total={total} (> budget {cum_budget})") + +# 1. Relative median spike — needs enough history. +median = threshold = None +if len(rows) >= 3: + median = statistics.median(tokens) + threshold = median * mult + for s, t in rows: + if t > threshold: + violations.append(f"SPIKE step={s} tokens={t} (> {threshold:.0f} = median x{mult})") + +med_str = f"{median}" if median is not None else "n/a(<3 records)" +print(f"records={len(rows)} total_tokens={total} median_tokens={med_str} " + f"abs_cap={abs_max or 'off'} cum_budget={cum_budget or 'off'}") +for v in violations: + print(v) + +if violations: + sys.stderr.write(f"COST_VIOLATION_DETECTED count={len(violations)}\n") raise SystemExit(2) + +# No violations. If we could not run the relative test AND no absolute/cumulative +# cap was configured, we truly had nothing to check -> preserve the old rc=3. +if len(rows) < 3 and abs_max == 0 and cum_budget == 0: + sys.stderr.write(f"COST_SPIKE_NO_DATA records={len(rows)} (need >=3, no absolute/cumulative cap set)\n") + raise SystemExit(3) + print("COST_SPIKE_NONE") PY diff --git a/AINative_OKR_CASAN5/.specify/scripts/bash/sign-audit-head.sh b/AINative_OKR_CASAN5/.specify/scripts/bash/sign-audit-head.sh index d94ac41..8f8c62c 100755 --- a/AINative_OKR_CASAN5/.specify/scripts/bash/sign-audit-head.sh +++ b/AINative_OKR_CASAN5/.specify/scripts/bash/sign-audit-head.sh @@ -110,3 +110,8 @@ else openssl dgst -sha256 -sign "$AUDIT_PRIV" -out "$HEAD_SIG" "$HEAD_FILE" echo "SIGN_AUDIT_HEAD_OK head=$HEAD_HASH anchor=local-file" fi + +# A4/V9: also bind token/cost telemetry to a signed manifest so tampering with +# provider-usage.jsonl / metrics.jsonl is detectable. Best-effort — never fails +# the audit signing step (verify-telemetry-integrity.sh is the gate). +bash "$SCRIPT_DIR/telemetry-integrity.sh" sign >/dev/null 2>&1 || true diff --git a/AINative_OKR_CASAN5/.specify/scripts/bash/telemetry-integrity.sh b/AINative_OKR_CASAN5/.specify/scripts/bash/telemetry-integrity.sh new file mode 100755 index 0000000..e6ea6d4 --- /dev/null +++ b/AINative_OKR_CASAN5/.specify/scripts/bash/telemetry-integrity.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +set -uo pipefail + +# CASAN H5 — Telemetry integrity proof (Track A, V9). +# +# Token/cost telemetry (provider-usage.jsonl, cost/metrics.jsonl) previously sat +# OUTSIDE the signed audit chain, so a forger could rewrite token counts to hide +# cost abuse and nothing would detect it. This binds those files to a signed +# manifest: any byte change flips the manifest head hash, and because the head +# is RSA-signed with an off-repo key, a forger cannot re-sign a rewritten head. +# +# Usage: +# telemetry-integrity.sh sign — hash telemetry files, write + sign manifest head +# telemetry-integrity.sh verify — recompute, compare head, verify signature +# +# Key resolution (sign): CASAN_AUDIT_PRIV, else level5/central-governance/audit-private.pem +# Key resolution (verify): CASAN_AUDIT_PUB, else level5/central-governance/audit-public.pem +# +# Outputs (under .specify/logs/level5/): +# telemetry-manifest.json — {basename: sha256} for each telemetry file +# telemetry-head.txt — sha256 over the canonical manifest text +# telemetry-head.sig — RSA signature of telemetry-head.txt (when a key exists) +# +# Exit: 0 ok, 1 tamper/mismatch/invalid-signature, 64 usage. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" +L5_DIR="$PROJECT_ROOT/.specify/logs/level5" +COST_DIR="$PROJECT_ROOT/.specify/logs/cost" +mkdir -p "$L5_DIR" + +CMD="${1:-}" +MANIFEST="$L5_DIR/telemetry-manifest.json" +HEAD_FILE="$L5_DIR/telemetry-head.txt" +HEAD_SIG="$L5_DIR/telemetry-head.sig" +AUDIT_PRIV="${CASAN_AUDIT_PRIV:-$PROJECT_ROOT/.specify/level5/central-governance/audit-private.pem}" +AUDIT_PUB="${CASAN_AUDIT_PUB:-$PROJECT_ROOT/.specify/level5/central-governance/audit-public.pem}" + +# Telemetry files to bind. Missing files hash to the literal "MISSING" so the +# manifest is stable and a deletion is itself a detectable change. +TELEMETRY_FILES=( + "$L5_DIR/provider-usage.jsonl" + "$COST_DIR/metrics.jsonl" +) + +compute_head() { + # Prints: manifest-json on line 1, head-hash on line 2. + python - "$@" <<'PY' +import hashlib, json, os, sys +files = sys.argv[1:] +manifest = {} +for path in files: + name = os.path.basename(path) + if os.path.isfile(path): + with open(path, "rb") as f: + manifest[name] = hashlib.sha256(f.read()).hexdigest() + else: + manifest[name] = "MISSING" +canonical = json.dumps(manifest, sort_keys=True, separators=(",", ":")) +head = hashlib.sha256(canonical.encode()).hexdigest() +print(canonical) +print(head) +PY +} + +case "$CMD" in + sign) + OUT="$(compute_head "${TELEMETRY_FILES[@]}")" + CANON="$(printf '%s' "$OUT" | sed -n '1p')" + HEAD="$(printf '%s' "$OUT" | sed -n '2p')" + printf '%s' "$CANON" > "$MANIFEST" + printf '%s' "$HEAD" > "$HEAD_FILE" + if [[ -f "$AUDIT_PRIV" ]] && command -v openssl >/dev/null 2>&1; then + openssl dgst -sha256 -sign "$AUDIT_PRIV" -out "$HEAD_SIG" "$HEAD_FILE" + echo "TELEMETRY_INTEGRITY_SIGNED head=$HEAD anchor=signed files=${#TELEMETRY_FILES[@]}" + else + rm -f "$HEAD_SIG" + echo "TELEMETRY_INTEGRITY_SIGNED head=$HEAD anchor=unsigned files=${#TELEMETRY_FILES[@]} (no private key)" + fi + ;; + verify) + if [[ ! -f "$HEAD_FILE" ]]; then + echo "TELEMETRY_INTEGRITY_MISSING no telemetry-head.txt (run: telemetry-integrity.sh sign)" >&2 + exit 1 + fi + OUT="$(compute_head "${TELEMETRY_FILES[@]}")" + HEAD_NOW="$(printf '%s' "$OUT" | sed -n '2p')" + HEAD_STORED="$(cat "$HEAD_FILE")" + if [[ "$HEAD_NOW" != "$HEAD_STORED" ]]; then + echo "TELEMETRY_INTEGRITY_MISMATCH computed=$HEAD_NOW stored=$HEAD_STORED" >&2 + exit 1 + fi + if [[ -f "$HEAD_SIG" && -f "$AUDIT_PUB" ]] && command -v openssl >/dev/null 2>&1; then + if ! openssl dgst -sha256 -verify "$AUDIT_PUB" -signature "$HEAD_SIG" "$HEAD_FILE" >/dev/null 2>&1; then + echo "TELEMETRY_INTEGRITY_SIGNATURE_INVALID head=$HEAD_STORED" >&2 + exit 1 + fi + echo "TELEMETRY_INTEGRITY_VALID anchor=signed head=$HEAD_STORED" + else + echo "TELEMETRY_INTEGRITY_VALID anchor=unsigned head=$HEAD_STORED" + fi + ;; + *) + echo "Usage: telemetry-integrity.sh {sign|verify}" >&2 + exit 64 + ;; +esac