feat(h5-h6-hardening): Plan-07 Track A — A4 telemetry integrity, A5 cost controls
A4 (V9): telemetry-integrity.sh binds provider-usage.jsonl + cost/metrics.jsonl to a signed manifest head. Tampering a token flips the head (MISMATCH); an attacker who rewrites the head cannot re-sign it (SIGNATURE_INVALID) without the off-repo key. sign-audit-head.sh now also signs telemetry (best-effort). Supports CASAN_AUDIT_PRIV/PUB overrides for self-contained verification. A5 (V12/V13/V14): cost-spike-detect.sh adds an absolute per-call cap (CASAN_COST_ABSOLUTE_MAX_TOKENS, enforced from record #1 → catches slow-boil and cold-start) and a cumulative budget (CASAN_COST_CUMULATIVE_BUDGET_TOKENS → catches under-threshold spray), keeping the existing median×mult spike test. Backward compatible: <3 records with no caps still exits 3. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
thanhnv
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
ac40c0b281
commit
7e998f67c2
@@ -1,15 +1,22 @@
|
||||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
# CASAN H6 cost-spike detector (WP-C).
|
||||
# CASAN H6 cost detector (WP-C + Track A cost controls A5 / V12,V13,V14).
|
||||
# Answers the H6 key question: "if a step suddenly costs 3x the tokens, does
|
||||
# anyone know?". Reads real per-step usage from provider-usage.jsonl, computes
|
||||
# the median total_tokens across steps, and flags any step exceeding
|
||||
# MULTIPLIER x median. Exits non-zero if a spike is found (so a pipeline/CI gate
|
||||
# goes red).
|
||||
# anyone know?". Reads real per-step usage from provider-usage.jsonl and applies
|
||||
# three complementary controls:
|
||||
# 1. Relative spike — any step > MULTIPLIER x median (needs >=3 records).
|
||||
# 2. Absolute cap — any single step > CASAN_COST_ABSOLUTE_MAX_TOKENS. Works
|
||||
# from the very first record, so a slow-boil that drags the median up
|
||||
# (V12) and a cold-start with no history (V14) are BOTH still caught.
|
||||
# 3. Cumulative budget — sum of tokens > CASAN_COST_CUMULATIVE_BUDGET_TOKENS,
|
||||
# catching a spray of many under-threshold calls (V13).
|
||||
#
|
||||
# Usage: cost-spike-detect.sh [provider-usage.jsonl] [multiplier]
|
||||
# Exit: 0 no spike, 2 spike detected, 64 usage, 3 not enough data.
|
||||
# Env: CASAN_COST_ABSOLUTE_MAX_TOKENS per-call hard cap (0/unset = off)
|
||||
# CASAN_COST_CUMULATIVE_BUDGET_TOKENS run/session budget (0/unset = off)
|
||||
# Exit: 0 no violation, 2 violation (spike/absolute/cumulative), 64 usage,
|
||||
# 3 not enough data for the RELATIVE test and no absolute/cumulative cap set.
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
|
||||
@@ -18,9 +25,14 @@ MULT="${2:-3.0}"
|
||||
|
||||
[[ -f "$LOG" ]] || { echo "COST_SPIKE_NO_DATA file=$LOG" >&2; exit 3; }
|
||||
|
||||
CASAN_COST_ABSOLUTE_MAX_TOKENS="${CASAN_COST_ABSOLUTE_MAX_TOKENS:-0}" \
|
||||
CASAN_COST_CUMULATIVE_BUDGET_TOKENS="${CASAN_COST_CUMULATIVE_BUDGET_TOKENS:-0}" \
|
||||
python - "$LOG" "$MULT" <<'PY'
|
||||
import json, sys, statistics
|
||||
import json, os, sys, statistics
|
||||
path, mult = sys.argv[1], float(sys.argv[2])
|
||||
abs_max = int(os.environ.get("CASAN_COST_ABSOLUTE_MAX_TOKENS", "0") or "0")
|
||||
cum_budget = int(os.environ.get("CASAN_COST_CUMULATIVE_BUDGET_TOKENS", "0") or "0")
|
||||
|
||||
rows = []
|
||||
for line in open(path, encoding="utf-8"):
|
||||
line = line.strip()
|
||||
@@ -32,18 +44,45 @@ for line in open(path, encoding="utf-8"):
|
||||
continue
|
||||
if "total_tokens" in r:
|
||||
rows.append((r.get("step", "?"), int(r["total_tokens"])))
|
||||
if len(rows) < 3:
|
||||
sys.stderr.write(f"COST_SPIKE_NO_DATA records={len(rows)} (need >=3)\n")
|
||||
raise SystemExit(3)
|
||||
|
||||
tokens = [t for _, t in rows]
|
||||
median = statistics.median(tokens)
|
||||
threshold = median * mult
|
||||
spikes = [(s, t) for s, t in rows if t > threshold]
|
||||
print(f"records={len(rows)} median_tokens={median} threshold={threshold:.0f} (x{mult})")
|
||||
for s, t in spikes:
|
||||
print(f"SPIKE step={s} tokens={t} (> {threshold:.0f})")
|
||||
if spikes:
|
||||
sys.stderr.write(f"COST_SPIKE_DETECTED count={len(spikes)}\n")
|
||||
total = sum(tokens)
|
||||
violations = []
|
||||
|
||||
# 2. Absolute per-call cap — enforced from the first record (cold-start safe).
|
||||
if abs_max > 0:
|
||||
over = [(s, t) for s, t in rows if t > abs_max]
|
||||
for s, t in over:
|
||||
violations.append(f"ABSOLUTE step={s} tokens={t} (> cap {abs_max})")
|
||||
|
||||
# 3. Cumulative budget over the whole run/session.
|
||||
if cum_budget > 0 and total > cum_budget:
|
||||
violations.append(f"CUMULATIVE total={total} (> budget {cum_budget})")
|
||||
|
||||
# 1. Relative median spike — needs enough history.
|
||||
median = threshold = None
|
||||
if len(rows) >= 3:
|
||||
median = statistics.median(tokens)
|
||||
threshold = median * mult
|
||||
for s, t in rows:
|
||||
if t > threshold:
|
||||
violations.append(f"SPIKE step={s} tokens={t} (> {threshold:.0f} = median x{mult})")
|
||||
|
||||
med_str = f"{median}" if median is not None else "n/a(<3 records)"
|
||||
print(f"records={len(rows)} total_tokens={total} median_tokens={med_str} "
|
||||
f"abs_cap={abs_max or 'off'} cum_budget={cum_budget or 'off'}")
|
||||
for v in violations:
|
||||
print(v)
|
||||
|
||||
if violations:
|
||||
sys.stderr.write(f"COST_VIOLATION_DETECTED count={len(violations)}\n")
|
||||
raise SystemExit(2)
|
||||
|
||||
# No violations. If we could not run the relative test AND no absolute/cumulative
|
||||
# cap was configured, we truly had nothing to check -> preserve the old rc=3.
|
||||
if len(rows) < 3 and abs_max == 0 and cum_budget == 0:
|
||||
sys.stderr.write(f"COST_SPIKE_NO_DATA records={len(rows)} (need >=3, no absolute/cumulative cap set)\n")
|
||||
raise SystemExit(3)
|
||||
|
||||
print("COST_SPIKE_NONE")
|
||||
PY
|
||||
|
||||
@@ -110,3 +110,8 @@ else
|
||||
openssl dgst -sha256 -sign "$AUDIT_PRIV" -out "$HEAD_SIG" "$HEAD_FILE"
|
||||
echo "SIGN_AUDIT_HEAD_OK head=$HEAD_HASH anchor=local-file"
|
||||
fi
|
||||
|
||||
# A4/V9: also bind token/cost telemetry to a signed manifest so tampering with
|
||||
# provider-usage.jsonl / metrics.jsonl is detectable. Best-effort — never fails
|
||||
# the audit signing step (verify-telemetry-integrity.sh is the gate).
|
||||
bash "$SCRIPT_DIR/telemetry-integrity.sh" sign >/dev/null 2>&1 || true
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
# CASAN H5 — Telemetry integrity proof (Track A, V9).
|
||||
#
|
||||
# Token/cost telemetry (provider-usage.jsonl, cost/metrics.jsonl) previously sat
|
||||
# OUTSIDE the signed audit chain, so a forger could rewrite token counts to hide
|
||||
# cost abuse and nothing would detect it. This binds those files to a signed
|
||||
# manifest: any byte change flips the manifest head hash, and because the head
|
||||
# is RSA-signed with an off-repo key, a forger cannot re-sign a rewritten head.
|
||||
#
|
||||
# Usage:
|
||||
# telemetry-integrity.sh sign — hash telemetry files, write + sign manifest head
|
||||
# telemetry-integrity.sh verify — recompute, compare head, verify signature
|
||||
#
|
||||
# Key resolution (sign): CASAN_AUDIT_PRIV, else level5/central-governance/audit-private.pem
|
||||
# Key resolution (verify): CASAN_AUDIT_PUB, else level5/central-governance/audit-public.pem
|
||||
#
|
||||
# Outputs (under .specify/logs/level5/):
|
||||
# telemetry-manifest.json — {basename: sha256} for each telemetry file
|
||||
# telemetry-head.txt — sha256 over the canonical manifest text
|
||||
# telemetry-head.sig — RSA signature of telemetry-head.txt (when a key exists)
|
||||
#
|
||||
# Exit: 0 ok, 1 tamper/mismatch/invalid-signature, 64 usage.
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
|
||||
L5_DIR="$PROJECT_ROOT/.specify/logs/level5"
|
||||
COST_DIR="$PROJECT_ROOT/.specify/logs/cost"
|
||||
mkdir -p "$L5_DIR"
|
||||
|
||||
CMD="${1:-}"
|
||||
MANIFEST="$L5_DIR/telemetry-manifest.json"
|
||||
HEAD_FILE="$L5_DIR/telemetry-head.txt"
|
||||
HEAD_SIG="$L5_DIR/telemetry-head.sig"
|
||||
AUDIT_PRIV="${CASAN_AUDIT_PRIV:-$PROJECT_ROOT/.specify/level5/central-governance/audit-private.pem}"
|
||||
AUDIT_PUB="${CASAN_AUDIT_PUB:-$PROJECT_ROOT/.specify/level5/central-governance/audit-public.pem}"
|
||||
|
||||
# Telemetry files to bind. Missing files hash to the literal "MISSING" so the
|
||||
# manifest is stable and a deletion is itself a detectable change.
|
||||
TELEMETRY_FILES=(
|
||||
"$L5_DIR/provider-usage.jsonl"
|
||||
"$COST_DIR/metrics.jsonl"
|
||||
)
|
||||
|
||||
compute_head() {
|
||||
# Prints: manifest-json on line 1, head-hash on line 2.
|
||||
python - "$@" <<'PY'
|
||||
import hashlib, json, os, sys
|
||||
files = sys.argv[1:]
|
||||
manifest = {}
|
||||
for path in files:
|
||||
name = os.path.basename(path)
|
||||
if os.path.isfile(path):
|
||||
with open(path, "rb") as f:
|
||||
manifest[name] = hashlib.sha256(f.read()).hexdigest()
|
||||
else:
|
||||
manifest[name] = "MISSING"
|
||||
canonical = json.dumps(manifest, sort_keys=True, separators=(",", ":"))
|
||||
head = hashlib.sha256(canonical.encode()).hexdigest()
|
||||
print(canonical)
|
||||
print(head)
|
||||
PY
|
||||
}
|
||||
|
||||
case "$CMD" in
|
||||
sign)
|
||||
OUT="$(compute_head "${TELEMETRY_FILES[@]}")"
|
||||
CANON="$(printf '%s' "$OUT" | sed -n '1p')"
|
||||
HEAD="$(printf '%s' "$OUT" | sed -n '2p')"
|
||||
printf '%s' "$CANON" > "$MANIFEST"
|
||||
printf '%s' "$HEAD" > "$HEAD_FILE"
|
||||
if [[ -f "$AUDIT_PRIV" ]] && command -v openssl >/dev/null 2>&1; then
|
||||
openssl dgst -sha256 -sign "$AUDIT_PRIV" -out "$HEAD_SIG" "$HEAD_FILE"
|
||||
echo "TELEMETRY_INTEGRITY_SIGNED head=$HEAD anchor=signed files=${#TELEMETRY_FILES[@]}"
|
||||
else
|
||||
rm -f "$HEAD_SIG"
|
||||
echo "TELEMETRY_INTEGRITY_SIGNED head=$HEAD anchor=unsigned files=${#TELEMETRY_FILES[@]} (no private key)"
|
||||
fi
|
||||
;;
|
||||
verify)
|
||||
if [[ ! -f "$HEAD_FILE" ]]; then
|
||||
echo "TELEMETRY_INTEGRITY_MISSING no telemetry-head.txt (run: telemetry-integrity.sh sign)" >&2
|
||||
exit 1
|
||||
fi
|
||||
OUT="$(compute_head "${TELEMETRY_FILES[@]}")"
|
||||
HEAD_NOW="$(printf '%s' "$OUT" | sed -n '2p')"
|
||||
HEAD_STORED="$(cat "$HEAD_FILE")"
|
||||
if [[ "$HEAD_NOW" != "$HEAD_STORED" ]]; then
|
||||
echo "TELEMETRY_INTEGRITY_MISMATCH computed=$HEAD_NOW stored=$HEAD_STORED" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ -f "$HEAD_SIG" && -f "$AUDIT_PUB" ]] && command -v openssl >/dev/null 2>&1; then
|
||||
if ! openssl dgst -sha256 -verify "$AUDIT_PUB" -signature "$HEAD_SIG" "$HEAD_FILE" >/dev/null 2>&1; then
|
||||
echo "TELEMETRY_INTEGRITY_SIGNATURE_INVALID head=$HEAD_STORED" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "TELEMETRY_INTEGRITY_VALID anchor=signed head=$HEAD_STORED"
|
||||
else
|
||||
echo "TELEMETRY_INTEGRITY_VALID anchor=unsigned head=$HEAD_STORED"
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
echo "Usage: telemetry-integrity.sh {sign|verify}" >&2
|
||||
exit 64
|
||||
;;
|
||||
esac
|
||||
Reference in New Issue
Block a user