CASAN/packages/casan-harness/scripts/bash/cost-spike-detect.sh

#!/usr/bin/env bash
set -uo pipefail

# CASAN H6 cost detector (WP-C + Track A cost controls A5 / V12,V13,V14).
# Answers the H6 key question: "if a step suddenly costs 3x the tokens, does
# anyone know?". Reads real per-step usage from provider-usage.jsonl and applies
# three complementary controls:
#   1. Relative spike  — any step > MULTIPLIER x median  (needs >=3 records).
#   2. Absolute cap    — any single step > CASAN_COST_ABSOLUTE_MAX_TOKENS. Works
#      from the very first record, so a slow-boil that drags the median up
#      (V12) and a cold-start with no history (V14) are BOTH still caught.
#   3. Cumulative budget — sum of tokens > CASAN_COST_CUMULATIVE_BUDGET_TOKENS,
#      catching a spray of many under-threshold calls (V13).
#
# Usage: cost-spike-detect.sh [provider-usage.jsonl] [multiplier]
# Env:   CASAN_COST_ABSOLUTE_MAX_TOKENS      per-call hard cap (0/unset = off)
#        CASAN_COST_CUMULATIVE_BUDGET_TOKENS run/session budget (0/unset = off)
# Exit: 0 no violation, 2 violation (spike/absolute/cumulative), 64 usage,
#       3 not enough data for the RELATIVE test and no absolute/cumulative cap set.

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/casan-paths.sh"
PROJECT_ROOT="$CASAN_APP_ROOT"
# SEC-23 (MT-03): per-tenant cost/quota. With no explicit log arg, a tenant run
# evaluates its OWN usage log so one tenant's spend never counts against another's
# budget (noisy-neighbor isolation). No tenant set -> the shared default log.
if [[ -n "${1:-}" ]]; then
  LOG="$1"
elif [[ -n "${CASAN_TENANT_ID:-}" ]]; then
  LOG="$(bash "$SCRIPT_DIR/tenant-store.sh" resolve telemetry/provider-usage.jsonl 2>/dev/null)" \
    || { echo "COST_SPIKE_TENANT_DENIED" >&2; exit 3; }
else
  LOG="$CASAN_STATE_ROOT/logs/level5/provider-usage.jsonl"
fi
MULT="${2:-3.0}"

[[ -f "$LOG" ]] || { echo "COST_SPIKE_NO_DATA file=$LOG" >&2; exit 3; }

CASAN_COST_ABSOLUTE_MAX_TOKENS="${CASAN_COST_ABSOLUTE_MAX_TOKENS:-0}" \
CASAN_COST_CUMULATIVE_BUDGET_TOKENS="${CASAN_COST_CUMULATIVE_BUDGET_TOKENS:-0}" \
python - "$LOG" "$MULT" <<'PY'
import json, os, sys, statistics
path, mult = sys.argv[1], float(sys.argv[2])
abs_max = int(os.environ.get("CASAN_COST_ABSOLUTE_MAX_TOKENS", "0") or "0")
cum_budget = int(os.environ.get("CASAN_COST_CUMULATIVE_BUDGET_TOKENS", "0") or "0")

rows = []
for line in open(path, encoding="utf-8"):
    line = line.strip()
    if not line:
        continue
    try:
        r = json.loads(line)
    except ValueError:
        continue
    if "total_tokens" in r:
        rows.append((r.get("step", "?"), int(r["total_tokens"])))

tokens = [t for _, t in rows]
total = sum(tokens)
violations = []

# 2. Absolute per-call cap — enforced from the first record (cold-start safe).
if abs_max > 0:
    over = [(s, t) for s, t in rows if t > abs_max]
    for s, t in over:
        violations.append(f"ABSOLUTE step={s} tokens={t} (> cap {abs_max})")

# 3. Cumulative budget over the whole run/session.
if cum_budget > 0 and total > cum_budget:
    violations.append(f"CUMULATIVE total={total} (> budget {cum_budget})")

# 1. Relative median spike — needs enough history.
median = threshold = None
if len(rows) >= 3:
    median = statistics.median(tokens)
    threshold = median * mult
    for s, t in rows:
        if t > threshold:
            violations.append(f"SPIKE step={s} tokens={t} (> {threshold:.0f} = median x{mult})")

med_str = f"{median}" if median is not None else "n/a(<3 records)"
print(f"records={len(rows)} total_tokens={total} median_tokens={med_str} "
      f"abs_cap={abs_max or 'off'} cum_budget={cum_budget or 'off'}")
for v in violations:
    print(v)

if violations:
    sys.stderr.write(f"COST_VIOLATION_DETECTED count={len(violations)}\n")
    raise SystemExit(2)

# No violations. If we could not run the relative test AND no absolute/cumulative
# cap was configured, we truly had nothing to check -> preserve the old rc=3.
if len(rows) < 3 and abs_max == 0 and cum_budget == 0:
    sys.stderr.write(f"COST_SPIKE_NO_DATA records={len(rows)} (need >=3, no absolute/cumulative cap set)\n")
    raise SystemExit(3)

print("COST_SPIKE_NONE")
PY