CASAN/packages/casan-harness/scripts/bash/circuit-breaker-check.sh

#!/usr/bin/env bash
set -uo pipefail

# CASAN WP-S6 — Circuit breaker check (two responsibilities):
#
# 1. NO-BYPASS SCAN: verifies none of the CASAN control scripts use
#    --no-verify, bypass flags, or short-circuit patterns that would
#    circumvent security/governance checks.
#
# 2. MODEL-FAILURE CIRCUIT BREAKER: reads provider-usage.jsonl and counts
#    consecutive recent failures. If ≥ CIRCUIT_BREAKER_THRESHOLD consecutive
#    model calls failed, prints CIRCUIT_OPEN and exits non-zero so the caller
#    can stop invoking the model (prevents cascading failures / cost runaway).
#    Also runs a SLIDING-WINDOW breaker (V15): a failure RATE ≥
#    CIRCUIT_WINDOW_FAIL_PCT over the last CIRCUIT_WINDOW records trips
#    CIRCUIT_OPEN_WINDOW — interleaving successes between failures no longer
#    evades the breaker.
#
# Usage: circuit-breaker-check.sh [--no-bypass-only | --breaker-only]
# Env:   CASAN_PROVIDER_LOG (log override), CIRCUIT_BREAKER_THRESHOLD,
#        CIRCUIT_WINDOW (default 10), CIRCUIT_WINDOW_FAIL_PCT (default 50)
# Exit: 0 all OK, 1 bypass found or circuit open.

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/casan-paths.sh"
ROOT="$CASAN_APP_ROOT"
PROVIDER_LOG="$CASAN_TELEMETRY_PROVIDER_LOG"
CIRCUIT_BREAKER_THRESHOLD="${CIRCUIT_BREAKER_THRESHOLD:-5}"
CIRCUIT_WINDOW="${CIRCUIT_WINDOW:-10}"
CIRCUIT_WINDOW_FAIL_PCT="${CIRCUIT_WINDOW_FAIL_PCT:-50}"
MODE="${1:-both}"
FAIL=0

fail() { echo "  FAIL  $1"; FAIL=$((FAIL+1)); }
ok()   { echo "  PASS  $1"; }

echo "=== CASAN WP-S6: no-bypass + circuit breaker ==="

# ── 1. NO-BYPASS SCAN ──────────────────────────────────────────────────────
if [[ "$MODE" != "--breaker-only" ]]; then
  echo "--- no-bypass scan ---"
  SCAN_DIRS=(
    "$CASAN_HARNESS_ROOT/scripts/bash"
    "$CASAN_HARNESS_ROOT/tests"
    "$ROOT/scripts"
  )

  BYPASS_PATTERNS=(
    "--no-verify"
    "SKIP_GOVERNANCE"
    "SKIP_SECURITY"
    "SKIP_CASAN"
    "bypass_gate"
    "force_approve"
    "# nocheck"
    "# no-check"
    "CASAN_SKIP"
    "hardcode.*APPROVED"
    "hardcode.*PASS"
  )

  bypass_hits=""
  for dir in "${SCAN_DIRS[@]}"; do
    [[ -d "$dir" ]] || continue
    for pat in "${BYPASS_PATTERNS[@]}"; do
      # grep non-comment lines only (skip lines starting with # or //)
      hit="$(grep -rl "$pat" "$dir" 2>/dev/null | \
        grep -v 'circuit-breaker-check.sh' | \
        grep -v '.specify/logs/' | \
        grep -v '.git/' | \
        while IFS= read -r file; do
          # re-check: must appear on a non-comment line
          if grep -qP "^[^#/].*${pat}" "$file" 2>/dev/null; then
            echo "$file"
          fi
        done || true)"
      [[ -n "$hit" ]] && bypass_hits="$bypass_hits
  pattern='$pat' in: $hit"
    done
  done

  if [[ -z "$bypass_hits" ]]; then
    ok "No bypass patterns found in control scripts"
  else
    fail "Bypass patterns found:$bypass_hits"
  fi
fi

# ── 2. CIRCUIT BREAKER ─────────────────────────────────────────────────────
if [[ "$MODE" != "--no-bypass-only" ]]; then
  echo "--- model failure circuit breaker ---"
  if [[ ! -f "$PROVIDER_LOG" ]]; then
    ok "Circuit breaker: no usage log yet — circuit closed (no calls to fail)"
  else
    # Count consecutive failures from the END of the log
    consecutive_fails="$(python - "$PROVIDER_LOG" "$CIRCUIT_BREAKER_THRESHOLD" << 'PY'
import json, sys

log_file, threshold = sys.argv[1], int(sys.argv[2])
try:
    lines = [l for l in open(log_file) if l.strip()]
    consecutive = 0
    for line in reversed(lines):
        try:
            r = json.loads(line)
            if r.get("status") == "error" or r.get("status") == "fail":
                consecutive += 1
            else:
                break  # a success resets the counter
        except json.JSONDecodeError:
            break
    print(consecutive)
except Exception as e:
    print(0)  # safe default: assume circuit closed
PY
)"
    if [[ "$consecutive_fails" -ge "$CIRCUIT_BREAKER_THRESHOLD" ]]; then
      fail "CIRCUIT_OPEN: $consecutive_fails consecutive model failures (threshold=$CIRCUIT_BREAKER_THRESHOLD) — stop calling model"
    else
      ok "Circuit breaker closed: consecutive_failures=$consecutive_fails (threshold=$CIRCUIT_BREAKER_THRESHOLD)"
    fi

    # Sliding-window failure RATE (V15): interleaved successes reset the
    # consecutive counter but do not hide a failing provider from the rate.
    window_stats="$(python - "$PROVIDER_LOG" "$CIRCUIT_WINDOW" << 'PY'
import json, sys

log_file, window = sys.argv[1], int(sys.argv[2])
try:
    lines = [l for l in open(log_file) if l.strip()]
    recent = lines[-window:]
    fails = 0
    total = 0
    for line in recent:
        try:
            r = json.loads(line)
        except json.JSONDecodeError:
            continue
        total += 1
        if r.get("status") in ("error", "fail", "failed"):
            fails += 1
    print(f"{fails} {total}")
except Exception:
    print("0 0")
PY
)"
    window_fails="${window_stats%% *}"
    window_total="${window_stats##* }"
    if [[ "$window_total" -ge "$CIRCUIT_WINDOW" ]]; then
      window_pct=$(( window_fails * 100 / window_total ))
      if [[ "$window_pct" -ge "$CIRCUIT_WINDOW_FAIL_PCT" ]]; then
        fail "CIRCUIT_OPEN_WINDOW: failure rate ${window_pct}% over last $window_total calls (threshold=${CIRCUIT_WINDOW_FAIL_PCT}%) — interleaved successes do not close the circuit"
      else
        ok "Window breaker closed: failure rate ${window_pct}% over last $window_total calls (threshold=${CIRCUIT_WINDOW_FAIL_PCT}%)"
      fi
    else
      ok "Window breaker closed: only $window_total records (< window=$CIRCUIT_WINDOW), rate not evaluated"
    fi
  fi
fi

echo ""
echo "=== Circuit breaker: FAIL=$FAIL ==="
[[ "$FAIL" -eq 0 ]] || exit 1