119 lines
4.0 KiB
Bash
Executable File
119 lines
4.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -uo pipefail
|
|
|
|
# CASAN WP-S6 — Circuit breaker check (two responsibilities):
|
|
#
|
|
# 1. NO-BYPASS SCAN: verifies none of the CASAN control scripts use
|
|
# --no-verify, bypass flags, or short-circuit patterns that would
|
|
# circumvent security/governance checks.
|
|
#
|
|
# 2. MODEL-FAILURE CIRCUIT BREAKER: reads provider-usage.jsonl and counts
|
|
# consecutive recent failures. If ≥ CIRCUIT_BREAKER_THRESHOLD consecutive
|
|
# model calls failed, prints CIRCUIT_OPEN and exits non-zero so the caller
|
|
# can stop invoking the model (prevents cascading failures / cost runaway).
|
|
#
|
|
# Usage: circuit-breaker-check.sh [--no-bypass-only | --breaker-only]
|
|
# Exit: 0 all OK, 1 bypass found or circuit open.
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
|
|
PROVIDER_LOG="$ROOT/.specify/logs/level5/provider-usage.jsonl"
|
|
CIRCUIT_BREAKER_THRESHOLD="${CIRCUIT_BREAKER_THRESHOLD:-5}"
|
|
MODE="${1:-both}"
|
|
FAIL=0
|
|
|
|
fail() { echo " FAIL $1"; FAIL=$((FAIL+1)); }
|
|
ok() { echo " PASS $1"; }
|
|
|
|
echo "=== CASAN WP-S6: no-bypass + circuit breaker ==="
|
|
|
|
# ── 1. NO-BYPASS SCAN ──────────────────────────────────────────────────────
|
|
if [[ "$MODE" != "--breaker-only" ]]; then
|
|
echo "--- no-bypass scan ---"
|
|
SCAN_DIRS=(
|
|
"$ROOT/.specify/scripts/bash"
|
|
"$ROOT/.specify/tests"
|
|
"$ROOT/scripts"
|
|
)
|
|
|
|
BYPASS_PATTERNS=(
|
|
"--no-verify"
|
|
"SKIP_GOVERNANCE"
|
|
"SKIP_SECURITY"
|
|
"SKIP_CASAN"
|
|
"bypass_gate"
|
|
"force_approve"
|
|
"# nocheck"
|
|
"# no-check"
|
|
"CASAN_SKIP"
|
|
"hardcode.*APPROVED"
|
|
"hardcode.*PASS"
|
|
)
|
|
|
|
bypass_hits=""
|
|
for dir in "${SCAN_DIRS[@]}"; do
|
|
[[ -d "$dir" ]] || continue
|
|
for pat in "${BYPASS_PATTERNS[@]}"; do
|
|
# grep non-comment lines only (skip lines starting with # or //)
|
|
hit="$(grep -rl "$pat" "$dir" 2>/dev/null | \
|
|
grep -v 'circuit-breaker-check.sh' | \
|
|
grep -v '.specify/logs/' | \
|
|
grep -v '.git/' | \
|
|
while IFS= read -r file; do
|
|
# re-check: must appear on a non-comment line
|
|
if grep -qP "^[^#/].*${pat}" "$file" 2>/dev/null; then
|
|
echo "$file"
|
|
fi
|
|
done || true)"
|
|
[[ -n "$hit" ]] && bypass_hits="$bypass_hits
|
|
pattern='$pat' in: $hit"
|
|
done
|
|
done
|
|
|
|
if [[ -z "$bypass_hits" ]]; then
|
|
ok "No bypass patterns found in control scripts"
|
|
else
|
|
fail "Bypass patterns found:$bypass_hits"
|
|
fi
|
|
fi
|
|
|
|
# ── 2. CIRCUIT BREAKER ─────────────────────────────────────────────────────
|
|
if [[ "$MODE" != "--no-bypass-only" ]]; then
|
|
echo "--- model failure circuit breaker ---"
|
|
if [[ ! -f "$PROVIDER_LOG" ]]; then
|
|
ok "Circuit breaker: no usage log yet — circuit closed (no calls to fail)"
|
|
else
|
|
# Count consecutive failures from the END of the log
|
|
consecutive_fails="$(python3 - "$PROVIDER_LOG" "$CIRCUIT_BREAKER_THRESHOLD" << 'PY'
|
|
import json, sys
|
|
|
|
log_file, threshold = sys.argv[1], int(sys.argv[2])
|
|
try:
|
|
lines = [l for l in open(log_file) if l.strip()]
|
|
consecutive = 0
|
|
for line in reversed(lines):
|
|
try:
|
|
r = json.loads(line)
|
|
if r.get("status") == "error" or r.get("status") == "fail":
|
|
consecutive += 1
|
|
else:
|
|
break # a success resets the counter
|
|
except json.JSONDecodeError:
|
|
break
|
|
print(consecutive)
|
|
except Exception as e:
|
|
print(0) # safe default: assume circuit closed
|
|
PY
|
|
)"
|
|
if [[ "$consecutive_fails" -ge "$CIRCUIT_BREAKER_THRESHOLD" ]]; then
|
|
fail "CIRCUIT_OPEN: $consecutive_fails consecutive model failures (threshold=$CIRCUIT_BREAKER_THRESHOLD) — stop calling model"
|
|
else
|
|
ok "Circuit breaker closed: consecutive_failures=$consecutive_fails (threshold=$CIRCUIT_BREAKER_THRESHOLD)"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
echo ""
|
|
echo "=== Circuit breaker: FAIL=$FAIL ==="
|
|
[[ "$FAIL" -eq 0 ]] || exit 1
|