feat: plan 16 P1 (SEC-08/09/19/20/21) — fail-open/DoS/authz hardening
- SEC-08 pii-mask: fail-closed on missing rules / broken regex (no unmasked leak) - SEC-09: input-size cap + fail-closed reads (security-check/drift-detect/context-compress); non-UTF8 no longer crashes - SEC-19: control-plane store POSIX flock + atomic tmp+rename write - SEC-20: new toolchain-verify.sh (missing/PATH-shadowed/in-workspace binary -> refuse); wired into harness-preflight - SEC-21: model-call timeout 180->60s configurable + per-run call budget - SEC-11 realized by SEC-17 prod profile (no code) - 5 fail-able test suites wired into ci-harness-gate.sh; test-integrity manifest regenerated Verify: SEC+integrity gate 16/0, run-casan4 0-FAIL, adversarial 44/44, no regressions. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
thanhnv
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
8c3c5e8bff
commit
e70f0815ab
@@ -0,0 +1,67 @@
|
||||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
# CASAN Plan-16 SEC-08 (M-06) — pii-mask fails CLOSED.
|
||||
#
|
||||
# Previously a missing rules file, an unreadable file, or a broken rule regex made
|
||||
# pii-mask emit the RAW stdin — silently leaking the very PII a mask rule was meant
|
||||
# to hide. Now any such condition emits NOTHING and exits non-zero. Normal masking
|
||||
# with valid rules is unchanged.
|
||||
#
|
||||
# Deterministic; hermetic; no model/network.
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
PM="$PROJECT_ROOT/.specify/scripts/bash/pii-mask.py"
|
||||
RULES="$PROJECT_ROOT/.specify/security/pii-rules.yaml"
|
||||
WORK="$(mktemp -d)"
|
||||
trap 'rm -rf "$WORK"' EXIT
|
||||
|
||||
PASS=0; FAIL=0
|
||||
pass() { echo "PASS: $1"; PASS=$((PASS + 1)); }
|
||||
fail() { echo "FAIL: $1"; FAIL=$((FAIL + 1)); }
|
||||
|
||||
echo "===== Plan-16 SEC-08: pii-mask fail-closed ====="
|
||||
|
||||
# 1) Valid rules still mask (regression).
|
||||
OUT="$(printf 'contact bob@example.com now' | python3 "$PM" "$RULES" 2>/dev/null)"
|
||||
if [[ "$OUT" == *"MASKED"* && "$OUT" != *"bob@example.com"* ]]; then
|
||||
pass "valid rules mask the email (no leak)"
|
||||
else
|
||||
fail "valid masking broken (out='$OUT')"
|
||||
fi
|
||||
|
||||
# 2) Missing rules file → no output, non-zero, and NO raw email leaks.
|
||||
set +e
|
||||
OUT="$(printf 'secret bob@example.com' | python3 "$PM" "$WORK/nope.yaml" 2>/dev/null)"; RC=$?
|
||||
set -e 2>/dev/null || true
|
||||
if [[ "$RC" -ne 0 && -z "$OUT" ]]; then
|
||||
pass "missing rules file → empty output, non-zero (fail-closed)"
|
||||
else
|
||||
fail "missing rules leaked (rc=$RC out='$OUT')"
|
||||
fi
|
||||
|
||||
# 3) Broken regex → no output, non-zero (that PII type would otherwise leak).
|
||||
printf -- '- id: email\n type: "email"\n regex: "([unterminated"\n action: mask\n' > "$WORK/bad.yaml"
|
||||
set +e
|
||||
OUT="$(printf 'secret bob@example.com' | python3 "$PM" "$WORK/bad.yaml" 2>/dev/null)"; RC=$?
|
||||
set -e 2>/dev/null || true
|
||||
if [[ "$RC" -ne 0 && -z "$OUT" ]]; then
|
||||
pass "broken rule regex → empty output, non-zero (fail-closed)"
|
||||
else
|
||||
fail "broken regex leaked (rc=$RC out='$OUT')"
|
||||
fi
|
||||
|
||||
# 4) No rules argument at all → fail-closed.
|
||||
set +e
|
||||
OUT="$(printf 'secret bob@example.com' | python3 "$PM" 2>/dev/null)"; RC=$?
|
||||
set -e 2>/dev/null || true
|
||||
if [[ "$RC" -ne 0 && -z "$OUT" ]]; then
|
||||
pass "no rules argument → empty output, non-zero (fail-closed)"
|
||||
else
|
||||
fail "no-arg leaked (rc=$RC out='$OUT')"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "===== SEC-08 SUMMARY: PASS=$PASS FAIL=$FAIL ====="
|
||||
[[ "$FAIL" -eq 0 ]] || exit 1
|
||||
@@ -0,0 +1,59 @@
|
||||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
# CASAN Plan-16 SEC-09 (M-10) — input-size cap + fail-closed reads.
|
||||
#
|
||||
# security-check / drift-detect / context-compress ran superlinear passes (regex,
|
||||
# SequenceMatcher) with no size cap (DoS) and crashed on missing / non-UTF8 input
|
||||
# instead of returning a fail-closed verdict. This proves oversize input is blocked,
|
||||
# undecodable input degrades without a crash, and normal input is unaffected.
|
||||
# Cap overridable via CASAN_MAX_INPUT_BYTES.
|
||||
#
|
||||
# Deterministic; hermetic; no model/network.
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
BASH_DIR="$PROJECT_ROOT/.specify/scripts/bash"
|
||||
WORK="$(mktemp -d)"
|
||||
# security-check / drift-detect touch shared logs; restore on exit.
|
||||
trap 'git -C "$PROJECT_ROOT" checkout -- .specify/logs/ .specify/level5/central-governance/ 2>/dev/null; rm -rf "$WORK"' EXIT
|
||||
|
||||
PASS=0; FAIL=0
|
||||
pass() { echo "PASS: $1"; PASS=$((PASS + 1)); }
|
||||
fail() { echo "FAIL: $1"; FAIL=$((FAIL + 1)); }
|
||||
rc_of() { set +e; "$@" >/dev/null 2>&1; echo $?; set -e 2>/dev/null || true; }
|
||||
|
||||
echo "===== Plan-16 SEC-09: input caps + fail-closed reads ====="
|
||||
|
||||
BIG="$WORK/big.txt"; head -c 200 /dev/zero | tr '\0' 'a' > "$BIG"
|
||||
SMALL="$WORK/small.txt"; echo "benign objective text" > "$SMALL"
|
||||
|
||||
# --- security-check ---
|
||||
[[ "$(rc_of env CASAN_MAX_INPUT_BYTES=10 bash "$BASH_DIR/security-check.sh" "$BIG" "$WORK/o1.txt" input)" -ne 0 ]] \
|
||||
&& pass "security-check blocks oversize input" || fail "security-check allowed oversize input"
|
||||
[[ "$(rc_of bash "$BASH_DIR/security-check.sh" "$SMALL" "$WORK/o2.txt" input)" -eq 0 ]] \
|
||||
&& pass "security-check allows normal input (regression)" || fail "security-check rejected normal input"
|
||||
|
||||
# --- drift-detect ---
|
||||
echo "golden reference" > "$WORK/golden.txt"
|
||||
[[ "$(rc_of bash "$BASH_DIR/drift-detect.sh" "$WORK/golden.txt" "$WORK/missing.txt" "$WORK/dr1.json")" -ne 0 ]] \
|
||||
&& pass "drift-detect blocks on missing candidate (fail-closed, no crash)" || fail "drift-detect did not block missing file"
|
||||
if [[ -f "$WORK/dr1.json" ]] && grep -q '"status": "fail"' "$WORK/dr1.json"; then
|
||||
pass "drift-detect writes a fail verdict report (not a traceback)"
|
||||
else
|
||||
fail "drift-detect did not emit a fail verdict report"
|
||||
fi
|
||||
[[ "$(rc_of env CASAN_MAX_INPUT_BYTES=10 bash "$BASH_DIR/drift-detect.sh" "$WORK/golden.txt" "$BIG" "$WORK/dr2.json")" -ne 0 ]] \
|
||||
&& pass "drift-detect blocks oversize candidate (DoS cap)" || fail "drift-detect allowed oversize"
|
||||
|
||||
# --- context-compress ---
|
||||
[[ "$(set +e; head -c 200 /dev/zero | tr '\0' 'a' | env CASAN_MAX_INPUT_BYTES=10 python3 "$BASH_DIR/context-compress.py" >/dev/null 2>&1; echo $?)" -ne 0 ]] \
|
||||
&& pass "context-compress rejects oversize input (fail-closed)" || fail "context-compress allowed oversize input"
|
||||
|
||||
RC="$(set +e; printf '\xff\xfe some error line\n' | python3 "$BASH_DIR/context-compress.py" --mode extractive >/dev/null 2>&1; echo $?)"
|
||||
[[ "$RC" -ne 2 ]] \
|
||||
&& pass "context-compress handles non-UTF8 without crashing (rc=$RC)" || fail "context-compress crashed on non-UTF8 (rc=$RC)"
|
||||
|
||||
echo ""
|
||||
echo "===== SEC-09 SUMMARY: PASS=$PASS FAIL=$FAIL ====="
|
||||
[[ "$FAIL" -eq 0 ]] || exit 1
|
||||
@@ -0,0 +1,58 @@
|
||||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
# CASAN Plan-16 SEC-19 (ARCH-05) — atomic + locked control-plane writes.
|
||||
#
|
||||
# The control-plane store did an unlocked, non-atomic load→modify→save, so two
|
||||
# concurrent `set`/`rollback` runs could lose a write or fork the audit hash chain.
|
||||
# Now the critical section holds a POSIX flock and the store is written atomically
|
||||
# (tmp + rename). Proves concurrent writes on distinct keys all survive and the
|
||||
# chain still verifies, with no torn/partial store file.
|
||||
#
|
||||
# Deterministic; hermetic (temp store + temp keys).
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
CP="$PROJECT_ROOT/.specify/scripts/bash/control-plane-settings.py"
|
||||
WORK="$(mktemp -d)"
|
||||
trap 'rm -rf "$WORK"' EXIT
|
||||
|
||||
export CASAN_CP_STORE_FILE="$WORK/store.json"
|
||||
export CASAN_CP_KEY_DIR="$WORK/keys"
|
||||
export CASAN_CP_PUB="$WORK/pub.pem"
|
||||
|
||||
PASS=0; FAIL=0
|
||||
pass() { echo "PASS: $1"; PASS=$((PASS + 1)); }
|
||||
fail() { echo "FAIL: $1"; FAIL=$((FAIL + 1)); }
|
||||
|
||||
echo "===== Plan-16 SEC-19: atomic + locked control-plane writes ====="
|
||||
|
||||
# Fire four concurrent writers on distinct non-sensitive keys.
|
||||
python3 "$CP" set compression.enabled true --actor a --reason r >/dev/null 2>&1 &
|
||||
python3 "$CP" set compression.mode structural --actor a --reason r >/dev/null 2>&1 &
|
||||
python3 "$CP" set cost.absolute_cap_usd 5 --actor a --reason r >/dev/null 2>&1 &
|
||||
python3 "$CP" set model.primary "ollama:ornith" --actor a --reason r >/dev/null 2>&1 &
|
||||
wait
|
||||
|
||||
COUNT="$(python3 "$CP" get-all | python3 -c 'import json,sys; print(len(json.load(sys.stdin)))' 2>/dev/null)"
|
||||
[[ "$COUNT" == "4" ]] \
|
||||
&& pass "all 4 concurrent writes survived (no lost update)" \
|
||||
|| fail "lost update under concurrency (only $COUNT/4 keys present)"
|
||||
|
||||
python3 "$CP" verify-audit >/dev/null 2>&1 \
|
||||
&& pass "audit chain intact after concurrent writes" \
|
||||
|| fail "audit chain forked/broken under concurrency"
|
||||
|
||||
# The store must always be valid JSON (atomic rename ⇒ never torn) and no tmp left.
|
||||
python3 -c "import json; json.load(open('$CASAN_CP_STORE_FILE'))" 2>/dev/null \
|
||||
&& pass "store file is valid JSON (atomic write, not torn)" \
|
||||
|| fail "store file torn / invalid JSON"
|
||||
if ls "$WORK"/*.tmp >/dev/null 2>&1; then
|
||||
fail "leftover .tmp file (atomic rename incomplete)"
|
||||
else
|
||||
pass "no leftover .tmp file"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "===== SEC-19 SUMMARY: PASS=$PASS FAIL=$FAIL ====="
|
||||
[[ "$FAIL" -eq 0 ]] || exit 1
|
||||
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
# CASAN Plan-16 SEC-20 (ARCH-04 / ARCH-09) — toolchain verification, fail-closed.
|
||||
#
|
||||
# Gate verdicts depend on external binaries. A missing tool (silent no-op) or a
|
||||
# PATH-shadowed fake (attacker-controlled verdict) must refuse, not run. Proves:
|
||||
# * present toolchain passes,
|
||||
# * a missing required tool fails,
|
||||
# * a required tool planted INSIDE the workspace is rejected (shadow),
|
||||
# * with a trusted-dir allowlist, a tool outside it is rejected,
|
||||
# * harness-preflight fails closed when the toolchain check fails.
|
||||
#
|
||||
# Deterministic; hermetic; no model/network.
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
BASH_DIR="$PROJECT_ROOT/.specify/scripts/bash"
|
||||
TV="$BASH_DIR/toolchain-verify.sh"
|
||||
WORK="$(mktemp -d)"
|
||||
trap 'rm -rf "$WORK"; rm -rf "$PROJECT_ROOT/.specify/_sec20probe"' EXIT
|
||||
|
||||
PASS=0; FAIL=0
|
||||
pass() { echo "PASS: $1"; PASS=$((PASS + 1)); }
|
||||
fail() { echo "FAIL: $1"; FAIL=$((FAIL + 1)); }
|
||||
rc_of() { set +e; "$@" >/dev/null 2>&1; echo $?; set -e 2>/dev/null || true; }
|
||||
|
||||
echo "===== Plan-16 SEC-20: toolchain verification fail-closed ====="
|
||||
|
||||
[[ "$(rc_of bash "$TV")" -eq 0 ]] \
|
||||
&& pass "present toolchain passes" || fail "present toolchain rejected"
|
||||
|
||||
[[ "$(rc_of bash "$TV" definitely_missing_tool_xyz)" -ne 0 ]] \
|
||||
&& pass "missing required tool → refuse (ARCH-09)" || fail "missing tool not refused"
|
||||
|
||||
# Plant a fake required tool INSIDE the workspace, put it first on PATH.
|
||||
PROBE="$PROJECT_ROOT/.specify/_sec20probe"; mkdir -p "$PROBE"
|
||||
printf '#!/bin/sh\necho fake\n' > "$PROBE/awk"; chmod +x "$PROBE/awk"
|
||||
RC="$(set +e; PATH="$PROBE:$PATH" bash "$TV" awk >/dev/null 2>&1; echo $?)"
|
||||
[[ "$RC" -ne 0 ]] && pass "in-workspace planted binary → refuse (shadow, ARCH-04)" \
|
||||
|| fail "workspace-shadowed binary accepted"
|
||||
|
||||
# Allowlist mode: a fake tool in a temp dir outside the trusted dirs is rejected.
|
||||
printf '#!/bin/sh\necho fake\n' > "$WORK/grep"; chmod +x "$WORK/grep"
|
||||
RC="$(set +e; PATH="$WORK:$PATH" CASAN_TOOLCHAIN_TRUSTED_DIRS="/usr/bin:/bin:/usr/local/bin:/opt/homebrew/bin" bash "$TV" grep >/dev/null 2>&1; echo $?)"
|
||||
[[ "$RC" -ne 0 ]] && pass "untrusted-dir binary rejected under allowlist" \
|
||||
|| fail "untrusted-dir binary accepted under allowlist"
|
||||
|
||||
# harness-preflight fails closed when toolchain-verify fails (planted required tool).
|
||||
printf '#!/bin/sh\necho fake\n' > "$PROBE/openssl"; chmod +x "$PROBE/openssl"
|
||||
RC="$(set +e; PATH="$PROBE:$PATH" bash "$BASH_DIR/harness-preflight.sh" "$WORK/none" --model local >/dev/null 2>&1; echo $?)"
|
||||
[[ "$RC" -ne 0 ]] && pass "harness-preflight blocks on shadowed toolchain" \
|
||||
|| fail "preflight did not block on shadowed toolchain"
|
||||
|
||||
echo ""
|
||||
echo "===== SEC-20 SUMMARY: PASS=$PASS FAIL=$FAIL ====="
|
||||
[[ "$FAIL" -eq 0 ]] || exit 1
|
||||
@@ -0,0 +1,56 @@
|
||||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
# CASAN Plan-16 SEC-21 (ARCH-07) — bounded model timeout + per-run call budget.
|
||||
#
|
||||
# A 180s-per-call timeout across many steps let a hung model stall a run for tens
|
||||
# of minutes. Now the per-call timeout is lower + configurable, and total model
|
||||
# calls per run are capped so a wedged model cannot amplify into a DoS. Proves the
|
||||
# budget refuses once exhausted, and no cap set means no limit (dev default).
|
||||
#
|
||||
# Deterministic (budget is checked BEFORE any backend call, so it holds whether or
|
||||
# not a model is reachable). No network required.
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
MC="$PROJECT_ROOT/.specify/scripts/bash/model-call.py"
|
||||
WORK="$(mktemp -d)"
|
||||
trap 'git -C "$PROJECT_ROOT" checkout -- .specify/logs/level5/provider-usage.jsonl 2>/dev/null; rm -rf "$WORK"' EXIT
|
||||
|
||||
PASS=0; FAIL=0
|
||||
pass() { echo "PASS: $1"; PASS=$((PASS + 1)); }
|
||||
fail() { echo "FAIL: $1"; FAIL=$((FAIL + 1)); }
|
||||
|
||||
echo "===== Plan-16 SEC-21: model timeout + per-run call budget ====="
|
||||
|
||||
# Per-call timeout is reduced from 180s and configurable.
|
||||
grep -q 'timeout=180' "$MC" && fail "hardcoded 180s timeout still present" \
|
||||
|| pass "no hardcoded 180s per-call timeout"
|
||||
grep -q 'CASAN_MODEL_TIMEOUT_SEC' "$MC" \
|
||||
&& pass "per-call timeout is configurable via CASAN_MODEL_TIMEOUT_SEC" \
|
||||
|| fail "timeout not configurable"
|
||||
|
||||
# Budget: cap=1. Call 1 charges the budget (proceeds); call 2 is refused BEFORE any
|
||||
# backend work, with a clear budget error, regardless of model availability.
|
||||
echo "hello" > "$WORK/p.txt"
|
||||
CNT="$WORK/counter"
|
||||
export CASAN_MODEL_MAX_CALLS=1 CASAN_MODEL_CALL_COUNTER_FILE="$CNT"
|
||||
|
||||
python3 "$MC" "$WORK/p.txt" "$WORK/o1.json" --role classify >/dev/null 2>&1 # charges to 1
|
||||
ERR2="$(python3 "$MC" "$WORK/p.txt" "$WORK/o2.json" --role classify 2>&1 >/dev/null)"; RC2=$?
|
||||
if [[ "$RC2" -ne 0 ]] && echo "$ERR2" | grep -q "run_call_budget_exceeded"; then
|
||||
pass "call over budget is REFUSED (rc=$RC2, budget error)"
|
||||
else
|
||||
fail "over-budget call not refused (rc=$RC2 err='$ERR2')"
|
||||
fi
|
||||
|
||||
# No cap set → no budget error (dev default unchanged).
|
||||
unset CASAN_MODEL_MAX_CALLS CASAN_MODEL_CALL_COUNTER_FILE
|
||||
ERR3="$(python3 "$MC" "$WORK/p.txt" "$WORK/o3.json" --role classify 2>&1 >/dev/null || true)"
|
||||
echo "$ERR3" | grep -q "run_call_budget_exceeded" \
|
||||
&& fail "budget error fired with no cap set" \
|
||||
|| pass "no cap set → no budget limit (dev default)"
|
||||
|
||||
echo ""
|
||||
echo "===== SEC-21 SUMMARY: PASS=$PASS FAIL=$FAIL ====="
|
||||
[[ "$FAIL" -eq 0 ]] || exit 1
|
||||
@@ -80,6 +80,14 @@
|
||||
"sha256": "71e04cba738f501e79dd6b2c02aa86c7434c5ce541af5c7e512a2c94219f1c33",
|
||||
"checks": 5
|
||||
},
|
||||
"phase-sec08-tests.sh": {
|
||||
"sha256": "ae4df77eed43cf235198c2489cc89a9fa083a66133137b3eb61b42fd021fd3c3",
|
||||
"checks": 4
|
||||
},
|
||||
"phase-sec09-tests.sh": {
|
||||
"sha256": "7d98d0a5a646d1500e4701710ceb4009eea22deb266c3b27b5f7b2850efc2f4a",
|
||||
"checks": 7
|
||||
},
|
||||
"phase-sec16-tests.sh": {
|
||||
"sha256": "3d826ca4f5c8f98837cc70846b8e2f2f83d5a598c2a5907795e8b9cc9db448c2",
|
||||
"checks": 6
|
||||
@@ -88,6 +96,18 @@
|
||||
"sha256": "d68b93ebc0d16ee2369a5525fe206a9133c95ef0f931f9bfa799d21b571dbf4c",
|
||||
"checks": 6
|
||||
},
|
||||
"phase-sec19-tests.sh": {
|
||||
"sha256": "86d5fc377775a71920922654f96aed58952f522f4b12d1e9a290e7fa30cd6e30",
|
||||
"checks": 4
|
||||
},
|
||||
"phase-sec20-tests.sh": {
|
||||
"sha256": "86d7e8365f88631341561615923bb834823ff4548af0711ebca28353d8346e91",
|
||||
"checks": 5
|
||||
},
|
||||
"phase-sec21-tests.sh": {
|
||||
"sha256": "303503fb3670be7d1b3c2737451eff64e7f140e940d2ae16188f686af0992dfa",
|
||||
"checks": 4
|
||||
},
|
||||
"phase-selfimprove-tests.sh": {
|
||||
"sha256": "e91db1af16e30b18def130553f27f05691ff5540eca0593ca5e07f624c0ef938",
|
||||
"checks": 7
|
||||
@@ -133,6 +153,6 @@
|
||||
"checks": 10
|
||||
}
|
||||
},
|
||||
"total_checks": 307,
|
||||
"suite_count": 33
|
||||
"total_checks": 331,
|
||||
"suite_count": 38
|
||||
}
|
||||
|
||||
@@ -1 +1 @@
|
||||
06273ff0acaaaabfa2b5ca8d6f06d722e1423d995bcd8ba772251c1282f4dd41
|
||||
30124645ed601ca08017707be27592b23c98c35dff08cb419362d585cffa0daf
|
||||
Binary file not shown.
Reference in New Issue
Block a user