feat: plan 16 P2 batch 2 (SEC-12 drift, SEC-29 audit fail-closed, SEC-30 replay, SEC-15 low)
- SEC-12: drift-detect adds semantic invariants — negation-flip detection (a dropped "not" now FAILS despite high char-similarity) + env must-keep patterns. - SEC-29 (X-05): governance-check audit write fails CLOSED — an unwritable audit log denies the action and empties the output (no unaudited output). - SEC-30 (X-06): approval-verify records a one-time-use nonce (sha of token/sig) and rejects replays (enforced mode / when a nonce ledger is set); dev unchanged. - SEC-15 (low): typosquat distance<=2 with the levenshtein length-sentinel bug fixed (no false positives); tool-exec fails closed with no timeout backend in enforced mode; validate-tool-input now validates nested objects/arrays recursively. Verify: new SEC suites all green via gate, run-casan4 0-FAIL, adversarial 44/44, track-c 29/0, h5-approval 12/0, no regressions. Plan-16 P2 remaining: infra-gated only (SEC-14/22/23/24/25/26). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
thanhnv
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
8c06a55aed
commit
d695a598ee
@@ -0,0 +1,54 @@
|
||||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
# CASAN Plan-16 SEC-12 — drift-detect semantic invariants (beyond char-similarity).
|
||||
#
|
||||
# Char-similarity alone misses a SEMANTIC inversion: dropping a negation
|
||||
# ("must NOT deploy" -> "must deploy") keeps similarity ~0.97 but flips the meaning,
|
||||
# and previously passed. Now a candidate that removes negations present in the
|
||||
# golden, or that drops a must-keep invariant, FAILS. Proves the flip is caught and
|
||||
# identical/benign content still passes.
|
||||
#
|
||||
# Deterministic; hermetic.
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
DD="$PROJECT_ROOT/.specify/scripts/bash/drift-detect.sh"
|
||||
WORK="$(mktemp -d)"
|
||||
trap 'rm -rf "$WORK"' EXIT
|
||||
|
||||
PASS=0; FAIL=0
|
||||
pass() { echo "PASS: $1"; PASS=$((PASS + 1)); }
|
||||
fail() { echo "FAIL: $1"; FAIL=$((FAIL + 1)); }
|
||||
rc_of() { set +e; "$@" >/dev/null 2>&1; echo $?; set -e 2>/dev/null || true; }
|
||||
|
||||
echo "===== Plan-16 SEC-12: drift semantic invariants ====="
|
||||
|
||||
printf 'You MUST NOT deploy to production without explicit human approval.\n' > "$WORK/golden.txt"
|
||||
printf 'You MUST deploy to production without explicit human approval.\n' > "$WORK/flip.txt"
|
||||
cp "$WORK/golden.txt" "$WORK/same.txt"
|
||||
|
||||
# Negation flip: near-identical text (similarity ~0.97) but a "NOT" vanished.
|
||||
[[ "$(rc_of bash "$DD" "$WORK/golden.txt" "$WORK/flip.txt" "$WORK/r1.json")" -ne 0 ]] \
|
||||
&& pass "negation flip FAILS despite high similarity" \
|
||||
|| fail "negation flip passed (semantic inversion missed)"
|
||||
grep -q 'negation_dropped' "$WORK/r1.json" \
|
||||
&& pass "report records the negation_dropped reason" || fail "reason not recorded"
|
||||
|
||||
# Identical golden/candidate still passes.
|
||||
[[ "$(rc_of bash "$DD" "$WORK/golden.txt" "$WORK/same.txt" "$WORK/r2.json")" -eq 0 ]] \
|
||||
&& pass "identical content passes (no false positive)" || fail "identical content flagged"
|
||||
|
||||
# must-keep invariant missing from candidate → FAIL.
|
||||
printf 'MUST NOT deploy\n' > "$WORK/mustkeep.txt"
|
||||
[[ "$(rc_of env CASAN_DRIFT_MUSTKEEP_FILE="$WORK/mustkeep.txt" bash "$DD" "$WORK/golden.txt" "$WORK/flip.txt" "$WORK/r3.json")" -ne 0 ]] \
|
||||
&& pass "missing must-keep invariant FAILS" || fail "missing must-keep not detected"
|
||||
|
||||
# must-keep invariant present → pass.
|
||||
printf 'production\n' > "$WORK/mustkeep2.txt"
|
||||
[[ "$(rc_of env CASAN_DRIFT_MUSTKEEP_FILE="$WORK/mustkeep2.txt" bash "$DD" "$WORK/golden.txt" "$WORK/same.txt" "$WORK/r4.json")" -eq 0 ]] \
|
||||
&& pass "present must-keep invariant passes" || fail "present must-keep wrongly failed"
|
||||
|
||||
echo ""
|
||||
echo "===== SEC-12 SUMMARY: PASS=$PASS FAIL=$FAIL ====="
|
||||
[[ "$FAIL" -eq 0 ]] || exit 1
|
||||
@@ -0,0 +1,65 @@
|
||||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
# CASAN Plan-16 SEC-15 — low-cluster hardening.
|
||||
# * supply-chain typosquat: catch distance<=2 (was distance==1), no false positives
|
||||
# (levenshtein length sentinel fixed so it no longer collides with the threshold),
|
||||
# * tool-exec: fail CLOSED in enforced mode when no timeout backend exists,
|
||||
# * validate-tool-input: recurse into nested objects/arrays (was one level deep).
|
||||
#
|
||||
# Deterministic; hermetic.
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
S="$PROJECT_ROOT/.specify/scripts/bash"
|
||||
WORK="$(mktemp -d)"
|
||||
trap 'rm -rf "$WORK"' EXIT
|
||||
|
||||
PASS=0; FAIL=0
|
||||
pass() { echo "PASS: $1"; PASS=$((PASS + 1)); }
|
||||
fail() { echo "FAIL: $1"; FAIL=$((FAIL + 1)); }
|
||||
rc_of() { set +e; "$@" >/dev/null 2>&1; echo $?; set -e 2>/dev/null || true; }
|
||||
|
||||
echo "===== Plan-16 SEC-15: low-cluster hardening ====="
|
||||
|
||||
# ---- typosquat (distance<=2) ----
|
||||
printf '{"dependencies":{}}' > "$WORK/base.json"
|
||||
printf 'requests\nnumpy\nexpress\n' > "$WORK/known.txt"; : > "$WORK/deny.txt"
|
||||
scan() { python3 "$S/supply-chain-scan.py" "$1" "$WORK/base.json" "$WORK/known.txt" "$WORK/deny.txt" "$WORK/r.json" "" >/dev/null 2>&1; }
|
||||
|
||||
printf '{"dependencies":{"reqeusts":"1.0.0"}}' > "$WORK/m1.json"; scan "$WORK/m1.json"
|
||||
grep -q "typosquat_of:requests" "$WORK/r.json" \
|
||||
&& pass "2-char typosquat (reqeusts→requests) flagged" || fail "2-char typosquat missed"
|
||||
|
||||
printf '{"dependencies":{"fastapi":"1.0.0"}}' > "$WORK/m2.json"; scan "$WORK/m2.json"
|
||||
grep -q "typosquat" "$WORK/r.json" \
|
||||
&& fail "legit package fastapi false-flagged as typosquat" \
|
||||
|| pass "legit distant package not false-flagged (levenshtein sentinel fixed)"
|
||||
|
||||
# ---- tool-exec fail-closed when no timeout backend ----
|
||||
# PATH=/bin has bash but not perl/timeout (both in /usr/bin) → no-backend branch.
|
||||
[[ "$(set +e; PATH=/bin CASAN_TOOL_EXEC_STRICT=1 /bin/bash "$S/tool-exec.sh" 2 -- echo hi >/dev/null 2>&1; echo $?)" -eq 2 ]] \
|
||||
&& pass "tool-exec refuses (fail-closed) with no timeout backend in enforced mode" \
|
||||
|| fail "tool-exec did not fail closed without a timeout backend"
|
||||
[[ "$(set +e; PATH=/bin /bin/bash "$S/tool-exec.sh" 2 -- echo hi >/dev/null 2>&1; echo $?)" -eq 0 ]] \
|
||||
&& pass "tool-exec dev: runs without backend (backward compatible)" \
|
||||
|| fail "tool-exec dev mode broke"
|
||||
|
||||
# ---- validate-tool-input recursion ----
|
||||
cat > "$WORK/schema.json" <<'J'
|
||||
{"type":"object","additionalProperties":false,"properties":{
|
||||
"cfg":{"type":"object","additionalProperties":false,"properties":{"port":{"type":"integer"}}}}}
|
||||
J
|
||||
printf '{"cfg":{"port":8080}}' > "$WORK/ok.json"
|
||||
printf '{"cfg":{"port":"NOPE"}}' > "$WORK/badtype.json"
|
||||
printf '{"cfg":{"port":80,"evil":"x"}}' > "$WORK/badextra.json"
|
||||
[[ "$(rc_of bash "$S/validate-tool-input.sh" "$WORK/schema.json" "$WORK/ok.json")" -eq 0 ]] \
|
||||
&& pass "valid nested object accepted" || fail "valid nested object rejected"
|
||||
[[ "$(rc_of bash "$S/validate-tool-input.sh" "$WORK/schema.json" "$WORK/badtype.json")" -eq 2 ]] \
|
||||
&& pass "nested wrong type rejected (recursion)" || fail "nested wrong type slipped through"
|
||||
[[ "$(rc_of bash "$S/validate-tool-input.sh" "$WORK/schema.json" "$WORK/badextra.json")" -eq 2 ]] \
|
||||
&& pass "nested unexpected field rejected (recursion)" || fail "nested unexpected field slipped through"
|
||||
|
||||
echo ""
|
||||
echo "===== SEC-15 SUMMARY: PASS=$PASS FAIL=$FAIL ====="
|
||||
[[ "$FAIL" -eq 0 ]] || exit 1
|
||||
@@ -0,0 +1,55 @@
|
||||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
# CASAN Plan-16 SEC-29 (X-05) — audit write fails CLOSED.
|
||||
#
|
||||
# If the audit log cannot be written (disk full, read-only, quota), a governed
|
||||
# action must NOT proceed — there is no action without its accountability record.
|
||||
# Proves governance-check denies (and empties the output) when the audit log is
|
||||
# unwritable, and still works normally otherwise.
|
||||
#
|
||||
# Restores the audit log it perturbs on exit.
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
GC="$PROJECT_ROOT/.specify/scripts/bash/governance-check.sh"
|
||||
AUD="$PROJECT_ROOT/.specify/logs/audit/audit.jsonl"
|
||||
WORK="$(mktemp -d)"
|
||||
export CASAN_AUDIT_KEY_DIR="$WORK/keys"
|
||||
|
||||
restore() {
|
||||
[[ -f "$AUD" ]] && chmod 644 "$AUD" 2>/dev/null || true
|
||||
git -C "$PROJECT_ROOT" checkout -- .specify/logs/ .specify/level5/central-governance/ 2>/dev/null || true
|
||||
}
|
||||
trap 'restore; rm -rf "$WORK"' EXIT
|
||||
|
||||
PASS=0; FAIL=0
|
||||
pass() { echo "PASS: $1"; PASS=$((PASS + 1)); }
|
||||
fail() { echo "FAIL: $1"; FAIL=$((FAIL + 1)); }
|
||||
rc_of() { set +e; "$@" >/dev/null 2>&1; echo $?; set -e 2>/dev/null || true; }
|
||||
|
||||
echo "===== Plan-16 SEC-29: audit fail-closed when unwritable ====="
|
||||
|
||||
echo "benign objective text" > "$WORK/in.txt"
|
||||
|
||||
# Normal (writable) path works.
|
||||
[[ "$(rc_of bash "$GC" "$WORK/in.txt" "$WORK/ok-out.txt" agent_step)" -eq 0 ]] \
|
||||
&& pass "governed action succeeds when audit is writable" || fail "normal governed action failed"
|
||||
|
||||
# Make the audit log read-only so the append fails; the action must be denied.
|
||||
echo "PREEXISTING_OUTPUT" > "$WORK/blocked-out.txt"
|
||||
[[ -f "$AUD" ]] || echo '{}' > "$AUD"
|
||||
chmod 444 "$AUD"
|
||||
RC="$(rc_of bash "$GC" "$WORK/in.txt" "$WORK/blocked-out.txt" agent_step)"
|
||||
chmod 644 "$AUD"
|
||||
[[ "$RC" -ne 0 ]] \
|
||||
&& pass "unwritable audit → governed action DENIED (fail-closed)" \
|
||||
|| fail "action proceeded despite unwritable audit (fail-open)"
|
||||
|
||||
[[ ! -s "$WORK/blocked-out.txt" ]] \
|
||||
&& pass "output emptied on audit failure (no unaudited output leaks)" \
|
||||
|| fail "stale output left after audit failure"
|
||||
|
||||
echo ""
|
||||
echo "===== SEC-29 SUMMARY: PASS=$PASS FAIL=$FAIL ====="
|
||||
[[ "$FAIL" -eq 0 ]] || exit 1
|
||||
@@ -0,0 +1,51 @@
|
||||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
# CASAN Plan-16 SEC-30 (X-06) — approval replay prevention (one-time-use nonce).
|
||||
#
|
||||
# A verified approval (offline signature or IdP JWT) is valid for its whole exp
|
||||
# window, so it could be replayed to approve repeatedly. approval-verify now records
|
||||
# a per-token nonce and rejects any repeat (in enforced mode / when a nonce ledger
|
||||
# is configured). Proves first use is accepted, replay is denied, and dev mode
|
||||
# (no ledger) is unchanged.
|
||||
#
|
||||
# Self-contained: ephemeral reviewer key + committed registry.
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
S="$PROJECT_ROOT/.specify/scripts/bash"
|
||||
REG="$PROJECT_ROOT/.specify/level5/central-governance/reviewers.registry"
|
||||
WORK="$(mktemp -d)"; RV="$WORK/reviewers"; mkdir -p "$RV"
|
||||
trap 'rm -rf "$WORK"' EXIT
|
||||
|
||||
openssl genrsa -out "$WORK/sl.priv" 2048 2>/dev/null
|
||||
openssl rsa -in "$WORK/sl.priv" -pubout -out "$RV/security-lead.pub.pem" 2>/dev/null
|
||||
printf 'security.strict' > "$WORK/inp"
|
||||
bash "$S/approval-sign.sh" policy_change alice "$WORK/inp" security-lead "$WORK/sl.priv" "$WORK/a.sig" >/dev/null 2>&1
|
||||
|
||||
export CASAN_REVIEWERS_FILE="$REG" CASAN_REVIEWERS_DIR="$RV"
|
||||
|
||||
PASS=0; FAIL=0
|
||||
pass() { echo "PASS: $1"; PASS=$((PASS + 1)); }
|
||||
fail() { echo "FAIL: $1"; FAIL=$((FAIL + 1)); }
|
||||
rc_of() { set +e; "$@" >/dev/null 2>&1; echo $?; set -e 2>/dev/null || true; }
|
||||
|
||||
echo "===== Plan-16 SEC-30: approval replay prevention ====="
|
||||
|
||||
# With a nonce ledger configured, the first use is accepted, a replay is denied.
|
||||
[[ "$(rc_of env CASAN_APPROVAL_NONCE_FILE="$WORK/nonces.txt" bash "$S/approval-verify.sh" policy_change alice "$WORK/inp" security-lead "$WORK/a.sig")" -eq 0 ]] \
|
||||
&& pass "first use of a valid approval accepted" || fail "first use rejected"
|
||||
|
||||
[[ "$(rc_of env CASAN_APPROVAL_NONCE_FILE="$WORK/nonces.txt" bash "$S/approval-verify.sh" policy_change alice "$WORK/inp" security-lead "$WORK/a.sig")" -ne 0 ]] \
|
||||
&& pass "replay of the same approval DENIED (one-time-use)" || fail "replay accepted"
|
||||
|
||||
# Dev default (no ledger, no prod profile): no replay tracking (backward compatible).
|
||||
R1="$(rc_of bash "$S/approval-verify.sh" policy_change alice "$WORK/inp" security-lead "$WORK/a.sig")"
|
||||
R2="$(rc_of bash "$S/approval-verify.sh" policy_change alice "$WORK/inp" security-lead "$WORK/a.sig")"
|
||||
[[ "$R1" -eq 0 && "$R2" -eq 0 ]] \
|
||||
&& pass "dev mode: no nonce ledger → replay allowed (backward compatible)" \
|
||||
|| fail "dev mode changed (r1=$R1 r2=$R2)"
|
||||
|
||||
echo ""
|
||||
echo "===== SEC-30 SUMMARY: PASS=$PASS FAIL=$FAIL ====="
|
||||
[[ "$FAIL" -eq 0 ]] || exit 1
|
||||
@@ -96,10 +96,18 @@
|
||||
"sha256": "099c81a0a37b51137dbf328dbe2ef778226d0716b97660d083c1fcc6a4aecd19",
|
||||
"checks": 5
|
||||
},
|
||||
"phase-sec12-tests.sh": {
|
||||
"sha256": "9993eec8c2384e59389723672ed7f2eadc9f112e680ec6505b62b94f4780ccb4",
|
||||
"checks": 5
|
||||
},
|
||||
"phase-sec13-tests.sh": {
|
||||
"sha256": "64d67fa0ccf9c199848323e8fb6207f0cbc9608f6e256a0a7337b371c0451b43",
|
||||
"checks": 6
|
||||
},
|
||||
"phase-sec15-tests.sh": {
|
||||
"sha256": "32736ec6e665d11dabf230cf26e2355a1d4d2ee4050bb5ce8aa9decd9e925d1b",
|
||||
"checks": 7
|
||||
},
|
||||
"phase-sec16-tests.sh": {
|
||||
"sha256": "3d826ca4f5c8f98837cc70846b8e2f2f83d5a598c2a5907795e8b9cc9db448c2",
|
||||
"checks": 6
|
||||
@@ -128,6 +136,14 @@
|
||||
"sha256": "8f082c85b80bf899ea1911e8165e28c14f691e1ac3605953cd1b968ee295e556",
|
||||
"checks": 4
|
||||
},
|
||||
"phase-sec29-tests.sh": {
|
||||
"sha256": "f7c339e628904b9bfbbbb9ed9fd88ddd50e74131d62630c93fdba3bf6fef4c04",
|
||||
"checks": 3
|
||||
},
|
||||
"phase-sec30-tests.sh": {
|
||||
"sha256": "534fc41302c6bbf60fb81e1185ad032d85fea2225517b01653d750044731437f",
|
||||
"checks": 3
|
||||
},
|
||||
"phase-selfimprove-tests.sh": {
|
||||
"sha256": "e91db1af16e30b18def130553f27f05691ff5540eca0593ca5e07f624c0ef938",
|
||||
"checks": 7
|
||||
@@ -173,6 +189,6 @@
|
||||
"checks": 10
|
||||
}
|
||||
},
|
||||
"total_checks": 356,
|
||||
"suite_count": 43
|
||||
"total_checks": 374,
|
||||
"suite_count": 47
|
||||
}
|
||||
|
||||
@@ -1 +1 @@
|
||||
636416c4a4e4e7392bcb48b91f7d4b9ea93338c6f3feb469605215917d317e12
|
||||
1b8b459788953626b052536f8fc7a8493c9ec72361af5fc05a940c21fcab03f4
|
||||
Binary file not shown.
Reference in New Issue
Block a user