feat: plan 16-01
This commit is contained in:
thanhnv
@@ -0,0 +1,150 @@
|
||||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
# CASAN Plan-16 SEC-01 (H-01) — "unsigned = FAIL" in enforced mode.
|
||||
#
|
||||
# Proves that verify-audit-chain / verify-tool-audit / telemetry-integrity /
|
||||
# evidence-pack all treat a missing (or unverifiable) signature as a FAILURE when
|
||||
# strict enforcement is on, while keeping the permissive dev default unchanged.
|
||||
# Enforcement is triggered by either CASAN_VERIFY_STRICT=1 or CASAN_PROFILE=prod.
|
||||
#
|
||||
# Deterministic; no model/app/network. Hermetic: builds throwaway logs in a temp
|
||||
# workspace; the telemetry case backs up + restores the real head artifacts.
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
BASH_DIR="$PROJECT_ROOT/.specify/scripts/bash"
|
||||
WORK="$(mktemp -d)"
|
||||
|
||||
# Back up telemetry head artifacts (fixed-path; the sign step mutates them).
|
||||
L5_DIR="$PROJECT_ROOT/.specify/logs/level5"
|
||||
TEL_BAK="$WORK/tel-bak"; mkdir -p "$TEL_BAK"
|
||||
for f in telemetry-manifest.json telemetry-head.txt telemetry-head.sig; do
|
||||
[[ -f "$L5_DIR/$f" ]] && cp -p "$L5_DIR/$f" "$TEL_BAK/$f"
|
||||
done
|
||||
restore_tel() {
|
||||
for f in telemetry-manifest.json telemetry-head.txt telemetry-head.sig; do
|
||||
if [[ -f "$TEL_BAK/$f" ]]; then cp -p "$TEL_BAK/$f" "$L5_DIR/$f"; else rm -f "$L5_DIR/$f"; fi
|
||||
done
|
||||
}
|
||||
trap 'restore_tel; rm -rf "$WORK"' EXIT
|
||||
|
||||
PASS=0; FAIL=0
|
||||
pass() { echo "PASS: $1"; PASS=$((PASS + 1)); }
|
||||
fail() { echo "FAIL: $1"; FAIL=$((FAIL + 1)); }
|
||||
|
||||
# rc_of <env-assignments-or-empty> <cmd...> : run, print exit code, never abort.
|
||||
rc_of() { set +e; "$@" >/dev/null 2>&1; echo $?; set -e 2>/dev/null || true; }
|
||||
|
||||
echo "===== Plan-16 SEC-01: unsigned = FAIL in enforced mode ====="
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 1) verify-audit-chain.sh — build a genuine unsigned chain (no head/sig files)
|
||||
# ---------------------------------------------------------------------------
|
||||
AUD_DIR="$WORK/audit"; mkdir -p "$AUD_DIR"
|
||||
python3 - "$AUD_DIR/audit.jsonl" <<'PY'
|
||||
import hashlib, json, sys
|
||||
path = sys.argv[1]
|
||||
prev = ""
|
||||
core = "|".join(["2026-07-06T00:00:00Z", "t1", "act", "a@x", "low", "ALLOW", "n/a", "", "ih", "oh", prev])
|
||||
h = hashlib.sha256(core.encode()).hexdigest()
|
||||
rec = {"timestamp":"2026-07-06T00:00:00Z","trace_id":"t1","action":"act","actor":"a@x",
|
||||
"risk_level":"low","decision":"ALLOW","approval_status":"n/a","approver":"",
|
||||
"input_hash":"ih","output_hash":"oh","previous_record_hash":prev,"record_hash":h}
|
||||
open(path,"w").write(json.dumps(rec)+"\n")
|
||||
PY
|
||||
|
||||
RC="$(rc_of bash "$BASH_DIR/verify-audit-chain.sh" "$AUD_DIR/audit.jsonl")"
|
||||
[[ "$RC" -eq 0 ]] && pass "audit-chain: unsigned OK in permissive mode (rc=0)" \
|
||||
|| fail "audit-chain: permissive mode should pass unsigned (rc=$RC)"
|
||||
|
||||
RC="$(rc_of env CASAN_VERIFY_STRICT=1 bash "$BASH_DIR/verify-audit-chain.sh" "$AUD_DIR/audit.jsonl")"
|
||||
[[ "$RC" -ne 0 ]] && pass "audit-chain: unsigned FAILS with CASAN_VERIFY_STRICT=1 (rc=$RC)" \
|
||||
|| fail "audit-chain: strict flag did not fail unsigned (rc=$RC)"
|
||||
|
||||
RC="$(rc_of env CASAN_PROFILE=prod bash "$BASH_DIR/verify-audit-chain.sh" "$AUD_DIR/audit.jsonl")"
|
||||
[[ "$RC" -ne 0 ]] && pass "audit-chain: unsigned FAILS with CASAN_PROFILE=prod (rc=$RC)" \
|
||||
|| fail "audit-chain: prod profile did not fail unsigned (rc=$RC)"
|
||||
|
||||
# Attack: tamper a field, recompute record_hash so the chain is internally valid,
|
||||
# leave no signature. Strict must STILL fail (recompute does not save the forger).
|
||||
python3 - "$AUD_DIR/audit.jsonl" <<'PY'
|
||||
import hashlib, json, sys
|
||||
path = sys.argv[1]
|
||||
rec = json.loads(open(path).read().strip())
|
||||
rec["decision"] = "DENY_BYPASSED" # forge the verdict
|
||||
prev = ""
|
||||
core = "|".join([rec["timestamp"],rec["trace_id"],rec["action"],rec["actor"],rec["risk_level"],
|
||||
rec["decision"],rec["approval_status"],rec["approver"],rec["input_hash"],
|
||||
rec["output_hash"],prev])
|
||||
rec["record_hash"] = hashlib.sha256(core.encode()).hexdigest() # recompute → chain valid
|
||||
open(path,"w").write(json.dumps(rec)+"\n")
|
||||
PY
|
||||
RC="$(rc_of env CASAN_VERIFY_STRICT=1 bash "$BASH_DIR/verify-audit-chain.sh" "$AUD_DIR/audit.jsonl")"
|
||||
[[ "$RC" -ne 0 ]] && pass "audit-chain: tamper+recompute+no-sig still FAILS in strict (rc=$RC)" \
|
||||
|| fail "audit-chain: recomputed forged chain passed strict (rc=$RC)"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 2) verify-tool-audit.sh — genuine unsigned tool-calls chain
|
||||
# ---------------------------------------------------------------------------
|
||||
TOOL_DIR="$WORK/tool"; mkdir -p "$TOOL_DIR"
|
||||
python3 - "$TOOL_DIR/tool-calls.jsonl" <<'PY'
|
||||
import hashlib, json, sys
|
||||
path = sys.argv[1]
|
||||
prev = ""
|
||||
rec = {"timestamp":"2026-07-06T00:00:00Z","tool":"bash","actor":"a@x","previous_record_hash":prev}
|
||||
core = json.dumps(rec, sort_keys=True, separators=(",", ":"))
|
||||
rec["record_hash"] = hashlib.sha256((prev + "|" + core).encode()).hexdigest()
|
||||
open(path,"w").write(json.dumps(rec)+"\n")
|
||||
PY
|
||||
|
||||
RC="$(rc_of bash "$BASH_DIR/verify-tool-audit.sh" "$TOOL_DIR/tool-calls.jsonl")"
|
||||
[[ "$RC" -eq 0 ]] && pass "tool-audit: unsigned OK in permissive mode (rc=0)" \
|
||||
|| fail "tool-audit: permissive mode should pass unsigned (rc=$RC)"
|
||||
|
||||
RC="$(rc_of env CASAN_VERIFY_STRICT=1 bash "$BASH_DIR/verify-tool-audit.sh" "$TOOL_DIR/tool-calls.jsonl")"
|
||||
[[ "$RC" -ne 0 ]] && pass "tool-audit: unsigned FAILS with CASAN_VERIFY_STRICT=1 (rc=$RC)" \
|
||||
|| fail "tool-audit: strict flag did not fail unsigned (rc=$RC)"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 3) telemetry-integrity.sh — sign (unsigned, no key), then strict verify fails
|
||||
# ---------------------------------------------------------------------------
|
||||
# Force the keyless path so the head is written WITHOUT a signature.
|
||||
env CASAN_AUDIT_PRIV="$WORK/nonexistent.pem" bash "$BASH_DIR/telemetry-integrity.sh" sign >/dev/null 2>&1
|
||||
RC="$(rc_of env CASAN_AUDIT_PUB="$WORK/nonexistent.pem" bash "$BASH_DIR/telemetry-integrity.sh" verify)"
|
||||
[[ "$RC" -eq 0 ]] && pass "telemetry: unsigned OK in permissive mode (rc=0)" \
|
||||
|| fail "telemetry: permissive mode should pass unsigned (rc=$RC)"
|
||||
RC="$(rc_of env CASAN_VERIFY_STRICT=1 CASAN_AUDIT_PUB="$WORK/nonexistent.pem" bash "$BASH_DIR/telemetry-integrity.sh" verify)"
|
||||
[[ "$RC" -ne 0 ]] && pass "telemetry: unsigned FAILS with CASAN_VERIFY_STRICT=1 (rc=$RC)" \
|
||||
|| fail "telemetry: strict flag did not fail unsigned (rc=$RC)"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 4) evidence-pack.sh verify-pack — build a valid unsigned pack (no .sig)
|
||||
# ---------------------------------------------------------------------------
|
||||
PACK_DIR="$WORK/pack"; mkdir -p "$PACK_DIR"
|
||||
echo "hello evidence" > "$PACK_DIR/report.txt"
|
||||
python3 - "$PACK_DIR" <<'PY'
|
||||
import hashlib, json, os, sys
|
||||
d = sys.argv[1]
|
||||
files = {}
|
||||
for fn in os.listdir(d):
|
||||
p = os.path.join(d, fn)
|
||||
if os.path.isfile(p):
|
||||
files[fn] = hashlib.sha256(open(p,"rb").read()).hexdigest()
|
||||
canonical = json.dumps(files, sort_keys=True, separators=(",", ":"))
|
||||
head = hashlib.sha256(canonical.encode()).hexdigest()
|
||||
json.dump({"files": files, "manifest_head": head}, open(os.path.join(d,"artifact-manifest.json"),"w"))
|
||||
open(os.path.join(d,"manifest-head.txt"),"w").write(head)
|
||||
PY
|
||||
|
||||
RC="$(rc_of bash "$BASH_DIR/evidence-pack.sh" verify-pack testrun --dir "$PACK_DIR")"
|
||||
[[ "$RC" -eq 0 ]] && pass "evidence-pack: unsigned OK in permissive mode (rc=0)" \
|
||||
|| fail "evidence-pack: permissive mode should pass unsigned (rc=$RC)"
|
||||
|
||||
RC="$(rc_of env CASAN_VERIFY_STRICT=1 bash "$BASH_DIR/evidence-pack.sh" verify-pack testrun --dir "$PACK_DIR")"
|
||||
[[ "$RC" -ne 0 ]] && pass "evidence-pack: unsigned FAILS with CASAN_VERIFY_STRICT=1 (rc=$RC)" \
|
||||
|| fail "evidence-pack: strict flag did not fail unsigned (rc=$RC)"
|
||||
|
||||
echo ""
|
||||
echo "===== SEC-01 SUMMARY: PASS=$PASS FAIL=$FAIL ====="
|
||||
[[ "$FAIL" -eq 0 ]] || exit 1
|
||||
@@ -0,0 +1,60 @@
|
||||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
# CASAN Plan-16 SEC-02 (H-02) — no local key auto-generation in enforced mode.
|
||||
#
|
||||
# The tamper-evidence of the audit chain rests on the head signature. If the
|
||||
# signer auto-generates a private key next to the data (as dev convenience does),
|
||||
# then any actor who can write the log can also mint a key and re-sign a forged
|
||||
# head. In enforced mode the signer must NOT auto-generate — the key is provisioned
|
||||
# out-of-band (KMS/HSM). This proves:
|
||||
# * permissive mode still auto-generates a local key (dev convenience),
|
||||
# * enforced mode does NOT create a key when none is provisioned, and the
|
||||
# resulting unsigned head then FAILS strict verification (fail-closed).
|
||||
#
|
||||
# Deterministic; hermetic (temp key dir + backup/restore of audit + gov dirs).
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
BASH_DIR="$PROJECT_ROOT/.specify/scripts/bash"
|
||||
WORK="$(mktemp -d)"
|
||||
|
||||
BK="$WORK/bak"; mkdir -p "$BK"
|
||||
cp -a "$PROJECT_ROOT/.specify/logs/audit" "$BK/audit" 2>/dev/null || true
|
||||
cp -a "$PROJECT_ROOT/.specify/level5/central-governance" "$BK/cg" 2>/dev/null || true
|
||||
restore() {
|
||||
rm -rf "$PROJECT_ROOT/.specify/logs/audit"; cp -a "$BK/audit" "$PROJECT_ROOT/.specify/logs/audit" 2>/dev/null || true
|
||||
rm -rf "$PROJECT_ROOT/.specify/level5/central-governance"; cp -a "$BK/cg" "$PROJECT_ROOT/.specify/level5/central-governance" 2>/dev/null || true
|
||||
}
|
||||
trap 'restore; rm -rf "$WORK"' EXIT
|
||||
|
||||
PASS=0; FAIL=0
|
||||
pass() { echo "PASS: $1"; PASS=$((PASS + 1)); }
|
||||
fail() { echo "FAIL: $1"; FAIL=$((FAIL + 1)); }
|
||||
rc_of() { set +e; "$@" >/dev/null 2>&1; echo $?; set -e 2>/dev/null || true; }
|
||||
|
||||
echo "===== Plan-16 SEC-02: no local key auto-gen in enforced mode ====="
|
||||
echo "benign input" > "$WORK/in.txt"
|
||||
|
||||
# 1) Permissive mode: a fresh key dir auto-generates a local key (dev convenience).
|
||||
KD1="$WORK/keys-permissive"
|
||||
env CASAN_AUDIT_KEY_DIR="$KD1" bash "$BASH_DIR/governance-check.sh" "$WORK/in.txt" "$WORK/out1.txt" agent_step >/dev/null 2>&1
|
||||
[[ -f "$KD1/audit-private.pem" ]] \
|
||||
&& pass "permissive mode auto-generates a local signing key" \
|
||||
|| fail "permissive mode did not auto-generate a key (dev convenience broken)"
|
||||
|
||||
# 2) Enforced mode: a fresh key dir must NOT auto-generate a key.
|
||||
KD2="$WORK/keys-enforced"
|
||||
env CASAN_VERIFY_STRICT=1 CASAN_AUDIT_KEY_DIR="$KD2" bash "$BASH_DIR/governance-check.sh" "$WORK/in.txt" "$WORK/out2.txt" agent_step >/dev/null 2>&1
|
||||
[[ ! -f "$KD2/audit-private.pem" ]] \
|
||||
&& pass "enforced mode does NOT auto-generate a local key (H-02 closed)" \
|
||||
|| fail "enforced mode auto-generated a local key (attacker could re-sign forgery)"
|
||||
|
||||
# 3) The unsigned head produced in enforced mode fails strict verification.
|
||||
[[ "$(rc_of env CASAN_VERIFY_STRICT=1 bash "$BASH_DIR/verify-audit-chain.sh")" -ne 0 ]] \
|
||||
&& pass "unsigned head from enforced run FAILS strict verify (fail-closed)" \
|
||||
|| fail "unsigned head passed strict verify"
|
||||
|
||||
echo ""
|
||||
echo "===== SEC-02 SUMMARY: PASS=$PASS FAIL=$FAIL ====="
|
||||
[[ "$FAIL" -eq 0 ]] || exit 1
|
||||
@@ -0,0 +1,77 @@
|
||||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
# CASAN Plan-16 SEC-03 (H-03) — rollback-manager: no `bash -c`, structured argv only.
|
||||
#
|
||||
# The rollback `execute` path used to `bash -c "$rollback_command"` where the
|
||||
# command was read from the unsigned transaction log — arbitrary code execution
|
||||
# for anyone who can append a line. This proves:
|
||||
# * a genuine checkpoint still restores the file (regression),
|
||||
# * a forged free-form record with an RCE payload is REFUSED (not executed),
|
||||
# * a forged structured record whose backup points outside the controlled
|
||||
# backup dir is REFUSED (cannot copy an arbitrary source file).
|
||||
#
|
||||
# Deterministic; no model/app/network. Hermetic tx log via a temp workspace.
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
RM="$PROJECT_ROOT/.specify/scripts/bash/rollback-manager.sh"
|
||||
TX_LOG="$PROJECT_ROOT/.specify/logs/level5/rollback-transactions.jsonl"
|
||||
WORK="$(mktemp -d)"
|
||||
|
||||
# rollback-manager uses a fixed tx-log path; back it up and restore on exit.
|
||||
[[ -f "$TX_LOG" ]] && cp -p "$TX_LOG" "$WORK/tx.bak"
|
||||
restore_tx() { if [[ -f "$WORK/tx.bak" ]]; then cp -p "$WORK/tx.bak" "$TX_LOG"; else rm -f "$TX_LOG"; fi; }
|
||||
trap 'restore_tx; rm -f "$WORK/PWNED"; rm -rf "$WORK"' EXIT
|
||||
|
||||
PASS=0; FAIL=0
|
||||
pass() { echo "PASS: $1"; PASS=$((PASS + 1)); }
|
||||
fail() { echo "FAIL: $1"; FAIL=$((FAIL + 1)); }
|
||||
rc_of() { set +e; "$@" >/dev/null 2>&1; echo $?; set -e 2>/dev/null || true; }
|
||||
|
||||
echo "===== Plan-16 SEC-03: rollback-manager no bash -c (RCE) ====="
|
||||
|
||||
# 1) Regression: genuine checkpoint -> execute restores exact content.
|
||||
TARGET="$WORK/plan.txt"; echo "ORIGINAL" > "$TARGET"
|
||||
TX="$(bash "$RM" checkpoint "$TARGET" | sed -n 's/.*transaction_id=\([^ ]*\).*/\1/p')"
|
||||
echo "OVERWRITTEN" > "$TARGET"
|
||||
bash "$RM" execute "$TX" >/dev/null 2>&1
|
||||
[[ "$(cat "$TARGET")" == "ORIGINAL" ]] \
|
||||
&& pass "genuine checkpoint restores exact pre-overwrite content" \
|
||||
|| fail "checkpoint did not restore (got: $(cat "$TARGET"))"
|
||||
|
||||
# 2) Fail-able: forged free-form record carrying an RCE payload must be REFUSED.
|
||||
rm -f "$WORK/PWNED"
|
||||
python3 - "$TX_LOG" "$WORK/PWNED" <<'PY'
|
||||
import json, sys
|
||||
log, marker = sys.argv[1], sys.argv[2]
|
||||
open(log, "a", encoding="utf-8").write(json.dumps({
|
||||
"transaction_id": "evil-rce", "action": "checkpoint",
|
||||
"rollback_command": f"touch {marker}"}) + "\n")
|
||||
PY
|
||||
RC="$(rc_of bash "$RM" execute evil-rce)"
|
||||
if [[ "$RC" -ne 0 && ! -f "$WORK/PWNED" ]]; then
|
||||
pass "forged free-form rollback_command REFUSED, no code executed (rc=$RC)"
|
||||
else
|
||||
fail "RCE not prevented (rc=$RC, marker exists=$([[ -f "$WORK/PWNED" ]] && echo yes || echo no))"
|
||||
fi
|
||||
|
||||
# 3) Fail-able: forged restore_file whose backup is outside the controlled dir.
|
||||
SECRET="$WORK/secret.txt"; echo "SECRET" > "$SECRET"
|
||||
python3 - "$TX_LOG" "$SECRET" "$WORK/stolen.txt" <<'PY'
|
||||
import json, sys
|
||||
log, backup, target = sys.argv[1], sys.argv[2], sys.argv[3]
|
||||
open(log, "a", encoding="utf-8").write(json.dumps({
|
||||
"transaction_id": "evil-src", "action": "checkpoint", "op": "restore_file",
|
||||
"backup": backup, "target": target}) + "\n")
|
||||
PY
|
||||
RC="$(rc_of bash "$RM" execute evil-src)"
|
||||
if [[ "$RC" -ne 0 && ! -f "$WORK/stolen.txt" ]]; then
|
||||
pass "forged restore source outside backup dir REFUSED (rc=$RC)"
|
||||
else
|
||||
fail "arbitrary-source restore not prevented (rc=$RC)"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "===== SEC-03 SUMMARY: PASS=$PASS FAIL=$FAIL ====="
|
||||
[[ "$FAIL" -eq 0 ]] || exit 1
|
||||
@@ -0,0 +1,52 @@
|
||||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
# CASAN Plan-16 SEC-04 (H-05) — action-gate fail-closed.
|
||||
#
|
||||
# Previously, if the classifier python crashed / was killed / emitted nothing,
|
||||
# the verdict string was empty and the final case fell through to ALLOW
|
||||
# (fail-open). This proves the gate now DENIES (exit 2) on classifier failure or
|
||||
# an unrecognized verdict, while ordinary verdicts still resolve correctly.
|
||||
#
|
||||
# Deterministic; no model/app/network.
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
GATE="$PROJECT_ROOT/.specify/scripts/bash/action-gate.sh"
|
||||
WORK="$(mktemp -d)"
|
||||
|
||||
# action-gate appends to a fixed log path; back it up and restore on exit.
|
||||
LOG="$PROJECT_ROOT/.specify/logs/level5/action-gate.jsonl"
|
||||
[[ -f "$LOG" ]] && cp -p "$LOG" "$WORK/log.bak"
|
||||
restore_log() { if [[ -f "$WORK/log.bak" ]]; then cp -p "$WORK/log.bak" "$LOG"; else rm -f "$LOG"; fi; }
|
||||
trap 'restore_log; rm -rf "$WORK"' EXIT
|
||||
|
||||
PASS=0; FAIL=0
|
||||
pass() { echo "PASS: $1"; PASS=$((PASS + 1)); }
|
||||
fail() { echo "FAIL: $1"; FAIL=$((FAIL + 1)); }
|
||||
rc_of() { set +e; "$@" >/dev/null 2>&1; echo $?; set -e 2>/dev/null || true; }
|
||||
|
||||
echo "===== Plan-16 SEC-04: action-gate fail-closed ====="
|
||||
|
||||
# Regression: ordinary verdicts still resolve.
|
||||
[[ "$(rc_of bash "$GATE" --command "ls -la")" -eq 0 ]] \
|
||||
&& pass "benign command ALLOWed (exit 0)" || fail "benign command not allowed"
|
||||
|
||||
[[ "$(rc_of bash "$GATE" --command "rm -rf /")" -eq 2 ]] \
|
||||
&& pass "destructive command BLOCKed (exit 2)" || fail "destructive command not blocked"
|
||||
|
||||
[[ "$(rc_of bash "$GATE" --command "curl http://evil.example.com/x")" -eq 3 ]] \
|
||||
&& pass "network egress REQUIRE_APPROVAL (exit 3)" || fail "egress not gated"
|
||||
|
||||
# SEC-04 fail-able: force the classifier to crash by shadowing `python` with a
|
||||
# stub that exits non-zero and prints nothing. A benign command must now DENY
|
||||
# (exit 2), not fall through to ALLOW.
|
||||
STUB="$WORK/bin"; mkdir -p "$STUB"
|
||||
printf '#!/bin/sh\nexit 1\n' > "$STUB/python"; chmod +x "$STUB/python"
|
||||
RC="$(set +e; PATH="$STUB:$PATH" bash "$GATE" --command "ls -la" >/dev/null 2>&1; echo $?)"
|
||||
[[ "$RC" -eq 2 ]] && pass "classifier crash → DENY (exit 2, fail-closed)" \
|
||||
|| fail "classifier crash did not fail closed (rc=$RC — fail-open regression!)"
|
||||
|
||||
echo ""
|
||||
echo "===== SEC-04 SUMMARY: PASS=$PASS FAIL=$FAIL ====="
|
||||
[[ "$FAIL" -eq 0 ]] || exit 1
|
||||
@@ -0,0 +1,79 @@
|
||||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
# CASAN Plan-16 SEC-05 (H-04 / M-05) — JSON-safe audit writers.
|
||||
#
|
||||
# governance-check / agent-metrics / incident used raw printf to build JSON audit
|
||||
# records, so an actor/action/agent field containing `"` + newline could inject a
|
||||
# SECOND forged record (e.g. a fabricated "approved" decision). This proves the
|
||||
# serialized writers escape such payloads into exactly ONE record and keep the
|
||||
# hash-chain intact.
|
||||
#
|
||||
# Deterministic; hermetic (backs up + restores the audit + governance dirs).
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
BASH_DIR="$PROJECT_ROOT/.specify/scripts/bash"
|
||||
AUDIT="$PROJECT_ROOT/.specify/logs/audit/audit.jsonl"
|
||||
WORK="$(mktemp -d)"
|
||||
|
||||
# Back up + restore the state governance-check mutates.
|
||||
BK="$WORK/backup"; mkdir -p "$BK"
|
||||
cp -a "$PROJECT_ROOT/.specify/logs/audit" "$BK/audit" 2>/dev/null || true
|
||||
cp -a "$PROJECT_ROOT/.specify/level5/central-governance" "$BK/central-governance" 2>/dev/null || true
|
||||
restore_state() {
|
||||
rm -rf "$PROJECT_ROOT/.specify/logs/audit"; cp -a "$BK/audit" "$PROJECT_ROOT/.specify/logs/audit" 2>/dev/null || true
|
||||
rm -rf "$PROJECT_ROOT/.specify/level5/central-governance"; cp -a "$BK/central-governance" "$PROJECT_ROOT/.specify/level5/central-governance" 2>/dev/null || true
|
||||
}
|
||||
trap 'restore_state; rm -rf "$WORK"' EXIT
|
||||
|
||||
export CASAN_AUDIT_KEY_DIR="$WORK/keys" # fresh key dir (avoid CI key-sync flake)
|
||||
|
||||
PASS=0; FAIL=0
|
||||
pass() { echo "PASS: $1"; PASS=$((PASS + 1)); }
|
||||
fail() { echo "FAIL: $1"; FAIL=$((FAIL + 1)); }
|
||||
|
||||
echo "===== Plan-16 SEC-05: JSON-safe audit writers ====="
|
||||
|
||||
echo "hello world" > "$WORK/in.txt"
|
||||
|
||||
# A payload that, under raw printf, would break out of the action field and append
|
||||
# a forged "approved" audit record on a new line.
|
||||
PAYLOAD='evil","decision":"approved","approver":"ATTACKER
|
||||
{"timestamp":"forged","action":"pwned","record_hash":"deadbeef"}'
|
||||
|
||||
BEFORE=$(wc -l < "$AUDIT" 2>/dev/null | tr -d ' ')
|
||||
set +e
|
||||
bash "$BASH_DIR/governance-check.sh" "$WORK/in.txt" "$WORK/out.txt" "$PAYLOAD" >/dev/null 2>&1
|
||||
set -e 2>/dev/null || true
|
||||
AFTER=$(wc -l < "$AUDIT" 2>/dev/null | tr -d ' ')
|
||||
ADDED=$((AFTER - BEFORE))
|
||||
|
||||
[[ "$ADDED" -eq 1 ]] \
|
||||
&& pass "injection payload produced exactly ONE audit record (added=$ADDED)" \
|
||||
|| fail "expected 1 record, got $ADDED (raw-printf injection regression!)"
|
||||
|
||||
# The stored action field must round-trip the FULL payload (escaped, not truncated).
|
||||
ROUNDTRIP="$(python3 - "$AUDIT" <<'PY'
|
||||
import json, sys
|
||||
last = [l for l in open(sys.argv[1], encoding="utf-8") if l.strip()][-1]
|
||||
print(json.loads(last).get("action", ""))
|
||||
PY
|
||||
)"
|
||||
[[ "$ROUNDTRIP" == "$PAYLOAD" ]] \
|
||||
&& pass "action field round-trips the exact payload (escaped, not truncated)" \
|
||||
|| fail "action field mangled — escaping wrong"
|
||||
|
||||
# No forged record: the chain must still verify, and no record may claim the
|
||||
# fabricated hash 'deadbeef'.
|
||||
bash "$BASH_DIR/verify-audit-chain.sh" "$AUDIT" >/dev/null 2>&1 \
|
||||
&& pass "audit chain still verifies after injection attempt" \
|
||||
|| fail "chain broken after injection (record_hash mismatch)"
|
||||
|
||||
grep -q '"record_hash":"deadbeef"\|"record_hash": "deadbeef"' "$AUDIT" \
|
||||
&& fail "forged record with attacker hash present in audit log" \
|
||||
|| pass "no forged record injected into audit log"
|
||||
|
||||
echo ""
|
||||
echo "===== SEC-05 SUMMARY: PASS=$PASS FAIL=$FAIL ====="
|
||||
[[ "$FAIL" -eq 0 ]] || exit 1
|
||||
@@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
# CASAN Plan-16 SEC-06 (H-06) — control-plane audit head is SIGNED.
|
||||
#
|
||||
# The control-plane audit chain is a recomputable hash chain: a file-writer who
|
||||
# edits the store could recompute every hash and the chain-only `verify-audit`
|
||||
# would still PASS, so governance-report would falsely report CERTIFIED. Signing
|
||||
# the head with an off-repo key closes this:
|
||||
# * an intact signed store verifies (permissive + strict),
|
||||
# * a recompute-tampered store FAILS (signature no longer matches the head),
|
||||
# * a fully unsigned store FAILS in enforced mode (governance-report inherits
|
||||
# CASAN_PROFILE=prod, so a certified run demands a signature).
|
||||
#
|
||||
# Deterministic; hermetic (temp store + temp keys + temp pubkey).
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
CP="$PROJECT_ROOT/.specify/scripts/bash/control-plane-settings.py"
|
||||
WORK="$(mktemp -d)"
|
||||
trap 'rm -rf "$WORK"' EXIT
|
||||
|
||||
export CASAN_CP_STORE_FILE="$WORK/store.json"
|
||||
export CASAN_CP_KEY_DIR="$WORK/keys"
|
||||
export CASAN_CP_PUB="$WORK/cp-public.pem" # provisioned out-of-band (attacker cannot rewrite in this test)
|
||||
|
||||
PASS=0; FAIL=0
|
||||
pass() { echo "PASS: $1"; PASS=$((PASS + 1)); }
|
||||
fail() { echo "FAIL: $1"; FAIL=$((FAIL + 1)); }
|
||||
rc_of() { set +e; "$@" >/dev/null 2>&1; echo $?; set -e 2>/dev/null || true; }
|
||||
|
||||
echo "===== Plan-16 SEC-06: control-plane audit head signed ====="
|
||||
|
||||
python3 "$CP" set compression.enabled true --actor a@x --reason r >/dev/null 2>&1
|
||||
python3 "$CP" set compression.mode structural --actor a@x --reason r >/dev/null 2>&1
|
||||
|
||||
[[ "$(rc_of python3 "$CP" verify-audit)" -eq 0 ]] \
|
||||
&& pass "intact signed store verifies (permissive)" || fail "intact store rejected"
|
||||
[[ "$(rc_of env CASAN_VERIFY_STRICT=1 python3 "$CP" verify-audit)" -eq 0 ]] \
|
||||
&& pass "intact signed store verifies (strict)" || fail "intact store rejected in strict"
|
||||
|
||||
# Recompute-attack: tamper a value AND recompute the entire hash chain so it is
|
||||
# internally consistent. The head signature (over the original head) must not match.
|
||||
python3 - "$CASAN_CP_STORE_FILE" <<'PY'
|
||||
import hashlib, json, sys
|
||||
p = sys.argv[1]; d = json.load(open(p))
|
||||
d["settings"]["compression.enabled"]["value"] = False
|
||||
prev = "0" * 64
|
||||
def he(e): return hashlib.sha256(json.dumps(e, sort_keys=True, ensure_ascii=False).encode()).hexdigest()
|
||||
for e in d["audit"]:
|
||||
if e.get("key") == "compression.enabled" and e.get("action") == "set":
|
||||
e["value"] = False
|
||||
rest = {k: v for k, v in e.items() if k != "hash"}; rest["prevHash"] = prev
|
||||
for k in list(e):
|
||||
if k != "hash": e[k] = rest[k]
|
||||
e["hash"] = he(rest); prev = e["hash"]
|
||||
json.dump(d, open(p, "w"), indent=2)
|
||||
PY
|
||||
[[ "$(rc_of python3 "$CP" verify-audit)" -ne 0 ]] \
|
||||
&& pass "recompute-tampered store FAILS (head signature mismatch)" \
|
||||
|| fail "recompute attack passed verification (H-06 not closed!)"
|
||||
|
||||
# Keyless attacker: delete the signature artifacts entirely. Enforced mode (what a
|
||||
# certified run / prod profile uses) must reject an unsigned store; dev stays lax.
|
||||
rm -f "$CASAN_CP_STORE_FILE.head" "$CASAN_CP_STORE_FILE.head.sig" "$CASAN_CP_PUB"
|
||||
# rebuild a clean, internally-valid but UNSIGNED store (fresh sets would re-sign, so
|
||||
# strip the signature after): easiest is to reuse the tampered chain which is
|
||||
# internally valid; with sig files gone it is "unsigned".
|
||||
[[ "$(rc_of env CASAN_VERIFY_STRICT=1 python3 "$CP" verify-audit)" -ne 0 ]] \
|
||||
&& pass "unsigned store FAILS in enforced mode (prod / certified run)" \
|
||||
|| fail "unsigned store accepted in enforced mode"
|
||||
[[ "$(rc_of python3 "$CP" verify-audit)" -eq 0 ]] \
|
||||
&& pass "unsigned store still OK in permissive dev mode (backward compat)" \
|
||||
|| fail "permissive mode wrongly rejected unsigned store"
|
||||
|
||||
echo ""
|
||||
echo "===== SEC-06 SUMMARY: PASS=$PASS FAIL=$FAIL ====="
|
||||
[[ "$FAIL" -eq 0 ]] || exit 1
|
||||
@@ -0,0 +1,79 @@
|
||||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
# CASAN Plan-16 SEC-16 (ARCH-01) — signed harness + policy bundle.
|
||||
#
|
||||
# Editing a gate or policy file (security-check.sh, prompt-filter.yaml,
|
||||
# thresholds.yaml, *.pin, reviewers.registry) disables a control with no input
|
||||
# trace. The signed bundle manifest binds the harness code + policy; the harness
|
||||
# verifies its self-hash and refuses on drift. Proves:
|
||||
# * intact bundle verifies (permissive + strict/signed),
|
||||
# * a 1-byte edit to a gate script → verify FAIL (drift),
|
||||
# * a new unmanifested policy file → verify FAIL,
|
||||
# * a tampered manifest → verify FAIL (signature invalid),
|
||||
# * casan-harness in prod REFUSES to run when the bundle has drifted.
|
||||
#
|
||||
# Deterministic; hermetic (temp bundle root + temp keys/manifest).
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
BASH_DIR="$PROJECT_ROOT/.specify/scripts/bash"
|
||||
BI="$BASH_DIR/bundle-integrity.py"
|
||||
WORK="$(mktemp -d)"
|
||||
trap 'rm -rf "$WORK"' EXIT
|
||||
|
||||
PASS=0; FAIL=0
|
||||
pass() { echo "PASS: $1"; PASS=$((PASS + 1)); }
|
||||
fail() { echo "FAIL: $1"; FAIL=$((FAIL + 1)); }
|
||||
rc_of() { set +e; "$@" >/dev/null 2>&1; echo $?; set -e 2>/dev/null || true; }
|
||||
|
||||
echo "===== Plan-16 SEC-16: signed harness + policy bundle ====="
|
||||
|
||||
# --- Tool-level (hermetic temp bundle) ---
|
||||
ROOT="$WORK/root"; mkdir -p "$ROOT/scripts/bash" "$ROOT/config"
|
||||
echo 'echo gate' > "$ROOT/scripts/bash/security-check.sh"
|
||||
echo 'block: [rm -rf]' > "$ROOT/config/prompt-filter.yaml"
|
||||
export CASAN_BUNDLE_ROOT="$ROOT" CASAN_BUNDLE_MANIFEST="$WORK/m.json"
|
||||
export CASAN_BUNDLE_KEY_DIR="$WORK/keys" CASAN_BUNDLE_PUB="$WORK/b.pub"
|
||||
|
||||
python3 "$BI" generate >/dev/null 2>&1
|
||||
[[ "$(rc_of python3 "$BI" verify)" -eq 0 ]] && pass "intact bundle verifies" || fail "intact verify failed"
|
||||
[[ "$(rc_of python3 "$BI" verify --strict)" -eq 0 ]] && pass "intact bundle verifies (strict/signed)" || fail "strict verify failed"
|
||||
|
||||
echo 'echo gate; echo BACKDOOR' > "$ROOT/scripts/bash/security-check.sh"
|
||||
[[ "$(rc_of python3 "$BI" verify)" -ne 0 ]] && pass "1-byte gate edit → FAIL (drift)" || fail "drift not detected"
|
||||
|
||||
echo 'echo gate' > "$ROOT/scripts/bash/security-check.sh" # restore
|
||||
echo 'x: 1' > "$ROOT/config/thresholds.yaml" # add unmanifested policy
|
||||
[[ "$(rc_of python3 "$BI" verify)" -ne 0 ]] && pass "unmanifested policy file → FAIL" || fail "unmanifested not detected"
|
||||
|
||||
rm -f "$ROOT/config/thresholds.yaml"; python3 "$BI" generate >/dev/null 2>&1
|
||||
python3 - "$WORK/m.json" <<'PY'
|
||||
import json, sys
|
||||
d = json.load(open(sys.argv[1])); k = list(d["files"]); d["files"][k[0]] = "0" * 64
|
||||
json.dump(d, open(sys.argv[1], "w"))
|
||||
PY
|
||||
[[ "$(rc_of python3 "$BI" verify)" -ne 0 ]] && pass "tampered manifest → FAIL (signature invalid)" || fail "manifest tamper not detected"
|
||||
|
||||
unset CASAN_BUNDLE_ROOT CASAN_BUNDLE_MANIFEST CASAN_BUNDLE_KEY_DIR CASAN_BUNDLE_PUB
|
||||
|
||||
# --- Harness-level: prod run refuses on bundle drift ---
|
||||
HROOT="$WORK/hroot"; mkdir -p "$HROOT/scripts/bash"
|
||||
echo 'echo gate' > "$HROOT/scripts/bash/security-check.sh"
|
||||
env CASAN_BUNDLE_ROOT="$HROOT" CASAN_BUNDLE_MANIFEST="$WORK/hm.json" \
|
||||
CASAN_BUNDLE_KEY_DIR="$WORK/hkeys" CASAN_BUNDLE_PUB="$WORK/h.pub" \
|
||||
python3 "$BI" generate >/dev/null 2>&1
|
||||
echo 'echo gate; echo BACKDOOR' > "$HROOT/scripts/bash/security-check.sh" # drift the bundle
|
||||
|
||||
echo "hi" > "$WORK/in.txt"
|
||||
OUT="$(set +e; env CASAN_PROFILE=prod CASAN_KILLSWITCH_DIR="$WORK/noks" \
|
||||
CASAN_BUNDLE_ROOT="$HROOT" CASAN_BUNDLE_MANIFEST="$WORK/hm.json" \
|
||||
CASAN_BUNDLE_KEY_DIR="$WORK/hkeys" CASAN_BUNDLE_PUB="$WORK/h.pub" \
|
||||
bash "$BASH_DIR/casan-harness.sh" "$WORK/in.txt" "$WORK/o.txt" act -- bash -c 'echo x' 2>&1)"
|
||||
echo "$OUT" | grep -q "BUNDLE_INTEGRITY_DRIFT" \
|
||||
&& pass "casan-harness (prod) REFUSES to run on bundle drift" \
|
||||
|| fail "harness did not refuse on bundle drift"
|
||||
|
||||
echo ""
|
||||
echo "===== SEC-16 SUMMARY: PASS=$PASS FAIL=$FAIL ====="
|
||||
[[ "$FAIL" -eq 0 ]] || exit 1
|
||||
@@ -0,0 +1,74 @@
|
||||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
# CASAN Plan-16 SEC-17 (ARCH-03) — CASAN_PROFILE=prod is secure-by-default.
|
||||
#
|
||||
# Many strong controls were opt-in env flags (off unless explicitly set), so a
|
||||
# lazy operator ran permissively. Under CASAN_PROFILE=prod they now default ON —
|
||||
# while an explicit `=0` still wins (internal scans that disable a flag on purpose
|
||||
# must stay disabled). Proven across three real controls:
|
||||
# * verify-audit-chain: unsigned chain fails under prod,
|
||||
# * control-plane verify-audit: unsigned store fails under prod,
|
||||
# * casan-harness kill-switch: engaged switch is enforced under prod, but an
|
||||
# explicit CASAN_KILLSWITCH_ENFORCE=0 overrides.
|
||||
#
|
||||
# Deterministic; hermetic (temp dirs); no model/network.
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
BASH_DIR="$PROJECT_ROOT/.specify/scripts/bash"
|
||||
WORK="$(mktemp -d)"
|
||||
trap 'rm -rf "$WORK"' EXIT
|
||||
|
||||
PASS=0; FAIL=0
|
||||
pass() { echo "PASS: $1"; PASS=$((PASS + 1)); }
|
||||
fail() { echo "FAIL: $1"; FAIL=$((FAIL + 1)); }
|
||||
rc_of() { set +e; "$@" >/dev/null 2>&1; echo $?; set -e 2>/dev/null || true; }
|
||||
|
||||
echo "===== Plan-16 SEC-17: CASAN_PROFILE=prod secure-by-default ====="
|
||||
|
||||
# --- Control 1: verify-audit-chain (unsigned) ---
|
||||
AUD="$WORK/audit"; mkdir -p "$AUD"
|
||||
python3 - "$AUD/audit.jsonl" <<'PY'
|
||||
import hashlib, json, sys
|
||||
core = "|".join(["2026-07-06T00:00:00Z","t1","act","a@x","low","ALLOW","n/a","","ih","oh",""])
|
||||
h = hashlib.sha256(core.encode()).hexdigest()
|
||||
open(sys.argv[1],"w").write(json.dumps({"timestamp":"2026-07-06T00:00:00Z","trace_id":"t1","action":"act",
|
||||
"actor":"a@x","risk_level":"low","decision":"ALLOW","approval_status":"n/a","approver":"",
|
||||
"input_hash":"ih","output_hash":"oh","previous_record_hash":"","record_hash":h})+"\n")
|
||||
PY
|
||||
[[ "$(rc_of bash "$BASH_DIR/verify-audit-chain.sh" "$AUD/audit.jsonl")" -eq 0 ]] \
|
||||
&& pass "verify-audit-chain: permissive default allows unsigned" || fail "permissive should allow unsigned"
|
||||
[[ "$(rc_of env CASAN_PROFILE=prod bash "$BASH_DIR/verify-audit-chain.sh" "$AUD/audit.jsonl")" -ne 0 ]] \
|
||||
&& pass "verify-audit-chain: prod profile enforces (unsigned FAILS)" || fail "prod did not enforce audit-chain"
|
||||
|
||||
# --- Control 2: control-plane verify-audit (unsigned store) ---
|
||||
export CASAN_CP_STORE_FILE="$WORK/cp.json" CASAN_CP_KEY_DIR="$WORK/cpkeys" CASAN_CP_PUB="$WORK/cp.pub"
|
||||
python3 "$BASH_DIR/control-plane-settings.py" set compression.enabled true --actor a --reason r >/dev/null 2>&1
|
||||
rm -f "$WORK/cp.json.head" "$WORK/cp.json.head.sig" "$WORK/cp.pub" # strip the signature -> unsigned store
|
||||
[[ "$(rc_of python3 "$BASH_DIR/control-plane-settings.py" verify-audit)" -eq 0 ]] \
|
||||
&& pass "control-plane: permissive default allows unsigned store" || fail "permissive should allow unsigned store"
|
||||
[[ "$(rc_of env CASAN_PROFILE=prod python3 "$BASH_DIR/control-plane-settings.py" verify-audit)" -ne 0 ]] \
|
||||
&& pass "control-plane: prod profile enforces (unsigned store FAILS)" || fail "prod did not enforce control-plane"
|
||||
unset CASAN_CP_STORE_FILE CASAN_CP_KEY_DIR CASAN_CP_PUB
|
||||
|
||||
# --- Control 3: casan-harness kill-switch (prod-on + explicit-0-wins) ---
|
||||
export CASAN_KILLSWITCH_DIR="$WORK/ks"
|
||||
echo "hi" > "$WORK/in.txt"
|
||||
bash "$BASH_DIR/kill-switch.sh" engage project sec17 "test" >/dev/null 2>&1
|
||||
|
||||
OUT="$(set +e; CASAN_PROFILE=prod CASAN_KILLSWITCH_SCOPE=project CASAN_KILLSWITCH_ID=sec17 \
|
||||
bash "$BASH_DIR/casan-harness.sh" "$WORK/in.txt" "$WORK/o.txt" act -- bash -c 'echo x' 2>&1)"
|
||||
echo "$OUT" | grep -q "KILL_SWITCH_ACTIVE" \
|
||||
&& pass "kill-switch: prod profile enforces an engaged switch (no explicit flag)" \
|
||||
|| fail "prod profile did not enforce kill-switch"
|
||||
|
||||
OUT="$(set +e; CASAN_PROFILE=prod CASAN_KILLSWITCH_ENFORCE=0 CASAN_KILLSWITCH_SCOPE=project CASAN_KILLSWITCH_ID=sec17 \
|
||||
bash "$BASH_DIR/casan-harness.sh" "$WORK/in.txt" "$WORK/o2.txt" act -- bash -c 'echo x' 2>&1)"
|
||||
echo "$OUT" | grep -q "KILL_SWITCH_ACTIVE" \
|
||||
&& fail "explicit CASAN_KILLSWITCH_ENFORCE=0 was overridden by prod (should win)" \
|
||||
|| pass "kill-switch: explicit =0 overrides prod default (internal opt-out preserved)"
|
||||
|
||||
echo ""
|
||||
echo "===== SEC-17 SUMMARY: PASS=$PASS FAIL=$FAIL ====="
|
||||
[[ "$FAIL" -eq 0 ]] || exit 1
|
||||
@@ -0,0 +1,84 @@
|
||||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
# CASAN Plan-16 SEC-18 (ARCH-02) — test-integrity manifest.
|
||||
#
|
||||
# Test suites are in the same repo an attacker can edit, so "green" proves nothing
|
||||
# if a fail-able check was quietly deleted. The signed manifest records per-suite
|
||||
# hash + fail-able-check count; CI re-verifies. This proves:
|
||||
# * intact suites verify (permissive + strict/signed),
|
||||
# * deleting a fail-able check -> FAIL (coverage regression),
|
||||
# * removing a whole suite -> FAIL (suite removed),
|
||||
# * tampering the manifest -> FAIL (signature invalid).
|
||||
#
|
||||
# Deterministic; hermetic (operates on a temp copy of the test dir).
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
TI="$PROJECT_ROOT/.specify/scripts/bash/test-integrity.py"
|
||||
WORK="$(mktemp -d)"
|
||||
trap 'rm -rf "$WORK"' EXIT
|
||||
|
||||
mkdir -p "$WORK/tests"
|
||||
cp "$PROJECT_ROOT/.specify/tests/phase-sec01-tests.sh" \
|
||||
"$PROJECT_ROOT/.specify/tests/phase-control-plane-tests.sh" "$WORK/tests/"
|
||||
export CASAN_TESTS_DIR="$WORK/tests"
|
||||
export CASAN_TEST_MANIFEST="$WORK/manifest.json"
|
||||
export CASAN_TI_KEY_DIR="$WORK/keys"
|
||||
export CASAN_TI_PUB="$WORK/ti.pub"
|
||||
|
||||
PASS=0; FAIL=0
|
||||
pass() { echo "PASS: $1"; PASS=$((PASS + 1)); }
|
||||
fail() { echo "FAIL: $1"; FAIL=$((FAIL + 1)); }
|
||||
rc_of() { set +e; "$@" >/dev/null 2>&1; echo $?; set -e 2>/dev/null || true; }
|
||||
|
||||
echo "===== Plan-16 SEC-18: test-integrity manifest ====="
|
||||
|
||||
python3 "$TI" generate >/dev/null 2>&1
|
||||
[[ "$(rc_of python3 "$TI" verify)" -eq 0 ]] \
|
||||
&& pass "intact suites verify" || fail "intact verify failed"
|
||||
[[ "$(rc_of python3 "$TI" verify --strict)" -eq 0 ]] \
|
||||
&& pass "intact suites verify under --strict (signed)" || fail "strict verify failed on signed manifest"
|
||||
|
||||
# Delete one fail-able check line from a suite.
|
||||
python3 - "$WORK/tests/phase-sec01-tests.sh" <<'PY'
|
||||
import re, sys
|
||||
p = sys.argv[1]
|
||||
lines = open(p, encoding="utf-8").read().splitlines(keepends=True)
|
||||
out, removed = [], False
|
||||
for ln in lines:
|
||||
if not removed and 'pass "' in ln:
|
||||
removed = True # drop the first assertion line
|
||||
continue
|
||||
out.append(ln)
|
||||
open(p, "w", encoding="utf-8").write("".join(out))
|
||||
PY
|
||||
[[ "$(rc_of python3 "$TI" verify)" -ne 0 ]] \
|
||||
&& pass "deleting a fail-able check → FAIL (coverage regression)" \
|
||||
|| fail "coverage regression not detected"
|
||||
|
||||
# Restore, then remove an entire suite.
|
||||
python3 "$TI" generate >/dev/null 2>&1
|
||||
rm -f "$WORK/tests/phase-control-plane-tests.sh"
|
||||
[[ "$(rc_of python3 "$TI" verify)" -ne 0 ]] \
|
||||
&& pass "removing a whole suite → FAIL (suite removed)" \
|
||||
|| fail "suite removal not detected"
|
||||
|
||||
# Restore, then tamper the manifest content (without re-signing).
|
||||
cp "$PROJECT_ROOT/.specify/tests/phase-control-plane-tests.sh" "$WORK/tests/"
|
||||
python3 "$TI" generate >/dev/null 2>&1
|
||||
python3 - "$WORK/manifest.json" <<'PY'
|
||||
import json, sys
|
||||
p = sys.argv[1]; d = json.load(open(p))
|
||||
# lower a recorded count so a later real drop would pass — the signature must catch this
|
||||
for name in d["suites"]:
|
||||
d["suites"][name]["checks"] = 0
|
||||
json.dump(d, open(p, "w"), indent=2)
|
||||
PY
|
||||
[[ "$(rc_of python3 "$TI" verify)" -ne 0 ]] \
|
||||
&& pass "tampering the manifest → FAIL (signature invalid)" \
|
||||
|| fail "manifest tamper not detected by signature"
|
||||
|
||||
echo ""
|
||||
echo "===== SEC-18 SUMMARY: PASS=$PASS FAIL=$FAIL ====="
|
||||
[[ "$FAIL" -eq 0 ]] || exit 1
|
||||
@@ -223,11 +223,16 @@ bash "$SCRIPTS/sign-audit-head.sh" "$PROJECT_ROOT/.specify/logs/audit/audit.json
|
||||
"$SCRIPTS/verify-tool-audit.sh" "$PROJECT_ROOT/.specify/logs/audit/tool-calls.jsonl" > "$LEVEL5_DIR/11c-tool-audit-verify.stdout"
|
||||
assert_contains "$LEVEL5_DIR/11c-tool-audit-verify.stdout" "TOOL_AUDIT_VALID"
|
||||
|
||||
# L5: rollback transaction record and execute
|
||||
# L5: rollback transaction — checkpoint a file and execute a genuine restore.
|
||||
# SEC-03 (H-03): `execute` runs only whitelisted STRUCTURED ops (no `bash -c`), so
|
||||
# rollback is exercised via `checkpoint` (the safe path) rather than a free-form
|
||||
# recorded shell command. The marker starts at the state we expect restored.
|
||||
ROLLBACK_MARKER="$LEVEL5_DIR/13-rollback-marker.txt"
|
||||
ROLLBACK_RECORD="$("$SCRIPTS/rollback-manager.sh" record deploy "printf rolled_back > '$ROLLBACK_MARKER'")"
|
||||
printf 'rolled_back' > "$ROLLBACK_MARKER"
|
||||
ROLLBACK_RECORD="$("$SCRIPTS/rollback-manager.sh" checkpoint "$ROLLBACK_MARKER")"
|
||||
printf '%s\n' "$ROLLBACK_RECORD" > "$LEVEL5_DIR/13-rollback-record.stdout"
|
||||
TX_ID="$(printf '%s\n' "$ROLLBACK_RECORD" | sed -n 's/.*transaction_id=//p')"
|
||||
TX_ID="$(printf '%s\n' "$ROLLBACK_RECORD" | sed -n 's/.*transaction_id=\([^ ]*\).*/\1/p')"
|
||||
printf 'MODIFIED_AFTER_CHECKPOINT' > "$ROLLBACK_MARKER"
|
||||
"$SCRIPTS/rollback-manager.sh" execute "$TX_ID" > "$LEVEL5_DIR/13-rollback-execute.stdout"
|
||||
assert_contains "$ROLLBACK_MARKER" "rolled_back"
|
||||
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
{
|
||||
"suites": {
|
||||
"adversarial-harness-tests.sh": {
|
||||
"sha256": "3cedc614214045ea3c58ab4ddb018ed7e69e2a9051ee1dd5f7228258ba2417a7",
|
||||
"checks": 41
|
||||
},
|
||||
"phase-c6-sandbox-tests.sh": {
|
||||
"sha256": "b1ae109f2449ddc59547551d1c0e457ce767d55451854b3d40ca2d9ca4e851b7",
|
||||
"checks": 4
|
||||
},
|
||||
"phase-c7-incident-tests.sh": {
|
||||
"sha256": "87f83d660bbc5ee541fd299472d991694e44013a1248ee5ebb802b5009b99327",
|
||||
"checks": 16
|
||||
},
|
||||
"phase-control-plane-tests.sh": {
|
||||
"sha256": "50c1c1b20ea43858d5f6a65b5fa0b7c09c13e3ae8b2348ca983bc15733b10130",
|
||||
"checks": 9
|
||||
},
|
||||
"phase-governance-report-tests.sh": {
|
||||
"sha256": "a0720d70fae876a346037bce6762576bfba0b65c0148593d860550befa7a0182",
|
||||
"checks": 5
|
||||
},
|
||||
"phase-h4-multilingual-tests.sh": {
|
||||
"sha256": "e99ca2b85c5d70987ff0cae162bb370a612574093f164567c92bf3075e965a7f",
|
||||
"checks": 3
|
||||
},
|
||||
"phase-h4-split-inject-tests.sh": {
|
||||
"sha256": "60bca85653e6568146056d4c50c71f51d85b025c30fcb75de8ac8b4f9f655d1c",
|
||||
"checks": 9
|
||||
},
|
||||
"phase-h5-approval-tests.sh": {
|
||||
"sha256": "091761e2c99821fec0a11b0099697fe24e3d30a3f6491930acbf10d02cacf7d3",
|
||||
"checks": 13
|
||||
},
|
||||
"phase-h5-infra-tests.sh": {
|
||||
"sha256": "4117c7a21af5be28c1f833efff239ee441cadfdbbec65427fa39d4349a94cdd2",
|
||||
"checks": 8
|
||||
},
|
||||
"phase-h6-agentops-tests.sh": {
|
||||
"sha256": "1b3e347fed4403a227eebb51a888c5b19653fe448e0b3a6e6094d4efccc510d1",
|
||||
"checks": 21
|
||||
},
|
||||
"phase-preflight-tests.sh": {
|
||||
"sha256": "b184211b1fe55d71ce4f0d371d26f5245f721a6bc4a51e3f61301b3126821eea",
|
||||
"checks": 5
|
||||
},
|
||||
"phase-prod-infra-lab-tests.sh": {
|
||||
"sha256": "a6bff248f0c495ebee56e2ba07de270d22503f357434b6cb948430f08cf996a8",
|
||||
"checks": 2
|
||||
},
|
||||
"phase-rai-tests.sh": {
|
||||
"sha256": "bad387ad26ae3088c016f51ff5d6e1c234c9b94fbdb8fa9f736b6bebedd81cd8",
|
||||
"checks": 12
|
||||
},
|
||||
"phase-rbac-tests.sh": {
|
||||
"sha256": "e7a2eb9110539f21eddf77f7fb9df0e616967d0dc870063d11a078e8920a3226",
|
||||
"checks": 12
|
||||
},
|
||||
"phase-sec01-tests.sh": {
|
||||
"sha256": "293b944f71298d38d7e89bea9d4906e40ffbe81f9d822c2ca320bb5a2ba21d92",
|
||||
"checks": 10
|
||||
},
|
||||
"phase-sec02-tests.sh": {
|
||||
"sha256": "4f76e12bcaf96f2d001436af10ddd21d5b858456660d01df8dfbeef3f3de53af",
|
||||
"checks": 3
|
||||
},
|
||||
"phase-sec03-tests.sh": {
|
||||
"sha256": "ea4d00e4dc35e5717dbdedc7fc84b87ec30f2f0c99c0898a113d63582e7ed982",
|
||||
"checks": 3
|
||||
},
|
||||
"phase-sec04-tests.sh": {
|
||||
"sha256": "c451b61e4efa8aba9394a548f8b248285f9f78c9e466d29045e1320877f28ef1",
|
||||
"checks": 4
|
||||
},
|
||||
"phase-sec05-tests.sh": {
|
||||
"sha256": "0800e34c4da323a0937b8594cdee256062575166723e413be9c334ce70290823",
|
||||
"checks": 4
|
||||
},
|
||||
"phase-sec06-tests.sh": {
|
||||
"sha256": "71e04cba738f501e79dd6b2c02aa86c7434c5ce541af5c7e512a2c94219f1c33",
|
||||
"checks": 5
|
||||
},
|
||||
"phase-sec16-tests.sh": {
|
||||
"sha256": "3d826ca4f5c8f98837cc70846b8e2f2f83d5a598c2a5907795e8b9cc9db448c2",
|
||||
"checks": 6
|
||||
},
|
||||
"phase-sec17-tests.sh": {
|
||||
"sha256": "d68b93ebc0d16ee2369a5525fe206a9133c95ef0f931f9bfa799d21b571dbf4c",
|
||||
"checks": 6
|
||||
},
|
||||
"phase-selfimprove-tests.sh": {
|
||||
"sha256": "e91db1af16e30b18def130553f27f05691ff5540eca0593ca5e07f624c0ef938",
|
||||
"checks": 7
|
||||
},
|
||||
"phase08-compression-tests.sh": {
|
||||
"sha256": "3efdb7b13b77579e6d7750add608be0ce894ecc2e816d2d507de8070e794aae7",
|
||||
"checks": 9
|
||||
},
|
||||
"phase1-track-a-tests.sh": {
|
||||
"sha256": "3cf50c42335b23571f75dfae0ac1ba5c0332747a35d99f39a1dddfbd9de8e787",
|
||||
"checks": 22
|
||||
},
|
||||
"phase10-traceability-tests.sh": {
|
||||
"sha256": "eac99b364172a43dee908ae77e3fe3a613d74aad2b70eb95d51ae5df776a1350",
|
||||
"checks": 6
|
||||
},
|
||||
"phase2-sourcegen-tests.sh": {
|
||||
"sha256": "1af646519088faff5ef7971191a30aaa05e3fa8d12fb35bb5483ac11385c2e2e",
|
||||
"checks": 3
|
||||
},
|
||||
"phase2-track-c-tests.sh": {
|
||||
"sha256": "6e57b358f4cc03c3ad393cf320c5a323f993a57647c4ab641266d6cdb3ec1120",
|
||||
"checks": 30
|
||||
},
|
||||
"phase3-evidence-pack-tests.sh": {
|
||||
"sha256": "9aa0f2a2c57a278ed3edb7170f1825cfc2f5c99ce179fd260a28368b446f5481",
|
||||
"checks": 8
|
||||
},
|
||||
"phase3-judge-gate-tests.sh": {
|
||||
"sha256": "3fc0bbbb9fa52bdcc7a0c98746cfe69db93329a09e1f78fa615e57b50a8ebaea",
|
||||
"checks": 0
|
||||
},
|
||||
"phase3-model-router-tests.sh": {
|
||||
"sha256": "f027104f2f115493bfff982218bd316b1cd7734a66a92f50e4d231c8df803aa5",
|
||||
"checks": 11
|
||||
},
|
||||
"phase3-redteam-metrics.sh": {
|
||||
"sha256": "fc8b411cd07cd15b04c836255dfaa50d23d9a11672673fe0caa6e5a55ab11c20",
|
||||
"checks": 0
|
||||
},
|
||||
"run-casan4-harness-tests.sh": {
|
||||
"sha256": "ac572791bdd4d195818932c4cbeb0cf9409f4c63492317334ac06bb2f7bc639a",
|
||||
"checks": 10
|
||||
}
|
||||
},
|
||||
"total_checks": 307,
|
||||
"suite_count": 33
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
06273ff0acaaaabfa2b5ca8d6f06d722e1423d995bcd8ba772251c1282f4dd41
|
||||
@@ -0,0 +1,9 @@
|
||||
-----BEGIN PUBLIC KEY-----
|
||||
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA30OELMJb2Ee2BXI7GlNb
|
||||
pPyX7CVoJhS4FhFzBAWsTmaVISnyTxte1938JwupAPJUoVE2kQTy3lwgPTFrzrlT
|
||||
ZP6SdmaRnT0yL4gN/eXQE7Kq5998Qcaf7VxFE96l9s3bBmonwu/FrgL/Ph05kMOK
|
||||
1yxBOnXlMWqQYyvbyi4BkIgB/fe/HQ04jotNjpPjkTyagPtzm9aZB+e2lThAJt11
|
||||
IRIQfbfyUt5k4aKMHKfUWWrnoFZDj1kXMXSY5HdKRvt03Z8HGo/Kr54XNlYWj8LO
|
||||
PPNKpWrb65JFRgF7TF6IPoInfHGAomTC9fHIxsYyanFe7IHGrCwx8GGAvTT1XgH9
|
||||
WwIDAQAB
|
||||
-----END PUBLIC KEY-----
|
||||
Binary file not shown.
Reference in New Issue
Block a user