Complete CASAN backlog tier 1 controls
This commit is contained in:
thanhnv
@@ -114,7 +114,7 @@ tr:nth-child(even) td {{ background: #f7f9fc; }}
|
||||
<span class="badge lv">CASAN Level 4 — chứng minh bằng tấn công</span>
|
||||
<span class="badge">Average {avg_score}/100</span>
|
||||
<span class="badge">Harness thấp nhất {lowest_score}</span>
|
||||
<span class="badge">175 test · 0 fail</span>
|
||||
<span class="badge">218 core tests · 0 fail</span>
|
||||
<span class="badge">Recall model 0.85 > regex 0.00</span>
|
||||
</div>
|
||||
|
||||
@@ -132,7 +132,7 @@ tr:nth-child(even) td {{ background: #f7f9fc; }}
|
||||
|
||||
<h2>Bảo mật & Governance đã kiểm chứng (test đối kháng thật)</h2>
|
||||
<div class="panel"><div class="chips">
|
||||
<span class="chip"><span class="d"></span>Kiểm thử đối kháng <b>175 / 0 fail</b></span>
|
||||
<span class="chip"><span class="d"></span>Kiểm thử đối kháng <b>218 / 0 fail</b></span>
|
||||
<span class="chip"><span class="d"></span>H4 recall model <b>0.85</b> > regex 0.00</span>
|
||||
<span class="chip"><span class="d"></span>Benign FP <b>0.00%</b> · block <b>100.00%</b></span>
|
||||
<span class="chip"><span class="d"></span>Audit hash-chain + ký KMS (rotate/non-exportable)</span>
|
||||
|
||||
@@ -94,6 +94,38 @@ expect_rc 0 "non-strict env approval unchanged (backward compatible)" \
|
||||
env CASAN_ACTOR=alice CASAN_APPROVAL_DECISION=approve CASAN_APPROVER=bob \
|
||||
bash "$S/governance-check.sh" "$REQ" "$WORK/out.txt" deploy
|
||||
|
||||
echo "===== H5 OIDC approval via mock IdP JWT (CASAN_APPROVAL_STRICT=1) ====="
|
||||
openssl genrsa -out "$WORK/idp.priv.pem" 2048 2>/dev/null
|
||||
openssl rsa -in "$WORK/idp.priv.pem" -pubout -out "$WORK/idp.pub.pem" 2>/dev/null
|
||||
openssl genrsa -out "$WORK/fake-idp.priv.pem" 2048 2>/dev/null
|
||||
|
||||
mint_jwt() {
|
||||
python3 "$S/approval-jwt-mint.py" --key "$1" --sub "$2" --role "$3" \
|
||||
--action deploy --actor alice --input "$REQ" --exp-offset "$4"
|
||||
}
|
||||
|
||||
# 9. Valid IdP-signed JWT by an authorized role -> APPROVED
|
||||
OIDC_JWT="$(mint_jwt "$WORK/idp.priv.pem" oidc-ops ops 300)"
|
||||
OUT="$(gc CASAN_APPROVER=oidc-ops CASAN_APPROVAL_JWT="$OIDC_JWT" CASAN_IDP_PUBLIC_KEY="$WORK/idp.pub.pem" 2>/dev/null)"; RC=$?
|
||||
{ [[ "$RC" -eq 0 ]] && printf '%s' "$OUT" | grep -q "human_approved_oidc"; } \
|
||||
&& pass "valid IdP JWT by authorized role -> APPROVED" \
|
||||
|| fail "valid IdP JWT rejected (rc=$RC out=$OUT)"
|
||||
|
||||
# 10. Expired JWT -> DENY
|
||||
EXPIRED_JWT="$(mint_jwt "$WORK/idp.priv.pem" oidc-ops ops -60)"
|
||||
expect_rc 2 "expired IdP JWT is denied" \
|
||||
gc CASAN_APPROVER=oidc-ops CASAN_APPROVAL_JWT="$EXPIRED_JWT" CASAN_IDP_PUBLIC_KEY="$WORK/idp.pub.pem"
|
||||
|
||||
# 11. Role not authorized for deploy -> DENY
|
||||
WRONG_ROLE_JWT="$(mint_jwt "$WORK/idp.priv.pem" oidc-tech tech_lead 300)"
|
||||
expect_rc 2 "IdP JWT with unauthorized role is denied" \
|
||||
gc CASAN_APPROVER=oidc-tech CASAN_APPROVAL_JWT="$WRONG_ROLE_JWT" CASAN_IDP_PUBLIC_KEY="$WORK/idp.pub.pem"
|
||||
|
||||
# 12. JWT signed by a different key than the trusted IdP pubkey -> DENY
|
||||
FORGED_JWT="$(mint_jwt "$WORK/fake-idp.priv.pem" oidc-ops ops 300)"
|
||||
expect_rc 2 "JWT with forged IdP signature is denied" \
|
||||
gc CASAN_APPROVER=oidc-ops CASAN_APPROVAL_JWT="$FORGED_JWT" CASAN_IDP_PUBLIC_KEY="$WORK/idp.pub.pem"
|
||||
|
||||
echo ""
|
||||
echo "===== H5 APPROVAL-IDENTITY SUMMARY: PASS=$PASS FAIL=$FAIL ====="
|
||||
[[ "$FAIL" -eq 0 ]] || exit 1
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
# CASAN Plan-10 — Traceability REQ→code→test MVP tests.
|
||||
#
|
||||
# Proves the matrix is generated from the real requirement document and the gate
|
||||
# fails when any FR loses test coverage.
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
S="$PROJECT_ROOT/.specify/scripts/bash"
|
||||
WORK="$(mktemp -d)"
|
||||
trap 'rm -rf "$WORK"' EXIT
|
||||
|
||||
PASS=0; FAIL=0
|
||||
pass() { echo "PASS: $1"; PASS=$((PASS + 1)); }
|
||||
fail() { echo "FAIL: $1"; FAIL=$((FAIL + 1)); }
|
||||
|
||||
REQ="$PROJECT_ROOT/docs/input/okr-requirement.md"
|
||||
MAP="$PROJECT_ROOT/.specify/traceability-map.json"
|
||||
OUT="$WORK/traceability-matrix.json"
|
||||
|
||||
echo "===== Plan-10 traceability matrix ====="
|
||||
if python3 "$S/traceability-matrix.py" --requirements "$REQ" --map "$MAP" --out "$OUT" --gate >/dev/null 2>"$WORK/pass.err"; then
|
||||
pass "complete FR→code→test matrix passes gate"
|
||||
else
|
||||
cat "$WORK/pass.err"
|
||||
fail "complete traceability matrix rejected"
|
||||
fi
|
||||
|
||||
COUNT="$(python3 -c "import json;d=json.load(open('$OUT'));print(d['summary']['requirements'])" 2>/dev/null)"
|
||||
FAILED="$(python3 -c "import json;d=json.load(open('$OUT'));print(d['summary']['failed'])" 2>/dev/null)"
|
||||
[[ "$COUNT" == "5" && "$FAILED" == "0" ]] \
|
||||
&& pass "matrix captures all 5 FRs with zero failures" \
|
||||
|| fail "matrix summary unexpected (requirements=$COUNT failed=$FAILED)"
|
||||
|
||||
BROKEN="$WORK/traceability-map-broken.json"
|
||||
python3 - "$MAP" "$BROKEN" <<'PY'
|
||||
import json, sys
|
||||
d = json.load(open(sys.argv[1]))
|
||||
d["FR-04"]["tests"] = []
|
||||
json.dump(d, open(sys.argv[2], "w"), indent=2)
|
||||
PY
|
||||
set +e
|
||||
python3 "$S/traceability-matrix.py" --requirements "$REQ" --map "$BROKEN" --out "$WORK/broken.json" --gate >/dev/null 2>"$WORK/broken.err"
|
||||
RC=$?
|
||||
set -e 2>/dev/null || true
|
||||
[[ "$RC" -eq 1 ]] && grep -q "TRACEABILITY_FAIL FR-04" "$WORK/broken.err" \
|
||||
&& pass "missing FR test coverage fails the gate" \
|
||||
|| fail "missing test coverage did not fail as expected (rc=$RC)"
|
||||
|
||||
echo ""
|
||||
echo "===== TRACEABILITY SUMMARY: PASS=$PASS FAIL=$FAIL ====="
|
||||
[[ "$FAIL" -eq 0 ]] || exit 1
|
||||
@@ -41,10 +41,10 @@ MISSING=0
|
||||
for f in run-summary.json h1-context-report.json h2-tool-audit.json h3-eval-scorecard.json \
|
||||
h4-security-report.json h5-audit-chain-proof.json h6-cost-telemetry.json \
|
||||
h7-orchestration-report.json redteam-result.json benign-fp-report.json \
|
||||
artifact-manifest.json decision-log.md; do
|
||||
artifact-manifest.json traceability-matrix.json decision-log.md; do
|
||||
[[ -f "$PACKDIR/$f" ]] || { echo " missing $f"; MISSING=$((MISSING+1)); }
|
||||
done
|
||||
[[ "$MISSING" -eq 0 ]] && pass "pack contains all 12 standard evidence files" || fail "pack missing $MISSING files"
|
||||
[[ "$MISSING" -eq 0 ]] && pass "pack contains all 13 standard evidence files" || fail "pack missing $MISSING files"
|
||||
[[ -f "$PACKDIR/evidence-pack.sig" ]] && pass "pack is signed (evidence-pack.sig present)" || fail "pack signature missing"
|
||||
|
||||
rc=0; bash "$EP" verify-pack "$RID" >/dev/null 2>&1 || rc=$?
|
||||
|
||||
@@ -66,7 +66,28 @@ else
|
||||
pass "no API-key/secret pattern in .specify/logs"
|
||||
fi
|
||||
|
||||
# 6: cloud backend reports unavailable honestly while keys are unset.
|
||||
# 6: model digest pinning catches a silent model swap, with warn mode available.
|
||||
PIN="$WORK/model-digest.pin"
|
||||
CASAN_MODEL_DIGEST_PIN="$PIN" CASAN_MODEL_DIGEST="digest-approved" bash "$SCRIPTS/model-digest-check.sh" pin ornith:9b >/dev/null 2>&1
|
||||
if CASAN_MODEL_DIGEST_PIN="$PIN" CASAN_MODEL_DIGEST="digest-approved" bash "$SCRIPTS/model-digest-check.sh" verify ornith:9b >/dev/null 2>&1; then
|
||||
pass "model digest pin verifies approved digest"
|
||||
else
|
||||
fail "model digest pin rejected matching digest"
|
||||
fi
|
||||
set +e
|
||||
CASAN_MODEL_DIGEST_PIN="$PIN" CASAN_MODEL_DIGEST="digest-swapped" bash "$SCRIPTS/model-digest-check.sh" verify ornith:9b >/dev/null 2>"$WORK/digest.err"
|
||||
DRC=$?
|
||||
CASAN_MODEL_DIGEST_PIN="$PIN" CASAN_MODEL_DIGEST="digest-swapped" CASAN_MODEL_DIGEST_MODE=warn bash "$SCRIPTS/model-digest-check.sh" verify ornith:9b >/dev/null 2>"$WORK/digest-warn.err"
|
||||
WRC=$?
|
||||
set -e 2>/dev/null || true
|
||||
[[ "$DRC" -eq 2 ]] && grep -q "MODEL_DIGEST_MISMATCH" "$WORK/digest.err" \
|
||||
&& pass "model digest mismatch blocks by default" \
|
||||
|| fail "model digest mismatch did not block (rc=$DRC)"
|
||||
[[ "$WRC" -eq 0 ]] && grep -q "MODEL_DIGEST_WARN" "$WORK/digest-warn.err" \
|
||||
&& pass "model digest mismatch can warn during rollout" \
|
||||
|| fail "model digest warn mode did not warn cleanly (rc=$WRC)"
|
||||
|
||||
# 7: cloud backend reports unavailable honestly while keys are unset.
|
||||
if [[ -z "${ANTHROPIC_API_KEY:-}" ]]; then
|
||||
set +e
|
||||
bash "$ROUTER" "$WORK/s.txt" "$WORK/cloud.json" --role classify --model anthropic:claude-opus-4-8 >/dev/null 2>"$WORK/cloud.err"
|
||||
@@ -76,7 +97,7 @@ else
|
||||
skip "cloud-unavailable test (ANTHROPIC_API_KEY is set)"
|
||||
fi
|
||||
|
||||
# 7: deliberate failing primary route -> fallback through the REAL router (not exit 9).
|
||||
# 8: deliberate failing primary route -> fallback through the REAL router (not exit 9).
|
||||
if [[ "$TUNNEL_UP" -eq 1 ]]; then
|
||||
printf 'Return exactly: OK\n' > "$WORK/f.txt"
|
||||
set +e
|
||||
|
||||
Reference in New Issue
Block a user