95 lines
4.4 KiB
Bash
95 lines
4.4 KiB
Bash
#!/usr/bin/env bash
|
|
set -uo pipefail
|
|
|
|
# CASAN Plan-15 — Responsible AI & Data Governance guard (harness core) tests.
|
|
# Deterministic. Proves data classification, PII→cloud denial without approval,
|
|
# and model-card enforcement (uncarded/incomplete cards blocked).
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
|
RAI="$PROJECT_ROOT/.specify/scripts/bash/rai-guard.py"
|
|
WORK="$(mktemp -d)"
|
|
trap 'rm -rf "$WORK"' EXIT
|
|
|
|
PASS=0; FAIL=0
|
|
pass() { echo "PASS: $1"; PASS=$((PASS + 1)); }
|
|
fail() { echo "FAIL: $1"; FAIL=$((FAIL + 1)); }
|
|
|
|
echo "===== Plan-15 Responsible AI & Data Governance (harness core) ====="
|
|
|
|
# 1) classify detects PII (email)
|
|
printf 'contact user at alice@example.com for details\n' > "$WORK/pii.txt"
|
|
python3 "$RAI" classify --input "$WORK/pii.txt" 2>/dev/null | grep -q "label=PII" \
|
|
&& pass "classify detects PII (email)" || fail "PII not classified"
|
|
|
|
# 2) classify detects confidential marker
|
|
printf 'This document is CONFIDENTIAL and internal.\n' > "$WORK/conf.txt"
|
|
python3 "$RAI" classify --input "$WORK/conf.txt" 2>/dev/null | grep -q "label=confidential" \
|
|
&& pass "classify detects confidential marker" || fail "confidential not classified"
|
|
|
|
# 3) benign classify → internal
|
|
printf 'refactor the dashboard grid layout\n' > "$WORK/benign.txt"
|
|
python3 "$RAI" classify --input "$WORK/benign.txt" 2>/dev/null | grep -q "label=internal" \
|
|
&& pass "benign text classified internal" || fail "benign misclassified"
|
|
|
|
# 4) PII → cloud without approval => DENY (fail-able)
|
|
set +e
|
|
python3 "$RAI" check-cloud --input "$WORK/pii.txt" --target cloud >/dev/null 2>"$WORK/4.err"; RC=$?
|
|
set -e 2>/dev/null || true
|
|
[[ "$RC" -eq 1 ]] && grep -q "DATA_TO_CLOUD" "$WORK/4.err" \
|
|
&& pass "PII→cloud without approval denied" || fail "PII→cloud not denied (rc=$RC)"
|
|
|
|
# 5) PII → cloud WITH approval => allow; benign → cloud => allow
|
|
python3 "$RAI" check-cloud --input "$WORK/pii.txt" --target cloud --approval jwt >/dev/null 2>&1 \
|
|
&& pass "PII→cloud with approval allowed" || fail "approved PII→cloud denied"
|
|
python3 "$RAI" check-cloud --input "$WORK/benign.txt" --target cloud >/dev/null 2>&1 \
|
|
&& pass "benign→cloud allowed" || fail "benign→cloud denied"
|
|
|
|
# 6) model-card enforcement
|
|
cat > "$WORK/cards.json" <<'JSON'
|
|
{
|
|
"ollama:ornith:9b": {"source": "local-ollama", "digest": "sha256:abc", "role": "generate", "risks": "hallucination"}
|
|
}
|
|
JSON
|
|
python3 "$RAI" model-card --model "ollama:ornith:9b" --cards "$WORK/cards.json" >/dev/null 2>&1 \
|
|
&& pass "carded model allowed" || fail "carded model denied"
|
|
set +e
|
|
python3 "$RAI" model-card --model "openai:gpt-4o" --cards "$WORK/cards.json" >/dev/null 2>"$WORK/6.err"; RC=$?
|
|
set -e 2>/dev/null || true
|
|
[[ "$RC" -eq 1 ]] && grep -q "MODEL_UNCARDED" "$WORK/6.err" \
|
|
&& pass "uncarded model blocked (fail-able)" || fail "uncarded model not blocked (rc=$RC)"
|
|
|
|
# 7) incomplete card blocked
|
|
cat > "$WORK/cards2.json" <<'JSON'
|
|
{ "openai:gpt-4o": {"source": "openai", "role": "judge"} }
|
|
JSON
|
|
set +e
|
|
python3 "$RAI" model-card --model "openai:gpt-4o" --cards "$WORK/cards2.json" >/dev/null 2>"$WORK/7.err"; RC=$?
|
|
set -e 2>/dev/null || true
|
|
[[ "$RC" -eq 1 ]] && grep -q "MODEL_CARD_INCOMPLETE" "$WORK/7.err" \
|
|
&& pass "incomplete model card blocked" || fail "incomplete card not blocked (rc=$RC)"
|
|
|
|
# 8) RAI aggregate report: sensitivity distribution over a set
|
|
cat > "$WORK/items.jsonl" <<'JSON'
|
|
{"id":"a","text":"email me at bob@example.com","created_epoch":100}
|
|
{"id":"b","text":"refactor the grid layout","created_epoch":100}
|
|
JSON
|
|
python3 "$RAI" report --items "$WORK/items.jsonl" 2>/dev/null | grep -q '"PII": 1' \
|
|
&& pass "RAI report aggregates sensitivity distribution" || fail "RAI report distribution wrong"
|
|
|
|
# 9) retention: expired items un-purged => gate fails (fail-able)
|
|
set +e
|
|
python3 "$RAI" retention --items "$WORK/items.jsonl" --days 1 --now 1000000 --gate >/dev/null 2>"$WORK/9.err"; RC=$?
|
|
set -e 2>/dev/null || true
|
|
[[ "$RC" -eq 1 ]] && grep -q "RETENTION_BREACH" "$WORK/9.err" \
|
|
&& pass "expired items un-purged ⇒ retention gate fails" || fail "retention breach not caught (rc=$RC)"
|
|
|
|
# 10) purge writes an audit record and passes the gate
|
|
python3 "$RAI" retention --items "$WORK/items.jsonl" --days 1 --now 1000000 --purge --audit "$WORK/ret-audit.jsonl" --gate >/dev/null 2>&1 \
|
|
&& [[ -s "$WORK/ret-audit.jsonl" ]] \
|
|
&& pass "purge records audit and passes gate" || fail "purge/audit failed"
|
|
|
|
echo ""
|
|
echo "===== RAI SUMMARY: PASS=$PASS FAIL=$FAIL ====="
|
|
[[ "$FAIL" -eq 0 ]] || exit 1
|