thanhnvandClaude Opus 4.8
664bd1f00c
Physically move the pure-code subtrees out of .specify into the package, leaving compat symlinks at the old .specify/<dir> paths so every existing reference (internal CASAN_HARNESS_ROOT + external CI/docker/mjs) keeps resolving. Runtime state stays put. Moved (git mv): scripts/ tests/ security/ templates/ config/ governance/ memory/ .specify/<dir> -> packages/casan-harness/<dir> (+ .specify/<dir> symlink) Stays in .specify (state/governance/domain, handled later): logs/ agentops/ level5/ init-options.json traceability-map.json Python `.resolve()` self-location followed the compat symlink into packages and lost the app root; generate-casan-demo-context.py, generate-agentops-dashboard.py and dashboard-server.py now walk UP for the `.specify` state marker instead of a fixed parent depth (fixes "missing trace files" in run-casan4). Full gate: PASS=64 FAIL=0 SKIP=3 (CASAN_CI_STEP_TIMEOUT_SEC=1200 — track-a ~450s runs close to the 600s default and can tip over under load; this is timing variance, not a regression — it passed cleanly with headroom). Runtime log/audit artifacts kept unstaged. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
96 lines
4.5 KiB
Bash
96 lines
4.5 KiB
Bash
#!/usr/bin/env bash
|
|
set -uo pipefail
|
|
|
|
# CASAN Plan-15 — Responsible AI & Data Governance guard (harness core) tests.
|
|
# Deterministic. Proves data classification, PII→cloud denial without approval,
|
|
# and model-card enforcement (uncarded/incomplete cards blocked).
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
source "$SCRIPT_DIR/../scripts/bash/casan-paths.sh"
|
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
|
RAI="$CASAN_HARNESS_ROOT/scripts/bash/rai-guard.py"
|
|
WORK="$(mktemp -d)"
|
|
trap 'rm -rf "$WORK"' EXIT
|
|
|
|
PASS=0; FAIL=0
|
|
pass() { echo "PASS: $1"; PASS=$((PASS + 1)); }
|
|
fail() { echo "FAIL: $1"; FAIL=$((FAIL + 1)); }
|
|
|
|
echo "===== Plan-15 Responsible AI & Data Governance (harness core) ====="
|
|
|
|
# 1) classify detects PII (email)
|
|
printf 'contact user at alice@example.com for details\n' > "$WORK/pii.txt"
|
|
python3 "$RAI" classify --input "$WORK/pii.txt" 2>/dev/null | grep -q "label=PII" \
|
|
&& pass "classify detects PII (email)" || fail "PII not classified"
|
|
|
|
# 2) classify detects confidential marker
|
|
printf 'This document is CONFIDENTIAL and internal.\n' > "$WORK/conf.txt"
|
|
python3 "$RAI" classify --input "$WORK/conf.txt" 2>/dev/null | grep -q "label=confidential" \
|
|
&& pass "classify detects confidential marker" || fail "confidential not classified"
|
|
|
|
# 3) benign classify → internal
|
|
printf 'refactor the dashboard grid layout\n' > "$WORK/benign.txt"
|
|
python3 "$RAI" classify --input "$WORK/benign.txt" 2>/dev/null | grep -q "label=internal" \
|
|
&& pass "benign text classified internal" || fail "benign misclassified"
|
|
|
|
# 4) PII → cloud without approval => DENY (fail-able)
|
|
set +e
|
|
python3 "$RAI" check-cloud --input "$WORK/pii.txt" --target cloud >/dev/null 2>"$WORK/4.err"; RC=$?
|
|
set -e 2>/dev/null || true
|
|
[[ "$RC" -eq 1 ]] && grep -q "DATA_TO_CLOUD" "$WORK/4.err" \
|
|
&& pass "PII→cloud without approval denied" || fail "PII→cloud not denied (rc=$RC)"
|
|
|
|
# 5) PII → cloud WITH approval => allow; benign → cloud => allow
|
|
python3 "$RAI" check-cloud --input "$WORK/pii.txt" --target cloud --approval jwt >/dev/null 2>&1 \
|
|
&& pass "PII→cloud with approval allowed" || fail "approved PII→cloud denied"
|
|
python3 "$RAI" check-cloud --input "$WORK/benign.txt" --target cloud >/dev/null 2>&1 \
|
|
&& pass "benign→cloud allowed" || fail "benign→cloud denied"
|
|
|
|
# 6) model-card enforcement
|
|
cat > "$WORK/cards.json" <<'JSON'
|
|
{
|
|
"ollama:ornith:9b": {"source": "local-ollama", "digest": "sha256:abc", "role": "generate", "risks": "hallucination"}
|
|
}
|
|
JSON
|
|
python3 "$RAI" model-card --model "ollama:ornith:9b" --cards "$WORK/cards.json" >/dev/null 2>&1 \
|
|
&& pass "carded model allowed" || fail "carded model denied"
|
|
set +e
|
|
python3 "$RAI" model-card --model "openai:gpt-4o" --cards "$WORK/cards.json" >/dev/null 2>"$WORK/6.err"; RC=$?
|
|
set -e 2>/dev/null || true
|
|
[[ "$RC" -eq 1 ]] && grep -q "MODEL_UNCARDED" "$WORK/6.err" \
|
|
&& pass "uncarded model blocked (fail-able)" || fail "uncarded model not blocked (rc=$RC)"
|
|
|
|
# 7) incomplete card blocked
|
|
cat > "$WORK/cards2.json" <<'JSON'
|
|
{ "openai:gpt-4o": {"source": "openai", "role": "judge"} }
|
|
JSON
|
|
set +e
|
|
python3 "$RAI" model-card --model "openai:gpt-4o" --cards "$WORK/cards2.json" >/dev/null 2>"$WORK/7.err"; RC=$?
|
|
set -e 2>/dev/null || true
|
|
[[ "$RC" -eq 1 ]] && grep -q "MODEL_CARD_INCOMPLETE" "$WORK/7.err" \
|
|
&& pass "incomplete model card blocked" || fail "incomplete card not blocked (rc=$RC)"
|
|
|
|
# 8) RAI aggregate report: sensitivity distribution over a set
|
|
cat > "$WORK/items.jsonl" <<'JSON'
|
|
{"id":"a","text":"email me at bob@example.com","created_epoch":100}
|
|
{"id":"b","text":"refactor the grid layout","created_epoch":100}
|
|
JSON
|
|
python3 "$RAI" report --items "$WORK/items.jsonl" 2>/dev/null | grep -q '"PII": 1' \
|
|
&& pass "RAI report aggregates sensitivity distribution" || fail "RAI report distribution wrong"
|
|
|
|
# 9) retention: expired items un-purged => gate fails (fail-able)
|
|
set +e
|
|
python3 "$RAI" retention --items "$WORK/items.jsonl" --days 1 --now 1000000 --gate >/dev/null 2>"$WORK/9.err"; RC=$?
|
|
set -e 2>/dev/null || true
|
|
[[ "$RC" -eq 1 ]] && grep -q "RETENTION_BREACH" "$WORK/9.err" \
|
|
&& pass "expired items un-purged ⇒ retention gate fails" || fail "retention breach not caught (rc=$RC)"
|
|
|
|
# 10) purge writes an audit record and passes the gate
|
|
python3 "$RAI" retention --items "$WORK/items.jsonl" --days 1 --now 1000000 --purge --audit "$WORK/ret-audit.jsonl" --gate >/dev/null 2>&1 \
|
|
&& [[ -s "$WORK/ret-audit.jsonl" ]] \
|
|
&& pass "purge records audit and passes gate" || fail "purge/audit failed"
|
|
|
|
echo ""
|
|
echo "===== RAI SUMMARY: PASS=$PASS FAIL=$FAIL ====="
|
|
[[ "$FAIL" -eq 0 ]] || exit 1
|