thanhnvandClaude Opus 4.8
3adc48ed82
Resolve every root by marker walk-up instead of a fixed depth that only lands on the app via the .specify compat symlink, so the harness runs correctly when invoked by its real packages/casan-harness path — proven by a full gate run via that path: 64/0/0. - 95 scripts/tests: PROJECT_ROOT/ROOT "$SCRIPT_DIR/../.."-style computations -> $CASAN_APP_ROOT. - 6 leaf scripts (infra-lab, context-validate, secrets-scan, path-guard, toolchain-verify, phase2-sourcegen) now source casan-paths + use CASAN_APP_ROOT. - run-casan4: source casan-paths as a package sibling (facade-independent), PROJECT_ROOT=CASAN_APP_ROOT. - 8 Python files: project_root()/REPO_ROOT/bundle_root walk UP for the .specify marker (control-plane-settings, loop_common, model-call, context-compress, test-integrity, bundle-integrity, traceability-matrix; generate-* fixed earlier). - evidence-pack-build.py + traceability-matrix.py: domain refs -> apps/okr/domain (input/, corpus/redteam-vectors.jsonl, traceability-map.json). - ci-harness-gate.sh: export CASAN_TESTS_DIR/CASAN_TEST_MANIFEST/CASAN_BUNDLE_ROOT so the integrity Python resolves via the harness root regardless of invocation path; ROOT=CASAN_APP_ROOT. - Remove the domain compat symlinks from packages/casan-harness/security (redteam-corpus, redteam-vectors, benign-corpus) — packages now holds NO domain data. Both invocation paths pass (compat facade still present): .specify/... and packages/... Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
96 lines
4.5 KiB
Bash
96 lines
4.5 KiB
Bash
#!/usr/bin/env bash
|
|
set -uo pipefail
|
|
|
|
# CASAN Plan-15 — Responsible AI & Data Governance guard (harness core) tests.
|
|
# Deterministic. Proves data classification, PII→cloud denial without approval,
|
|
# and model-card enforcement (uncarded/incomplete cards blocked).
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
source "$SCRIPT_DIR/../scripts/bash/casan-paths.sh"
|
|
PROJECT_ROOT="$CASAN_APP_ROOT"
|
|
RAI="$CASAN_HARNESS_ROOT/scripts/bash/rai-guard.py"
|
|
WORK="$(mktemp -d)"
|
|
trap 'rm -rf "$WORK"' EXIT
|
|
|
|
PASS=0; FAIL=0
|
|
pass() { echo "PASS: $1"; PASS=$((PASS + 1)); }
|
|
fail() { echo "FAIL: $1"; FAIL=$((FAIL + 1)); }
|
|
|
|
echo "===== Plan-15 Responsible AI & Data Governance (harness core) ====="
|
|
|
|
# 1) classify detects PII (email)
|
|
printf 'contact user at alice@example.com for details\n' > "$WORK/pii.txt"
|
|
python3 "$RAI" classify --input "$WORK/pii.txt" 2>/dev/null | grep -q "label=PII" \
|
|
&& pass "classify detects PII (email)" || fail "PII not classified"
|
|
|
|
# 2) classify detects confidential marker
|
|
printf 'This document is CONFIDENTIAL and internal.\n' > "$WORK/conf.txt"
|
|
python3 "$RAI" classify --input "$WORK/conf.txt" 2>/dev/null | grep -q "label=confidential" \
|
|
&& pass "classify detects confidential marker" || fail "confidential not classified"
|
|
|
|
# 3) benign classify → internal
|
|
printf 'refactor the dashboard grid layout\n' > "$WORK/benign.txt"
|
|
python3 "$RAI" classify --input "$WORK/benign.txt" 2>/dev/null | grep -q "label=internal" \
|
|
&& pass "benign text classified internal" || fail "benign misclassified"
|
|
|
|
# 4) PII → cloud without approval => DENY (fail-able)
|
|
set +e
|
|
python3 "$RAI" check-cloud --input "$WORK/pii.txt" --target cloud >/dev/null 2>"$WORK/4.err"; RC=$?
|
|
set -e 2>/dev/null || true
|
|
[[ "$RC" -eq 1 ]] && grep -q "DATA_TO_CLOUD" "$WORK/4.err" \
|
|
&& pass "PII→cloud without approval denied" || fail "PII→cloud not denied (rc=$RC)"
|
|
|
|
# 5) PII → cloud WITH approval => allow; benign → cloud => allow
|
|
python3 "$RAI" check-cloud --input "$WORK/pii.txt" --target cloud --approval jwt >/dev/null 2>&1 \
|
|
&& pass "PII→cloud with approval allowed" || fail "approved PII→cloud denied"
|
|
python3 "$RAI" check-cloud --input "$WORK/benign.txt" --target cloud >/dev/null 2>&1 \
|
|
&& pass "benign→cloud allowed" || fail "benign→cloud denied"
|
|
|
|
# 6) model-card enforcement
|
|
cat > "$WORK/cards.json" <<'JSON'
|
|
{
|
|
"ollama:ornith:9b": {"source": "local-ollama", "digest": "sha256:abc", "role": "generate", "risks": "hallucination"}
|
|
}
|
|
JSON
|
|
python3 "$RAI" model-card --model "ollama:ornith:9b" --cards "$WORK/cards.json" >/dev/null 2>&1 \
|
|
&& pass "carded model allowed" || fail "carded model denied"
|
|
set +e
|
|
python3 "$RAI" model-card --model "openai:gpt-4o" --cards "$WORK/cards.json" >/dev/null 2>"$WORK/6.err"; RC=$?
|
|
set -e 2>/dev/null || true
|
|
[[ "$RC" -eq 1 ]] && grep -q "MODEL_UNCARDED" "$WORK/6.err" \
|
|
&& pass "uncarded model blocked (fail-able)" || fail "uncarded model not blocked (rc=$RC)"
|
|
|
|
# 7) incomplete card blocked
|
|
cat > "$WORK/cards2.json" <<'JSON'
|
|
{ "openai:gpt-4o": {"source": "openai", "role": "judge"} }
|
|
JSON
|
|
set +e
|
|
python3 "$RAI" model-card --model "openai:gpt-4o" --cards "$WORK/cards2.json" >/dev/null 2>"$WORK/7.err"; RC=$?
|
|
set -e 2>/dev/null || true
|
|
[[ "$RC" -eq 1 ]] && grep -q "MODEL_CARD_INCOMPLETE" "$WORK/7.err" \
|
|
&& pass "incomplete model card blocked" || fail "incomplete card not blocked (rc=$RC)"
|
|
|
|
# 8) RAI aggregate report: sensitivity distribution over a set
|
|
cat > "$WORK/items.jsonl" <<'JSON'
|
|
{"id":"a","text":"email me at bob@example.com","created_epoch":100}
|
|
{"id":"b","text":"refactor the grid layout","created_epoch":100}
|
|
JSON
|
|
python3 "$RAI" report --items "$WORK/items.jsonl" 2>/dev/null | grep -q '"PII": 1' \
|
|
&& pass "RAI report aggregates sensitivity distribution" || fail "RAI report distribution wrong"
|
|
|
|
# 9) retention: expired items un-purged => gate fails (fail-able)
|
|
set +e
|
|
python3 "$RAI" retention --items "$WORK/items.jsonl" --days 1 --now 1000000 --gate >/dev/null 2>"$WORK/9.err"; RC=$?
|
|
set -e 2>/dev/null || true
|
|
[[ "$RC" -eq 1 ]] && grep -q "RETENTION_BREACH" "$WORK/9.err" \
|
|
&& pass "expired items un-purged ⇒ retention gate fails" || fail "retention breach not caught (rc=$RC)"
|
|
|
|
# 10) purge writes an audit record and passes the gate
|
|
python3 "$RAI" retention --items "$WORK/items.jsonl" --days 1 --now 1000000 --purge --audit "$WORK/ret-audit.jsonl" --gate >/dev/null 2>&1 \
|
|
&& [[ -s "$WORK/ret-audit.jsonl" ]] \
|
|
&& pass "purge records audit and passes gate" || fail "purge/audit failed"
|
|
|
|
echo ""
|
|
echo "===== RAI SUMMARY: PASS=$PASS FAIL=$FAIL ====="
|
|
[[ "$FAIL" -eq 0 ]] || exit 1
|