WV4-A: Added 16 Vitest/RTL tests to frontend (jsdom env, fail-before proof verified) WV4-B: Created 12 stub traces for pipeline retention gap; fixed MSYS2/Python path mismatch in context-validate.sh; run-casan4-harness-tests.sh now preserves retention-gap stubs across log rotation WV4-E: Fixed 3 adversarial test failures: H1 MSYS2 path, H3 fnm node PATH, H7 sed tx-id pattern → PASS=40 FAIL=0 WV4-F: Security gate PASS=7 FAIL=0 SKIP=1 (Ollama skip non-blocking); added WV4-A frontend gate WV4-C/D: BLOCKED (Windows execFileSync+bash, no cloud API keys) — documented with real error output Baseline: fixed python3→python (Windows Store stub RC=49) and SECRET_REGEX POSIX class in output-policy.yaml Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
295 lines
10 KiB
Bash
Executable File
295 lines
10 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
# CASAN H4 Security Harness
|
|
# Usage:
|
|
# security-check.sh <input-file> <output-file> [input|output]
|
|
#
|
|
# input mode: blocks prompt injection / critical secrets, masks PII, writes safe prompt.
|
|
# output mode: redacts PII/secrets from generated output, flags risky language, writes safe output.
|
|
|
|
INPUT_FILE="${1:-}"
|
|
OUTPUT_FILE="${2:-}"
|
|
MODE="${3:-input}"
|
|
|
|
if [[ -z "$INPUT_FILE" || -z "$OUTPUT_FILE" ]]; then
|
|
echo "Usage: security-check.sh <input-file> <output-file> [input|output]" >&2
|
|
exit 64
|
|
fi
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
|
|
LOG_DIR="$PROJECT_ROOT/.specify/logs"
|
|
TRACE_DIR="$LOG_DIR/trace"
|
|
AUDIT_DIR="$LOG_DIR/audit"
|
|
SECURITY_DIR="$PROJECT_ROOT/.specify/security"
|
|
|
|
mkdir -p "$TRACE_DIR" "$AUDIT_DIR" "$(dirname "$OUTPUT_FILE")"
|
|
|
|
if [[ ! -f "$INPUT_FILE" ]]; then
|
|
echo "SECURITY_BLOCKED: input file not found: $INPUT_FILE" >&2
|
|
exit 1
|
|
fi
|
|
|
|
timestamp() {
|
|
date -u +"%Y-%m-%dT%H:%M:%SZ"
|
|
}
|
|
|
|
new_trace_id() {
|
|
if command -v uuidgen >/dev/null 2>&1; then
|
|
uuidgen | tr '[:upper:]' '[:lower:]'
|
|
else
|
|
printf 'trace-%s-%s\n' "$(date +%s)" "$$"
|
|
fi
|
|
}
|
|
|
|
json_escape() {
|
|
python -c 'import json,sys; print(json.dumps(sys.stdin.read()))' 2>/dev/null || sed 's/\\/\\\\/g; s/"/\\"/g'
|
|
}
|
|
|
|
hash_text() {
|
|
if command -v sha256sum >/dev/null 2>&1; then
|
|
sha256sum | awk '{print $1}'
|
|
else
|
|
shasum -a 256 | awk '{print $1}'
|
|
fi
|
|
}
|
|
|
|
# Normalize text to defeat trivial injection bypasses:
|
|
# lowercase, fold common leetspeak to letters, collapse punctuation/whitespace.
|
|
# Used ONLY for injection/jailbreak phrase matching, never for PII/secret regexes.
|
|
normalize_for_match() {
|
|
printf '%s' "$1" \
|
|
| tr '[:upper:]' '[:lower:]' \
|
|
| tr '013457@$' 'oieastas' \
|
|
| tr -c 'a-z0-9' ' ' \
|
|
| tr -s ' '
|
|
}
|
|
|
|
load_yaml_values() {
|
|
local file="$1"
|
|
local key="$2"
|
|
[[ -f "$file" ]] || return 0
|
|
python - "$file" "$key" <<'PY'
|
|
import re
|
|
import sys
|
|
path, key = sys.argv[1], sys.argv[2]
|
|
pattern = re.compile(rf'^\s*{re.escape(key)}:\s*"([^"]+)"\s*$')
|
|
with open(path, encoding="utf-8") as f:
|
|
for line in f:
|
|
match = pattern.search(line)
|
|
if match:
|
|
print(match.group(1))
|
|
PY
|
|
}
|
|
|
|
TRACE_ID="$(new_trace_id)"
|
|
TIMESTAMP="$(timestamp)"
|
|
CONTENT="$(cat "$INPUT_FILE")"
|
|
STATUS="pass"
|
|
ACTION="allow"
|
|
RISK_LEVEL="low"
|
|
MATCHED_RULES=()
|
|
|
|
BLOCK_PATTERNS=(
|
|
"ignore (all |any )?(previous|prior|above|the above|earlier) (instruction|instructions|rule|rules|prompt|prompts|guideline|guidelines)"
|
|
"ignore system instruction"
|
|
"disregard (all |any |the )?.*(rule|instruction|previous|prior|above|earlier|policy|guideline)"
|
|
"forget (all |everything |the |your )?.*(instruction|rule|previous|prior|above)"
|
|
"bypass (the )?(system|policy|policies|filter|guardrail|guardrails|rule|rules|safety|restriction|restrictions)"
|
|
"override (the )?(policy|policies|rule|rules|system|instruction|instructions|guardrail|safety)"
|
|
"you are now"
|
|
"act as (an? )?(admin|root|developer|unrestricted|unfiltered|different|evil|dan)"
|
|
"act without (any )?(restriction|restrictions|rule|rules|limit|limits|filter|guardrail)"
|
|
"pretend (to be|you are|you're) (system|not|no longer|an unrestricted)"
|
|
"do anything now"
|
|
"enable (dan|developer mode|jailbreak)"
|
|
"developer mode"
|
|
"reveal (the )?(system|hidden|internal|original) (prompt|instruction|instructions)"
|
|
"show (the )?(hidden|internal|system) (instruction|instructions|prompt)"
|
|
)
|
|
|
|
while IFS= read -r pattern; do
|
|
[[ -n "$pattern" ]] && BLOCK_PATTERNS+=("$pattern")
|
|
done < <(load_yaml_values "$SECURITY_DIR/prompt-filter.yaml" "pattern")
|
|
|
|
APPROVAL_PATTERNS=(
|
|
"delete[[:space:]].*"
|
|
"drop table"
|
|
"shutdown system"
|
|
"export secrets"
|
|
"dump database"
|
|
)
|
|
|
|
ALERT_PATTERNS=(
|
|
"show all data"
|
|
"internal prompt"
|
|
"system message"
|
|
"hidden instruction"
|
|
)
|
|
|
|
EMAIL_REGEX='[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}'
|
|
PHONE_REGEX='(\+?[0-9][0-9 .-]{8,}[0-9])'
|
|
PERSONAL_ID_REGEX='\b[0-9]{9,12}\b'
|
|
CREDIT_CARD_REGEX='\b([0-9]{4}[- ]?){3}[0-9]{4}\b'
|
|
SECRET_REGEX='(API[_-]?KEY|ACCESS[_-]?TOKEN|REFRESH[_-]?TOKEN|PASSWORD|JWT[_-]?SECRET|SECRET)[[:space:]]*[:=][[:space:]]*[^[:space:]]+'
|
|
PRIVATE_KEY_REGEX='-----BEGIN (RSA |EC )?PRIVATE KEY-----'
|
|
DB_CONN_REGEX='(postgres|mysql|mongodb)://[^@[:space:]]+@'
|
|
AWS_KEY_REGEX='AKIA[0-9A-Z]{16}'
|
|
|
|
while IFS= read -r regex; do
|
|
case "$regex" in
|
|
*API*|*TOKEN*|*PASSWORD*|*SECRET*)
|
|
regex="${regex//\\s/[[:space:]]}"
|
|
regex="${regex//\\S/[^[:space:]]}"
|
|
SECRET_REGEX="$regex"
|
|
;;
|
|
esac
|
|
done < <(load_yaml_values "$SECURITY_DIR/output-policy.yaml" "regex")
|
|
|
|
lower_content="$(printf '%s' "$CONTENT" | tr '[:upper:]' '[:lower:]')"
|
|
NORM_CONTENT="$(normalize_for_match "$CONTENT")"
|
|
|
|
# Matches a pattern against either the raw (case-insensitive) or the
|
|
# normalization-folded content, so leetspeak/whitespace/punctuation
|
|
# obfuscation cannot slip past a phrase blocklist.
|
|
match_either() {
|
|
local pattern="$1"
|
|
printf '%s' "$CONTENT" | grep -Eiq -- "$pattern" \
|
|
|| printf '%s' "$NORM_CONTENT" | grep -Eq -- "$pattern"
|
|
}
|
|
|
|
if [[ "$MODE" == "input" ]]; then
|
|
for pattern in "${BLOCK_PATTERNS[@]}"; do
|
|
if match_either "$pattern"; then
|
|
STATUS="blocked"
|
|
ACTION="block"
|
|
RISK_LEVEL="high"
|
|
MATCHED_RULES+=("prompt-injection:$pattern")
|
|
fi
|
|
done
|
|
|
|
if printf '%s' "$CONTENT" | grep -Eq -- "$CREDIT_CARD_REGEX"; then
|
|
STATUS="blocked"
|
|
ACTION="block"
|
|
RISK_LEVEL="high"
|
|
MATCHED_RULES+=("pii-credit-card")
|
|
fi
|
|
|
|
if printf '%s' "$CONTENT" | grep -Eiq -- "$SECRET_REGEX" \
|
|
|| printf '%s' "$CONTENT" | grep -Eiq -- "$PRIVATE_KEY_REGEX" \
|
|
|| printf '%s' "$CONTENT" | grep -Eiq -- "$DB_CONN_REGEX" \
|
|
|| printf '%s' "$CONTENT" | grep -Eiq -- "$AWS_KEY_REGEX"; then
|
|
STATUS="blocked"
|
|
ACTION="block"
|
|
RISK_LEVEL="high"
|
|
MATCHED_RULES+=("secret-in-input")
|
|
fi
|
|
|
|
if [[ "$STATUS" != "blocked" ]]; then
|
|
for pattern in "${APPROVAL_PATTERNS[@]}"; do
|
|
if match_either "$pattern"; then
|
|
STATUS="requires_approval"
|
|
ACTION="require_approval"
|
|
RISK_LEVEL="high"
|
|
MATCHED_RULES+=("unsafe-action:$pattern")
|
|
fi
|
|
done
|
|
fi
|
|
|
|
if [[ "$STATUS" != "blocked" ]]; then
|
|
for pattern in "${ALERT_PATTERNS[@]}"; do
|
|
if match_either "$pattern"; then
|
|
[[ "$RISK_LEVEL" == "low" ]] && RISK_LEVEL="medium"
|
|
ACTION="alert"
|
|
MATCHED_RULES+=("suspicious:$pattern")
|
|
fi
|
|
done
|
|
fi
|
|
|
|
# Optional semantic escalation (opt-in: CASAN_SEMANTIC_CLASSIFY=1). The regex
|
|
# layer above catches known phrasings; a genuinely novel paraphrase slips
|
|
# through as low-risk. When enabled, route still-allowed input to the model
|
|
# classifier. It can only ADD a block, never remove one. If the model backend
|
|
# is unreachable, record it and keep the regex verdict (no silent pass of a
|
|
# blocked item; no hard pipeline failure on infra outage).
|
|
if [[ "$STATUS" != "blocked" && "${CASAN_SEMANTIC_CLASSIFY:-0}" == "1" && -x "$SCRIPT_DIR/model-router.sh" ]]; then
|
|
SEM_JSON="$TRACE_DIR/semantic-$TRACE_ID.json"
|
|
"$SCRIPT_DIR/model-router.sh" "$INPUT_FILE" "$SEM_JSON" --role classify >/dev/null 2>&1 || true
|
|
if [[ -f "$SEM_JSON" ]]; then
|
|
SEM_VERDICT="$(python -c "import json;print(json.load(open('$SEM_JSON')).get('verdict',''))" 2>/dev/null || echo "")"
|
|
if [[ "$SEM_VERDICT" == "INJECTION" ]]; then
|
|
STATUS="blocked"; ACTION="block"; RISK_LEVEL="high"
|
|
MATCHED_RULES+=("semantic-injection")
|
|
fi
|
|
else
|
|
MATCHED_RULES+=("semantic-unavailable")
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
SAFE_CONTENT="$CONTENT"
|
|
# Policy-driven PII masking (source of truth: pii-rules.yaml). Built-in sed
|
|
# masking below remains as defense-in-depth if the policy file is unavailable.
|
|
if [[ -f "$SECURITY_DIR/pii-rules.yaml" ]] && command -v python >/dev/null 2>&1; then
|
|
SAFE_CONTENT="$(printf '%s' "$SAFE_CONTENT" | python "$SCRIPT_DIR/pii-mask.py" "$SECURITY_DIR/pii-rules.yaml")"
|
|
fi
|
|
SAFE_CONTENT="$(printf '%s' "$SAFE_CONTENT" | sed -E "s/$EMAIL_REGEX/***MASKED_EMAIL***/g")"
|
|
SAFE_CONTENT="$(printf '%s' "$SAFE_CONTENT" | sed -E "s/$PHONE_REGEX/***MASKED_PHONE***/g")"
|
|
SAFE_CONTENT="$(printf '%s' "$SAFE_CONTENT" | sed -E "s/$PERSONAL_ID_REGEX/***MASKED_ID***/g")"
|
|
SAFE_CONTENT="$(printf '%s' "$SAFE_CONTENT" | sed -E "s/$SECRET_REGEX/[REDACTED_SECRET]/Ig")"
|
|
SAFE_CONTENT="$(printf '%s' "$SAFE_CONTENT" | sed -E "s/$PRIVATE_KEY_REGEX/[REDACTED_PRIVATE_KEY]/Ig")"
|
|
SAFE_CONTENT="$(printf '%s' "$SAFE_CONTENT" | sed -E "s#$DB_CONN_REGEX#[REDACTED_CONNSTRING]://#Ig")"
|
|
SAFE_CONTENT="$(printf '%s' "$SAFE_CONTENT" | sed -E "s/$AWS_KEY_REGEX/[REDACTED_AWS_KEY]/Ig")"
|
|
|
|
if [[ "$MODE" == "output" ]]; then
|
|
if printf '%s' "$CONTENT" | grep -Eiq -- "$SECRET_REGEX" \
|
|
|| printf '%s' "$CONTENT" | grep -Eiq -- "$PRIVATE_KEY_REGEX" \
|
|
|| printf '%s' "$CONTENT" | grep -Eiq -- "$DB_CONN_REGEX" \
|
|
|| printf '%s' "$CONTENT" | grep -Eiq -- "$AWS_KEY_REGEX"; then
|
|
# output-policy.yaml level4_gate.fail_on: unredacted_secret -> fail closed.
|
|
STATUS="blocked"
|
|
ACTION="block"
|
|
RISK_LEVEL="high"
|
|
MATCHED_RULES+=("secret-in-output")
|
|
fi
|
|
if printf '%s' "$lower_content" | grep -Eq -- "(maybe|might be incorrect|i am not sure|uncertain)"; then
|
|
ACTION="flag"
|
|
[[ "$RISK_LEVEL" == "low" ]] && RISK_LEVEL="medium"
|
|
MATCHED_RULES+=("hallucination-risk-language")
|
|
fi
|
|
fi
|
|
|
|
INPUT_HASH="$(printf '%s' "$CONTENT" | hash_text)"
|
|
OUTPUT_HASH="$(printf '%s' "$SAFE_CONTENT" | hash_text)"
|
|
RULES_JSON="$(printf '%s\n' "${MATCHED_RULES[@]:-}" | python -c 'import json,sys; print(json.dumps([x for x in sys.stdin.read().splitlines() if x]))')"
|
|
|
|
TRACE_FILE="$TRACE_DIR/security-$TRACE_ID.json"
|
|
cat > "$TRACE_FILE" <<EOF
|
|
{
|
|
"trace_id": "$TRACE_ID",
|
|
"timestamp": "$TIMESTAMP",
|
|
"harness": "H4-security",
|
|
"mode": "$MODE",
|
|
"status": "$STATUS",
|
|
"action": "$ACTION",
|
|
"risk_level": "$RISK_LEVEL",
|
|
"matched_rules": $RULES_JSON,
|
|
"input_hash": "$INPUT_HASH",
|
|
"output_hash": "$OUTPUT_HASH"
|
|
}
|
|
EOF
|
|
|
|
printf '{"timestamp":"%s","trace_id":"%s","harness":"H4-security","mode":"%s","status":"%s","action":"%s","risk_level":"%s","input_hash":"%s","output_hash":"%s"}\n' \
|
|
"$TIMESTAMP" "$TRACE_ID" "$MODE" "$STATUS" "$ACTION" "$RISK_LEVEL" "$INPUT_HASH" "$OUTPUT_HASH" >> "$AUDIT_DIR/security.jsonl"
|
|
|
|
if [[ "$STATUS" == "blocked" ]]; then
|
|
: > "$OUTPUT_FILE"
|
|
echo "SECURITY_BLOCKED trace_id=$TRACE_ID risk=$RISK_LEVEL rules=$RULES_JSON" >&2
|
|
exit 2
|
|
fi
|
|
|
|
printf '%s\n' "$SAFE_CONTENT" > "$OUTPUT_FILE"
|
|
STATUS_UPPER="$(printf '%s' "$STATUS" | tr '[:lower:]' '[:upper:]')"
|
|
echo "SECURITY_${STATUS_UPPER} trace_id=$TRACE_ID risk=$RISK_LEVEL action=$ACTION output=$OUTPUT_FILE"
|