#!/usr/bin/env bash set -euo pipefail # CASAN Level 5 drift detector. # Usage: # drift-detect.sh GOLDEN="${1:-}" CANDIDATE="${2:-}" REPORT="${3:-}" if [[ -z "$GOLDEN" || -z "$CANDIDATE" || -z "$REPORT" ]]; then echo "Usage: drift-detect.sh " >&2 exit 64 fi SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/casan-paths.sh" PROJECT_ROOT="$CASAN_APP_ROOT" mkdir -p "$(dirname "$REPORT")" "$CASAN_STATE_ROOT/logs/level5" python - "$GOLDEN" "$CANDIDATE" "$REPORT" <<'PY' import difflib import hashlib import json import os import pathlib import sys from datetime import datetime, timezone golden_path = pathlib.Path(sys.argv[1]) candidate_path = pathlib.Path(sys.argv[2]) report_path = pathlib.Path(sys.argv[3]) # SEC-09 (M-10): cap input size (SequenceMatcher is O(n^2) → DoS) and read # fail-closed. A missing/oversize/undecodable input yields a BLOCK verdict, never # a crash/traceback (which under set -e would abort ambiguously) and never a silent # pass. MAX_BYTES = int(os.environ.get("CASAN_MAX_INPUT_BYTES", str(2 * 1024 * 1024))) def block(reason): report_path.parent.mkdir(parents=True, exist_ok=True) report = { "timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), "harness": "L5-drift-detection", "status": "fail", "action": "block", "reason": reason, "golden_file": str(golden_path), "candidate_file": str(candidate_path), } report_path.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") print(f"DRIFT_FAIL reason={reason} report={report_path}") raise SystemExit(2) def read_capped(path): try: size = path.stat().st_size except OSError: block(f"unreadable:{path.name}") if size > MAX_BYTES: block(f"oversize:{path.name}({size}>{MAX_BYTES})") try: # errors="replace" so non-UTF8 bytes degrade to a marker instead of crashing. return path.read_text(encoding="utf-8", errors="replace") except OSError: block(f"unreadable:{path.name}") golden = read_capped(golden_path) candidate = read_capped(candidate_path) similarity = difflib.SequenceMatcher(None, golden, candidate).ratio() length_delta = abs(len(candidate) - len(golden)) / max(len(golden), 1) status = "pass" action = "allow" reasons = [] if similarity < 0.70 or length_delta > 0.50: status = "fail" action = "block_or_fallback" reasons.append("low_similarity_or_length_delta") elif similarity < 0.85 or length_delta > 0.30: status = "warn" action = "require_review" # SEC-12: char-similarity alone misses SEMANTIC inversion — dropping a negation # ("must NOT deploy" -> "must deploy") keeps similarity high but flips meaning. A # candidate that removes negation tokens present in the golden is treated as drift. import re as _re NEG = _re.compile( r"\b(not|no|never|cannot|can't|don't|must not|mustn't|deny|denied|reject|disable|" r"disabled|forbid|prohibit|block|blocked|không|đừng|cấm|từ chối)\b", _re.IGNORECASE, ) golden_neg = len(NEG.findall(golden)) cand_neg = len(NEG.findall(candidate)) if golden_neg > cand_neg: # The dangerous case: looks nearly identical but a negation vanished. status, action = "fail", "block_or_fallback" reasons.append(f"negation_dropped(golden={golden_neg},candidate={cand_neg})") # must-keep invariants: regex patterns that MUST still appear in the candidate. _mk = os.environ.get("CASAN_DRIFT_MUSTKEEP_FILE", "") missing = [] if _mk and os.path.isfile(_mk): for _line in open(_mk, encoding="utf-8", errors="replace"): _pat = _line.strip() if _pat and not _re.search(_pat, candidate): missing.append(_pat) if missing: status, action = "fail", "block_or_fallback" reasons.append(f"must_keep_missing={len(missing)}") report = { "timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), "harness": "L5-drift-detection", "status": status, "action": action, "reasons": reasons, "golden_negations": golden_neg, "candidate_negations": cand_neg, "must_keep_missing": missing, "similarity_ratio": round(similarity, 4), "length_delta_ratio": round(length_delta, 4), "golden_hash": hashlib.sha256(golden.encode()).hexdigest(), "candidate_hash": hashlib.sha256(candidate.encode()).hexdigest(), "golden_file": str(golden_path), "candidate_file": str(candidate_path), } report_path.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") print(f"DRIFT_{status.upper()} similarity={report['similarity_ratio']} length_delta={report['length_delta_ratio']} report={report_path}") if status == "fail": raise SystemExit(2) PY