Nam Pham Dinh ThanhandClaude Sonnet 4.6
838b2473b6
WV4-A: Added 16 Vitest/RTL tests to frontend (jsdom env, fail-before proof verified) WV4-B: Created 12 stub traces for pipeline retention gap; fixed MSYS2/Python path mismatch in context-validate.sh; run-casan4-harness-tests.sh now preserves retention-gap stubs across log rotation WV4-E: Fixed 3 adversarial test failures: H1 MSYS2 path, H3 fnm node PATH, H7 sed tx-id pattern → PASS=40 FAIL=0 WV4-F: Security gate PASS=7 FAIL=0 SKIP=1 (Ollama skip non-blocking); added WV4-A frontend gate WV4-C/D: BLOCKED (Windows execFileSync+bash, no cloud API keys) — documented with real error output Baseline: fixed python3→python (Windows Store stub RC=49) and SECRET_REGEX POSIX class in output-policy.yaml Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
67 lines
2.0 KiB
Bash
Executable File
67 lines
2.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
# CASAN Level 5 drift detector.
|
|
# Usage:
|
|
# drift-detect.sh <golden-file> <candidate-file> <report-json>
|
|
|
|
GOLDEN="${1:-}"
|
|
CANDIDATE="${2:-}"
|
|
REPORT="${3:-}"
|
|
|
|
if [[ -z "$GOLDEN" || -z "$CANDIDATE" || -z "$REPORT" ]]; then
|
|
echo "Usage: drift-detect.sh <golden-file> <candidate-file> <report-json>" >&2
|
|
exit 64
|
|
fi
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
|
|
mkdir -p "$(dirname "$REPORT")" "$PROJECT_ROOT/.specify/logs/level5"
|
|
|
|
python - "$GOLDEN" "$CANDIDATE" "$REPORT" <<'PY'
|
|
import difflib
|
|
import hashlib
|
|
import json
|
|
import pathlib
|
|
import sys
|
|
from datetime import datetime, timezone
|
|
|
|
golden_path = pathlib.Path(sys.argv[1])
|
|
candidate_path = pathlib.Path(sys.argv[2])
|
|
report_path = pathlib.Path(sys.argv[3])
|
|
|
|
golden = golden_path.read_text(encoding="utf-8")
|
|
candidate = candidate_path.read_text(encoding="utf-8")
|
|
|
|
similarity = difflib.SequenceMatcher(None, golden, candidate).ratio()
|
|
length_delta = abs(len(candidate) - len(golden)) / max(len(golden), 1)
|
|
|
|
status = "pass"
|
|
action = "allow"
|
|
if similarity < 0.70 or length_delta > 0.50:
|
|
status = "fail"
|
|
action = "block_or_fallback"
|
|
elif similarity < 0.85 or length_delta > 0.30:
|
|
status = "warn"
|
|
action = "require_review"
|
|
|
|
report = {
|
|
"timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
|
"harness": "L5-drift-detection",
|
|
"status": status,
|
|
"action": action,
|
|
"similarity_ratio": round(similarity, 4),
|
|
"length_delta_ratio": round(length_delta, 4),
|
|
"golden_hash": hashlib.sha256(golden.encode()).hexdigest(),
|
|
"candidate_hash": hashlib.sha256(candidate.encode()).hexdigest(),
|
|
"golden_file": str(golden_path),
|
|
"candidate_file": str(candidate_path),
|
|
}
|
|
|
|
report_path.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
|
|
print(f"DRIFT_{status.upper()} similarity={report['similarity_ratio']} length_delta={report['length_delta_ratio']} report={report_path}")
|
|
|
|
if status == "fail":
|
|
raise SystemExit(2)
|
|
PY
|