QA and context compress
This commit is contained in:
thanhnv
+8
@@ -0,0 +1,8 @@
|
||||
# NEHOPS analyzer invariants. These expressions are project/domain policy,
|
||||
# while generic severity-tag preservation belongs to the Core compressor.
|
||||
VBSource root not found
|
||||
common code tracing skipped
|
||||
Error Messages
|
||||
Output Directory
|
||||
Generated [0-9]+ files
|
||||
exit code
|
||||
@@ -10,6 +10,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$SCRIPT_DIR/../scripts/bash/casan-paths.sh"
|
||||
PROJECT_ROOT="$CASAN_APP_ROOT"
|
||||
CC="$CASAN_HARNESS_ROOT/scripts/bash/context-compress.py"
|
||||
FIXTURES="$SCRIPT_DIR/fixtures/context-compression/nehops"
|
||||
WORK="$(mktemp -d)"
|
||||
trap 'rm -rf "$WORK"' EXIT
|
||||
|
||||
@@ -89,6 +90,178 @@ set -e 2>/dev/null || true
|
||||
[[ "$RC" -eq 0 ]] && pass "protecting the pattern keeps the line and passes the gate" \
|
||||
|| fail "protected must-keep still failed the gate (rc=$RC)"
|
||||
|
||||
# 8) Realistic NEHOPS regression: abbreviated [WRN] tags and domain invariants
|
||||
# survive structural compression, with hash-bound evidence that labels token
|
||||
# figures as whitespace estimates.
|
||||
set +e
|
||||
python3 "$CC" --mode structural \
|
||||
--input "$FIXTURES/analyzer-sanitized.log" \
|
||||
--must-keep-file "$FIXTURES/must-keep.patterns" \
|
||||
--require-must-keep-file "$FIXTURES/must-keep.patterns" \
|
||||
--report-json "$WORK/nehops-report.json" \
|
||||
>"$WORK/nehops-compressed.log" 2>"$WORK/nehops.err"
|
||||
RC=$?
|
||||
set -e 2>/dev/null || true
|
||||
RAW_WARNINGS="$(grep -c '\[WRN\]' "$FIXTURES/analyzer-sanitized.log")"
|
||||
OUT_WARNINGS="$(grep -c '\[WRN\]' "$WORK/nehops-compressed.log" || true)"
|
||||
if [[ "$RC" -eq 0 && "$RAW_WARNINGS" -eq 2 && "$OUT_WARNINGS" -eq "$RAW_WARNINGS" ]] \
|
||||
&& grep -q "FRR04700_CL.*common code tracing skipped" "$WORK/nehops-compressed.log" \
|
||||
&& grep -q "FRR04700_SV.*common code tracing skipped" "$WORK/nehops-compressed.log" \
|
||||
&& grep -q "Error Messages: 0" "$WORK/nehops-compressed.log"; then
|
||||
pass "NEHOPS [WRN] + CL/SV/domain invariants survive compression"
|
||||
else
|
||||
fail "NEHOPS warning preservation failed (rc=$RC raw=$RAW_WARNINGS out=$OUT_WARNINGS)"
|
||||
fi
|
||||
|
||||
set +e
|
||||
python3 - "$WORK/nehops-report.json" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
report = json.load(open(sys.argv[1], encoding="utf-8"))
|
||||
assert report["decision"] == "compressed"
|
||||
assert report["preservation"]["status"] == "passed"
|
||||
assert report["preservation"]["protected_line_missing"] == 0
|
||||
assert report["preservation"]["required_pattern_deficits"] == []
|
||||
assert report["measurement_source"] == "whitespace_estimate"
|
||||
assert report["provider_telemetry"] is None
|
||||
assert report["saving_status"] == "estimate_only_quality_gate_required"
|
||||
assert report["severity"]["raw"]["warning"] == 2
|
||||
assert report["severity"]["candidate"]["warning"] == 2
|
||||
assert len(report["hashes"]["raw_sha256"]) == 64
|
||||
assert len(report["hashes"]["candidate_sha256"]) == 64
|
||||
PY
|
||||
RC=$?
|
||||
set -e 2>/dev/null || true
|
||||
[[ "$RC" -eq 0 ]] \
|
||||
&& pass "JSON evidence binds hashes and labels estimate/provider status" \
|
||||
|| fail "JSON evidence contract invalid"
|
||||
|
||||
# 9) Required-pattern multiplicity: retaining one of two matching lines is not
|
||||
# sufficient. Rejection defaults to raw fallback and reports zero savings.
|
||||
printf 'TRACE-MUST plain line\nERROR TRACE-MUST protected line\n' > "$WORK/multi.txt"
|
||||
printf 'TRACE-MUST\n' > "$WORK/multi.patterns"
|
||||
set +e
|
||||
python3 "$CC" --mode extractive --input "$WORK/multi.txt" \
|
||||
--require-must-keep-file "$WORK/multi.patterns" \
|
||||
>"$WORK/multi.out" 2>"$WORK/multi.err"
|
||||
RC=$?
|
||||
set -e 2>/dev/null || true
|
||||
if [[ "$RC" -eq 1 ]] \
|
||||
&& cmp -s "$WORK/multi.txt" "$WORK/multi.out" \
|
||||
&& grep -q "TRACE-MUST(1/2)" "$WORK/multi.err" \
|
||||
&& grep -q "saving_status=rejected" "$WORK/multi.err" \
|
||||
&& grep -q "saved=0" "$WORK/multi.err"; then
|
||||
pass "partial must-keep loss is rejected with raw fallback and no saving claim"
|
||||
else
|
||||
fail "multiplicity/fallback contract failed (rc=$RC)"
|
||||
fi
|
||||
|
||||
# 10) A caller may choose halt instead of raw fallback. Halt returns non-zero
|
||||
# and emits no candidate bytes.
|
||||
set +e
|
||||
python3 "$CC" --mode extractive --input "$WORK/multi.txt" \
|
||||
--require-must-keep-file "$WORK/multi.patterns" \
|
||||
--on-preservation-failure halt \
|
||||
>"$WORK/halt.out" 2>"$WORK/halt.err"
|
||||
RC=$?
|
||||
set -e 2>/dev/null || true
|
||||
if [[ "$RC" -eq 1 && ! -s "$WORK/halt.out" ]] \
|
||||
&& grep -q "COMPRESS_REJECTED fallback=halt" "$WORK/halt.err" \
|
||||
&& grep -q "saving_status=rejected" "$WORK/halt.err" \
|
||||
&& grep -q "saved=0" "$WORK/halt.err"; then
|
||||
pass "halt policy emits no lossy candidate on preservation failure"
|
||||
else
|
||||
fail "halt policy leaked output or returned success (rc=$RC)"
|
||||
fi
|
||||
|
||||
# 11) Required patterns absent from raw are not fabricated and do not create a
|
||||
# false failure; preservation checks only claims that raw evidence contained.
|
||||
printf 'ERROR real failure line\n' > "$WORK/absent.txt"
|
||||
printf 'NEVER_PRESENT_IN_RAW\n' > "$WORK/absent.patterns"
|
||||
set +e
|
||||
python3 "$CC" --mode extractive --input "$WORK/absent.txt" \
|
||||
--require-must-keep-file "$WORK/absent.patterns" \
|
||||
>"$WORK/absent.out" 2>"$WORK/absent.err"
|
||||
RC=$?
|
||||
set -e 2>/dev/null || true
|
||||
[[ "$RC" -eq 0 ]] \
|
||||
&& pass "pattern absent from raw does not create a false preservation failure" \
|
||||
|| fail "absent raw pattern incorrectly failed preservation (rc=$RC)"
|
||||
|
||||
# 12) Invalid project regex is a configuration defect and fails closed without
|
||||
# emitting a candidate.
|
||||
printf '[unterminated\n' > "$WORK/invalid.patterns"
|
||||
set +e
|
||||
python3 "$CC" --mode structural --input "$WORK/absent.txt" \
|
||||
--must-keep-file "$WORK/invalid.patterns" \
|
||||
>"$WORK/invalid.out" 2>"$WORK/invalid.err"
|
||||
RC=$?
|
||||
set -e 2>/dev/null || true
|
||||
if [[ "$RC" -eq 1 && ! -s "$WORK/invalid.out" ]] \
|
||||
&& grep -q "COMPRESS_FAIL invalid_must_keep_regex" "$WORK/invalid.err"; then
|
||||
pass "invalid must-keep regex fails closed"
|
||||
else
|
||||
fail "invalid regex did not fail closed (rc=$RC)"
|
||||
fi
|
||||
|
||||
# 13) Non-empty raw input may not silently compress to an empty view.
|
||||
printf 'ordinary informational chatter\n' > "$WORK/empty-loss.txt"
|
||||
set +e
|
||||
python3 "$CC" --mode structural --input "$WORK/empty-loss.txt" \
|
||||
>"$WORK/empty-loss.out" 2>"$WORK/empty-loss.err"
|
||||
RC=$?
|
||||
set -e 2>/dev/null || true
|
||||
if [[ "$RC" -eq 1 ]] \
|
||||
&& cmp -s "$WORK/empty-loss.txt" "$WORK/empty-loss.out" \
|
||||
&& grep -q "COMPRESS_REJECTED fallback=raw" "$WORK/empty-loss.err"; then
|
||||
pass "empty compressed view is rejected and raw is preserved"
|
||||
else
|
||||
fail "non-empty raw input was allowed to disappear (rc=$RC)"
|
||||
fi
|
||||
|
||||
# 14) Dedup may collapse repeated warnings only when the multiplicity marker
|
||||
# proves how many occurrences were present.
|
||||
printf '[WRN] repeated warning\n[WRN] repeated warning\n[WRN] repeated warning\n' > "$WORK/repeated-warning.txt"
|
||||
set +e
|
||||
python3 "$CC" --mode structural --input "$WORK/repeated-warning.txt" \
|
||||
>"$WORK/repeated-warning.out" 2>"$WORK/repeated-warning.err"
|
||||
RC=$?
|
||||
set -e 2>/dev/null || true
|
||||
if [[ "$RC" -eq 0 ]] \
|
||||
&& grep -q '\[WRN\] repeated warning (x3)' "$WORK/repeated-warning.out"; then
|
||||
pass "dedup preserves repeated-warning multiplicity"
|
||||
else
|
||||
fail "repeated-warning multiplicity was lost (rc=$RC)"
|
||||
fi
|
||||
|
||||
# 15) A literal line ending in "(xN)" is content, not necessarily a dedup
|
||||
# marker. It must survive in both non-dedup and dedup modes.
|
||||
printf 'ERROR literal marker (x3)\n' > "$WORK/literal-marker.txt"
|
||||
set +e
|
||||
python3 "$CC" --mode extractive --input "$WORK/literal-marker.txt" \
|
||||
>"$WORK/literal-marker.out" 2>"$WORK/literal-marker.err"
|
||||
RC=$?
|
||||
set -e 2>/dev/null || true
|
||||
if [[ "$RC" -eq 0 ]] \
|
||||
&& cmp -s "$WORK/literal-marker.txt" "$WORK/literal-marker.out"; then
|
||||
pass "extractive mode does not misread a literal (xN) suffix"
|
||||
else
|
||||
fail "literal (xN) content triggered a false preservation failure (rc=$RC)"
|
||||
fi
|
||||
|
||||
set +e
|
||||
python3 "$CC" --mode structural --input "$WORK/literal-marker.txt" \
|
||||
>"$WORK/literal-marker-structural.out" 2>"$WORK/literal-marker-structural.err"
|
||||
RC=$?
|
||||
set -e 2>/dev/null || true
|
||||
if [[ "$RC" -eq 0 ]] \
|
||||
&& cmp -s "$WORK/literal-marker.txt" "$WORK/literal-marker-structural.out"; then
|
||||
pass "structural mode disambiguates a literal (xN) suffix"
|
||||
else
|
||||
fail "structural mode misread literal (xN) content (rc=$RC)"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "===== Plan-08 ⟷ Control Plane: settings govern harness ====="
|
||||
CPS="$CASAN_HARNESS_ROOT/scripts/bash/control-plane-settings.py"
|
||||
|
||||
Reference in New Issue
Block a user