thanhnvandClaude Opus 4.8
2c765c9a45
Task 1.2: introduce a single path resolver so no harness script hardcodes `.specify/...` scattered across the tree. casan-paths.sh resolves four roots (HARNESS/STATE/GOVERNANCE/APP) by marker-based walk-up from its own location — never `git rev-parse` (git root is the repo PARENT here, not the app dir). - 101 bash scripts/tests: 238 hardcoded `$PROJECT_ROOT/.specify/...` refs rewritten to CASAN_HARNESS_ROOT (code) / CASAN_STATE_ROOT (logs,state) / CASAN_GOVERNANCE_ROOT. Sandbox test vars ($WORK/$TP/$FP/$T1_WORK) left untouched. - Roots are NOT exported: each script/subprocess self-resolves from its own tree, matching the original per-script semantics and preserving hermetic sandbox isolation (node casan-step.mjs, copied telemetry/rollback scripts must not inherit real roots). - Sandbox tests that copy a harness script now also copy casan-paths.sh (its new sibling dependency): adversarial (verify-audit-chain/verify-tool-audit/rollback) + track-a (security-check/telemetry-integrity). - control-plane-settings.json reclassified as STATE (untracked runtime store). Roots all still resolve to `.specify` in this monolithic layout, so behavior is unchanged. Full gate: PASS=64 FAIL=0 SKIP=3 (adversarial 44/0, track-a 25/0). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
185 lines
9.8 KiB
Bash
Executable File
185 lines
9.8 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -uo pipefail
|
|
|
|
# Resolve node binary for Windows+fnm environments where node is not in default PATH
|
|
if ! command -v node >/dev/null 2>&1; then
|
|
FNM_NODE_DIR="$HOME/AppData/Roaming/fnm/node-versions"
|
|
if [[ -d "$FNM_NODE_DIR" ]]; then
|
|
NODE_BIN=$(find "$FNM_NODE_DIR" -name "node.exe" -maxdepth 4 2>/dev/null | sort -V | tail -1)
|
|
[[ -n "$NODE_BIN" ]] && export PATH="$(dirname "$NODE_BIN"):$PATH"
|
|
fi
|
|
fi
|
|
|
|
# CASAN WP-B — H3 model judge gate tests.
|
|
# Verifies that the review gates in casan-step.mjs apply AND(rule, model) logic:
|
|
# 1. Rule-rejected plans are REJECTED without calling the model.
|
|
# 2. A complete plan that passes rules reaches the model judge.
|
|
# 3. A rule-passing but semantically poor artifact can still be REJECTED by model.
|
|
# 4. Judge SKIP (Ollama down) is non-blocking — rules alone decide.
|
|
#
|
|
# Honest scope: model verdicts depend on ornith:9b being live.
|
|
# If tunnel is down, model tests SKIP not fail.
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
source "$SCRIPT_DIR/../scripts/bash/casan-paths.sh"
|
|
ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
|
WORK="$(mktemp -d)"; trap 'rm -rf "$WORK"' EXIT
|
|
PASS=0; FAIL=0; SKIP=0
|
|
|
|
ok() { echo " PASS $1"; PASS=$((PASS+1)); }
|
|
fail() { echo " FAIL $1"; FAIL=$((FAIL+1)); }
|
|
skip() { echo " SKIP $1 (${2:-reason})"; SKIP=$((SKIP+1)); }
|
|
|
|
expect_verdict() {
|
|
local label="$1" report_file="$2" expected="$3"
|
|
if [[ ! -f "$report_file" ]]; then
|
|
fail "$label (report file missing: $report_file)"; return
|
|
fi
|
|
local actual
|
|
actual="$(grep -oE 'verdict: (APPROVED|REJECTED)' "$report_file" | head -1 | awk '{print $2}')"
|
|
if [[ "$actual" == "$expected" ]]; then ok "$label (verdict=$actual)"
|
|
else fail "$label (expected=$expected got=$actual)"; fi
|
|
}
|
|
|
|
OLLAMA_UP=false
|
|
curl -sS -m 5 http://127.0.0.1:11434/api/tags >/dev/null 2>&1 && OLLAMA_UP=true
|
|
|
|
echo "=== WP-B: model judge gate tests ==="
|
|
|
|
# --- Setup: create fake pipeline context ---
|
|
mkdir -p "$WORK/docs/input" "$WORK/docs/output/ipa-docs/srs" \
|
|
"$WORK/docs/output/ipa-docs/bd" "$WORK/docs/output/ipa-docs/dd" \
|
|
"$WORK/docs/output/ipa-docs/testcase" \
|
|
"$WORK/docs/output/specs/001-okr-web-app/contracts" \
|
|
"$WORK/docs/output/output_logs/001-okr-web-app/reports" \
|
|
"$WORK/scripts" "$WORK/.specify/scripts/bash" \
|
|
"$WORK/.specify/logs/level5" "$WORK/.specify/logs/idempotency" \
|
|
"$WORK/.specify/logs/tmp"
|
|
|
|
# Minimal requirement and architecture stubs
|
|
printf "FR-01 Login\nFR-02 Create Objective\nFR-03 Key Result\nFR-04 Progress\nFR-05 Dashboard\n" > "$WORK/docs/input/okr-requirement.md"
|
|
printf "NestJS SQLite React\n" > "$WORK/docs/technical_architecture.md"
|
|
|
|
# Copy the real model-router.sh + model-call.py so the judge can run
|
|
cp "$CASAN_HARNESS_ROOT/scripts/bash/model-router.sh" "$WORK/.specify/scripts/bash/"
|
|
cp "$CASAN_HARNESS_ROOT/scripts/bash/model-call.py" "$WORK/.specify/scripts/bash/"
|
|
# Point provider log to work dir so we don't pollute main repo
|
|
export CASAN_PROVIDER_LOG="$WORK/.specify/logs/level5/provider-usage.jsonl"
|
|
|
|
# Symlink the scripts dir so casan-step.mjs resolves SCRIPTS_DIR correctly
|
|
# casan-step.mjs uses: join(dirname(__filename), '..', '.specify', 'scripts', 'bash')
|
|
# __filename = WORK/scripts/casan-step.mjs → dirname = WORK/scripts
|
|
# join(.., '..', ...) = WORK/.specify/scripts/bash ✓
|
|
cp "$ROOT/scripts/casan-step.mjs" "$WORK/scripts/"
|
|
|
|
run_step() {
|
|
local step="$1" attempt="${2:-1}"
|
|
# casan-step.mjs uses relative paths resolved from CWD — must run from WORK
|
|
( cd "$WORK" && CASAN_OUTPUT="$WORK/step-out-$step-$attempt.md" \
|
|
node "$WORK/scripts/casan-step.mjs" "$step" "$attempt" 2>/dev/null )
|
|
}
|
|
|
|
# ───────────────────────────────────────────────────────────────
|
|
# T1: FAIL-BEFORE — attempt=1 plan is REJECTED by rules
|
|
# (missing "Golden regression test" and "Rollback strategy")
|
|
# ───────────────────────────────────────────────────────────────
|
|
echo "--- T1: fail-before (plan attempt=1 missing rollback) ---"
|
|
run_step 01-srs 2>/dev/null || true
|
|
run_step 02-bd 2>/dev/null || true
|
|
run_step 03-spec 2>/dev/null || true
|
|
run_step 04-reviewspec 2>/dev/null || true
|
|
run_step 05-plan 1 2>/dev/null || true # attempt=1 → incomplete plan
|
|
REPORT_06_A1="$WORK/docs/output/output_logs/001-okr-web-app/reports/06-review-plan-report-attempt-1.md"
|
|
run_step 06-reviewplan 1 2>/dev/null || true
|
|
expect_verdict "T1: incomplete plan → REJECTED by rules" "$REPORT_06_A1" "REJECTED"
|
|
|
|
# Also verify the report mentions the missing criterion
|
|
if grep -q "missing plan criterion: Golden regression test\|missing plan criterion: Rollback strategy" "$REPORT_06_A1" 2>/dev/null; then
|
|
ok "T1b: report lists specific missing criteria"
|
|
else
|
|
fail "T1b: report does not name missing criteria"
|
|
fi
|
|
|
|
# ───────────────────────────────────────────────────────────────
|
|
# T2: PASS-AFTER — attempt=2 plan passes rules → reaches model judge
|
|
# ───────────────────────────────────────────────────────────────
|
|
echo "--- T2: pass-after (plan attempt=2 complete) ---"
|
|
run_step 05-plan 2 2>/dev/null || true # attempt=2 → complete plan + companion artifacts
|
|
REPORT_06_A2="$WORK/docs/output/output_logs/001-okr-web-app/reports/06-review-plan-report-attempt-2.md"
|
|
run_step 06-reviewplan 2 2>/dev/null || true
|
|
if [[ "$OLLAMA_UP" == "true" ]]; then
|
|
# Report should show model-judge verdict (APPROVED or REJECTED)
|
|
if grep -qE "model-judge: (APPROVED|REJECTED|SKIP)" "$REPORT_06_A2" 2>/dev/null; then
|
|
ok "T2: complete plan report contains model-judge verdict"
|
|
else
|
|
fail "T2: complete plan report missing model-judge verdict"
|
|
fi
|
|
else
|
|
skip "T2" "Ollama down"
|
|
fi
|
|
|
|
# ───────────────────────────────────────────────────────────────
|
|
# T3: JUDGE SKIP IS NON-BLOCKING — model skip doesn't fail a rule-APPROVED plan
|
|
# ───────────────────────────────────────────────────────────────
|
|
echo "--- T3: judge SKIP is non-blocking ---"
|
|
# If Ollama is down, the judge returns SKIP and the verdict should still be APPROVED
|
|
# (rules already passed). We simulate by checking that a rule-passing step without
|
|
# Ollama doesn't get forced to REJECTED.
|
|
if [[ "$OLLAMA_UP" == "false" ]]; then
|
|
# run step 04 with Ollama down — verdict should be APPROVED (rules pass, judge skips)
|
|
REPORT_04="$WORK/docs/output/output_logs/001-okr-web-app/reports/04-review-spec-report.md"
|
|
if [[ -f "$REPORT_04" ]]; then
|
|
actual_v="$(grep -oE 'verdict: (APPROVED|REJECTED)' "$REPORT_04" | head -1 | awk '{print $2}')"
|
|
if [[ "$actual_v" == "APPROVED" ]]; then
|
|
ok "T3: judge SKIP is non-blocking (Ollama down → verdict=APPROVED from rules)"
|
|
else
|
|
fail "T3: judge SKIP caused unwanted REJECTED"
|
|
fi
|
|
else
|
|
skip "T3" "report missing"
|
|
fi
|
|
else
|
|
# Ollama is up: verify report contains 'model-judge:' annotation
|
|
REPORT_04="$WORK/docs/output/output_logs/001-okr-web-app/reports/04-review-spec-report.md"
|
|
if grep -qE "model-judge:" "$REPORT_04" 2>/dev/null; then
|
|
ok "T3: review report includes model-judge annotation"
|
|
else
|
|
fail "T3: review report missing model-judge annotation"
|
|
fi
|
|
fi
|
|
|
|
# ───────────────────────────────────────────────────────────────
|
|
# T4: MODEL JUDGE FAIL-CLOSED — malformed response → REJECTED
|
|
# ───────────────────────────────────────────────────────────────
|
|
echo "--- T4: model fail-closed on malformed response ---"
|
|
if [[ "$OLLAMA_UP" == "true" ]]; then
|
|
# Create a tiny file whose combined content with criteria will make the model
|
|
# return something unusual. We verify model-call.py exits 3 on malformed → REJECTED.
|
|
# We test this by calling model-call.py directly with a file that asks for
|
|
# a number (not APPROVED/REJECTED) — the model won't give APPROVED or REJECTED.
|
|
MALFORM_FILE="$WORK/malform-judge.txt"
|
|
printf 'Return only the number 42, nothing else.\n' > "$MALFORM_FILE"
|
|
MALFORM_OUT="$WORK/malform-judge-out.json"
|
|
set +e
|
|
python "$CASAN_HARNESS_ROOT/scripts/bash/model-call.py" "$MALFORM_FILE" "$MALFORM_OUT" --role judge 2>/dev/null
|
|
mrc=$?
|
|
set -e 2>/dev/null || true
|
|
verdict_m="$(python -c "import json;print(json.load(open('$MALFORM_OUT')).get('verdict',''))" 2>/dev/null || echo "")"
|
|
malformed_m="$(python -c "import json;print(json.load(open('$MALFORM_OUT')).get('malformed',''))" 2>/dev/null || echo "")"
|
|
# Fail-closed: if model says "42" that's neither APPROVED nor REJECTED → REJECTED + exit 3
|
|
if [[ "$mrc" -eq 3 && "$malformed_m" == "True" && "$verdict_m" == "REJECTED" ]]; then
|
|
ok "T4: malformed model output → REJECTED fail-closed (rc=3)"
|
|
elif [[ "$verdict_m" == "APPROVED" || "$verdict_m" == "REJECTED" ]]; then
|
|
# model may actually output APPROVED or REJECTED even with that prompt — not malformed
|
|
ok "T4: model produced valid verdict=$verdict_m (not malformed — model followed instruction)"
|
|
else
|
|
fail "T4: unexpected state rc=$mrc verdict=$verdict_m malformed=$malformed_m"
|
|
fi
|
|
else
|
|
skip "T4" "Ollama down"
|
|
fi
|
|
|
|
echo ""
|
|
echo "=== WP-B judge gate results: PASS=$PASS FAIL=$FAIL SKIP=$SKIP ==="
|
|
[[ "$FAIL" -eq 0 ]] || exit 1
|