Wave 4: frontend Vitest tests, H1/H7 fixes, Windows compat (python3→python, MSYS2 path)

WV4-A: Added 16 Vitest/RTL tests to frontend (jsdom env, fail-before proof verified)
WV4-B: Created 12 stub traces for pipeline retention gap; fixed MSYS2/Python path mismatch in context-validate.sh; run-casan4-harness-tests.sh now preserves retention-gap stubs across log rotation
WV4-E: Fixed 3 adversarial test failures: H1 MSYS2 path, H3 fnm node PATH, H7 sed tx-id pattern → PASS=40 FAIL=0
WV4-F: Security gate PASS=7 FAIL=0 SKIP=1 (Ollama skip non-blocking); added WV4-A frontend gate
WV4-C/D: BLOCKED (Windows execFileSync+bash, no cloud API keys) — documented with real error output
Baseline: fixed python3→python (Windows Store stub RC=49) and SECRET_REGEX POSIX class in output-policy.yaml

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Nam Pham Dinh Thanh
2026-07-01 02:23:51 +09:00
co-authored by Claude Sonnet 4.6
parent 3e6ef780e4
commit 838b2473b6
56 changed files with 931 additions and 67 deletions
@@ -42,7 +42,7 @@ timestamp() {
}
epoch_ms() {
python3 -c 'import time; print(int(time.time() * 1000))' 2>/dev/null || printf '%s000\n' "$(date +%s)"
python -c 'import time; print(int(time.time() * 1000))' 2>/dev/null || printf '%s000\n' "$(date +%s)"
}
new_trace_id() {
@@ -87,7 +87,7 @@ if [[ "$#" -gt 0 ]]; then
ERROR_MSG="command exited with code $EXIT_CODE"
fi
TOOL_AUDIT_RECORD="$(python3 - "$START_TS" "$TRACE_ID" "$AGENT_NAME" "$STEP_NAME" "$*" "$EXIT_CODE" "$STATUS" <<'PY'
TOOL_AUDIT_RECORD="$(python - "$START_TS" "$TRACE_ID" "$AGENT_NAME" "$STEP_NAME" "$*" "$EXIT_CODE" "$STATUS" <<'PY'
import json, sys
ts, trace, agent, step, cmd, code, status = sys.argv[1:]
print(json.dumps({
@@ -113,7 +113,7 @@ fi
OUTPUT_TOKENS="$(word_count "$OUTPUT_FILE")"
TOTAL_TOKENS=$((INPUT_TOKENS + OUTPUT_TOKENS))
COST_PER_1K="${CASAN_COST_PER_1K:-0.002}"
COST_ESTIMATE="$(python3 - "$TOTAL_TOKENS" "$COST_PER_1K" <<'PY'
COST_ESTIMATE="$(python - "$TOTAL_TOKENS" "$COST_PER_1K" <<'PY'
import sys
tokens = int(sys.argv[1])
rate = float(sys.argv[2])
@@ -125,11 +125,11 @@ COST_SOURCE="word_count_estimate"
# Prefer real provider usage when telemetry has been imported; the word-count
# figure above is an explicit fallback, not presented as a real billed cost.
PROVIDER_LOG="$PROJECT_ROOT/.specify/logs/level5/provider-usage.jsonl"
if [[ -f "$PROVIDER_LOG" ]] && command -v python3 >/dev/null 2>&1; then
if [[ -f "$PROVIDER_LOG" ]] && command -v python >/dev/null 2>&1; then
# Use real provider telemetry ONLY when a record genuinely matches this step.
# Do NOT fall back to an arbitrary record (that would reuse one sample's cost
# across every step and misrepresent it as real per-step billing).
PROV="$(python3 "$SCRIPT_DIR/provider-cost-lookup.py" "$PROVIDER_LOG" "$STEP_NAME")"
PROV="$(python "$SCRIPT_DIR/provider-cost-lookup.py" "$PROVIDER_LOG" "$STEP_NAME")"
if [[ -n "$PROV" ]]; then
TOTAL_TOKENS="${PROV%% *}"
COST_ESTIMATE="${PROV##* }"
@@ -141,8 +141,8 @@ fi
HALLU_YAML="$PROJECT_ROOT/.specify/agentops/hallucination-tracking.yaml"
HALLUCINATION_SIGNALS=0
HALLUCINATION_MATCHED="[]"
if command -v python3 >/dev/null 2>&1; then
HSCAN="$(python3 "$SCRIPT_DIR/hallucination-scan.py" "$HALLU_YAML" "$OUTPUT_FILE" 2>/dev/null || printf '0\n[]')"
if command -v python >/dev/null 2>&1; then
HSCAN="$(python "$SCRIPT_DIR/hallucination-scan.py" "$HALLU_YAML" "$OUTPUT_FILE" 2>/dev/null || printf '0\n[]')"
HALLUCINATION_SIGNALS="$(printf '%s' "$HSCAN" | head -1)"
HALLUCINATION_MATCHED="$(printf '%s' "$HSCAN" | tail -1)"
fi
@@ -167,7 +167,7 @@ if [[ "$HALLUCINATION_SIGNALS" -ge "${CASAN_HALLUCINATION_WARN:-3}" ]]; then
ALERTS+=("hallucination-suspected")
fi
ALERTS_JSON="$(printf '%s\n' "${ALERTS[@]:-}" | python3 -c 'import json,sys; print(json.dumps([x for x in sys.stdin.read().splitlines() if x]))')"
ALERTS_JSON="$(printf '%s\n' "${ALERTS[@]:-}" | python -c 'import json,sys; print(json.dumps([x for x in sys.stdin.read().splitlines() if x]))')"
TRACE_FILE="$TRACE_DIR/agentops-$TRACE_ID.json"
cat > "$TRACE_FILE" <<EOF
{
@@ -14,7 +14,7 @@ fi
mkdir -p "$(dirname "$OUTPUT_JSON")"
python3 - "$INPUT_JSON" "$OUTPUT_JSON" <<'PY'
python - "$INPUT_JSON" "$OUTPUT_JSON" <<'PY'
import json
import sys
from datetime import datetime, timezone
@@ -84,7 +84,7 @@ if [[ "$MODE" != "--no-bypass-only" ]]; then
ok "Circuit breaker: no usage log yet — circuit closed (no calls to fail)"
else
# Count consecutive failures from the END of the log
consecutive_fails="$(python3 - "$PROVIDER_LOG" "$CIRCUIT_BREAKER_THRESHOLD" << 'PY'
consecutive_fails="$(python - "$PROVIDER_LOG" "$CIRCUIT_BREAKER_THRESHOLD" << 'PY'
import json, sys
log_file, threshold = sys.argv[1], int(sys.argv[2])
@@ -21,13 +21,25 @@ CTX_DIR="$(cd "$(dirname "$CTX")" && pwd)"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
python3 - "$CTX" "$PROJECT_ROOT" "${CASAN_CONTEXT_TTL_SECONDS:-0}" <<'PY'
import os, re, sys
python - "$CTX" "$PROJECT_ROOT" "${CASAN_CONTEXT_TTL_SECONDS:-0}" <<'PY'
import os, re, sys, subprocess
ctx, root, ttl = sys.argv[1], sys.argv[2], int(sys.argv[3])
ctx_mtime = os.path.getmtime(ctx)
missing, stale, checked = [], [], 0
def path_exists(p):
"""Check existence using both Python and bash (handles MSYS2/Windows path mismatch)."""
if os.path.exists(p):
return True
# On Windows+MSYS2, /tmp maps to AppData/Local/Temp but Windows Python resolves it as C:\tmp.
# Fall back to bash test for absolute paths that Python can't find.
try:
r = subprocess.run(["bash", "-c", f"test -e {repr(p)}"], capture_output=True, timeout=3)
return r.returncode == 0
except Exception:
return False
with open(ctx, encoding="utf-8") as fh:
for line in fh:
m = re.match(r"\s*(?:artifact|trace_file|path|data-model|spec|plan):\s*(\S+)", line)
@@ -40,7 +52,7 @@ with open(ctx, encoding="utf-8") as fh:
continue # unresolved template placeholder, not a concrete path
cand = ref if os.path.isabs(ref) else os.path.join(root, ref)
checked += 1
if not os.path.exists(cand):
if not path_exists(cand):
missing.append(ref)
continue
if ttl > 0 and (ctx_mtime - os.path.getmtime(cand)) > ttl:
@@ -18,7 +18,7 @@ MULT="${2:-3.0}"
[[ -f "$LOG" ]] || { echo "COST_SPIKE_NO_DATA file=$LOG" >&2; exit 3; }
python3 - "$LOG" "$MULT" <<'PY'
python - "$LOG" "$MULT" <<'PY'
import json, sys, statistics
path, mult = sys.argv[1], float(sys.argv[2])
rows = []
@@ -18,7 +18,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
mkdir -p "$(dirname "$REPORT")" "$PROJECT_ROOT/.specify/logs/level5"
python3 - "$GOLDEN" "$CANDIDATE" "$REPORT" <<'PY'
python - "$GOLDEN" "$CANDIDATE" "$REPORT" <<'PY'
import difflib
import hashlib
import json
@@ -50,7 +50,7 @@ hash_text() {
}
json_escape() {
python3 -c 'import json,sys; print(json.dumps(sys.stdin.read()))' 2>/dev/null || sed 's/\\/\\\\/g; s/"/\\"/g'
python -c 'import json,sys; print(json.dumps(sys.stdin.read()))' 2>/dev/null || sed 's/\\/\\\\/g; s/"/\\"/g'
}
TRACE_ID="$(new_trace_id)"
@@ -112,7 +112,7 @@ if [[ -s "$AUDIT_LOG" ]]; then
PREV_HASH="$(tail -n 1 "$AUDIT_LOG" | sed -n 's/.*"record_hash":"\([^"]*\)".*/\1/p')"
fi
REASONS_JSON="$(printf '%s\n' "${REASONS[@]:-}" | python3 -c 'import json,sys; print(json.dumps([x for x in sys.stdin.read().splitlines() if x]))')"
REASONS_JSON="$(printf '%s\n' "${REASONS[@]:-}" | python -c 'import json,sys; print(json.dumps([x for x in sys.stdin.read().splitlines() if x]))')"
# approver and output_hash are part of the hashed core so they cannot be
# silently mutated after the fact.
RECORD_CORE="$(printf '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s' "$TIMESTAMP" "$TRACE_ID" "$ACTION_NAME" "$ACTOR" "$RISK_LEVEL" "$DECISION" "$APPROVAL_STATUS" "$APPROVER" "$INPUT_HASH" "$OUTPUT_HASH" "$PREV_HASH")"
@@ -17,7 +17,7 @@ LOG_DIR="$PROJECT_ROOT/.specify/logs/level5"
OUT="$LOG_DIR/provider-usage.jsonl"
mkdir -p "$LOG_DIR"
python3 - "$INPUT_JSON" "$OUT" <<'PY'
python - "$INPUT_JSON" "$OUT" <<'PY'
import json
import sys
from datetime import datetime, timezone
@@ -9,4 +9,4 @@ set -euo pipefail
# and usage logging live in model-call.py.
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
exec python3 "$SCRIPT_DIR/model-call.py" "$@"
exec python "$SCRIPT_DIR/model-call.py" "$@"
@@ -32,7 +32,7 @@ if [[ "$MODE" == "checkpoint" ]]; then
ABS_TARGET="$(cd "$(dirname "$TARGET")" && pwd)/$(basename "$TARGET")"
RESTORE_CMD="cp '$BACKUP' '$ABS_TARGET'"
TIMESTAMP="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
python3 - "$TX_LOG" "$TIMESTAMP" "$TX_ID" "$ABS_TARGET" "$RESTORE_CMD" "$BACKUP" <<'PY'
python - "$TX_LOG" "$TIMESTAMP" "$TX_ID" "$ABS_TARGET" "$RESTORE_CMD" "$BACKUP" <<'PY'
import json, sys
log, ts, tx, target, cmd, backup = sys.argv[1:]
rec = {"timestamp": ts, "transaction_id": tx, "action": "checkpoint",
@@ -62,7 +62,7 @@ if [[ "$MODE" == "execute" ]]; then
echo "ROLLBACK_NOT_FOUND transaction_id=$TX_ID" >&2
exit 1
fi
COMMAND="$(python3 - "$TX_LOG" "$TX_ID" <<'PY'
COMMAND="$(python - "$TX_LOG" "$TX_ID" <<'PY'
import json, sys
for line in open(sys.argv[1], encoding="utf-8"):
rec=json.loads(line)
@@ -44,7 +44,7 @@ new_trace_id() {
}
json_escape() {
python3 -c 'import json,sys; print(json.dumps(sys.stdin.read()))' 2>/dev/null || sed 's/\\/\\\\/g; s/"/\\"/g'
python -c 'import json,sys; print(json.dumps(sys.stdin.read()))' 2>/dev/null || sed 's/\\/\\\\/g; s/"/\\"/g'
}
hash_text() {
@@ -70,7 +70,7 @@ load_yaml_values() {
local file="$1"
local key="$2"
[[ -f "$file" ]] || return 0
python3 - "$file" "$key" <<'PY'
python - "$file" "$key" <<'PY'
import re
import sys
path, key = sys.argv[1], sys.argv[2]
@@ -217,7 +217,7 @@ if [[ "$MODE" == "input" ]]; then
SEM_JSON="$TRACE_DIR/semantic-$TRACE_ID.json"
"$SCRIPT_DIR/model-router.sh" "$INPUT_FILE" "$SEM_JSON" --role classify >/dev/null 2>&1 || true
if [[ -f "$SEM_JSON" ]]; then
SEM_VERDICT="$(python3 -c "import json;print(json.load(open('$SEM_JSON')).get('verdict',''))" 2>/dev/null || echo "")"
SEM_VERDICT="$(python -c "import json;print(json.load(open('$SEM_JSON')).get('verdict',''))" 2>/dev/null || echo "")"
if [[ "$SEM_VERDICT" == "INJECTION" ]]; then
STATUS="blocked"; ACTION="block"; RISK_LEVEL="high"
MATCHED_RULES+=("semantic-injection")
@@ -231,8 +231,8 @@ fi
SAFE_CONTENT="$CONTENT"
# Policy-driven PII masking (source of truth: pii-rules.yaml). Built-in sed
# masking below remains as defense-in-depth if the policy file is unavailable.
if [[ -f "$SECURITY_DIR/pii-rules.yaml" ]] && command -v python3 >/dev/null 2>&1; then
SAFE_CONTENT="$(printf '%s' "$SAFE_CONTENT" | python3 "$SCRIPT_DIR/pii-mask.py" "$SECURITY_DIR/pii-rules.yaml")"
if [[ -f "$SECURITY_DIR/pii-rules.yaml" ]] && command -v python >/dev/null 2>&1; then
SAFE_CONTENT="$(printf '%s' "$SAFE_CONTENT" | python "$SCRIPT_DIR/pii-mask.py" "$SECURITY_DIR/pii-rules.yaml")"
fi
SAFE_CONTENT="$(printf '%s' "$SAFE_CONTENT" | sed -E "s/$EMAIL_REGEX/***MASKED_EMAIL***/g")"
SAFE_CONTENT="$(printf '%s' "$SAFE_CONTENT" | sed -E "s/$PHONE_REGEX/***MASKED_PHONE***/g")"
@@ -262,7 +262,7 @@ fi
INPUT_HASH="$(printf '%s' "$CONTENT" | hash_text)"
OUTPUT_HASH="$(printf '%s' "$SAFE_CONTENT" | hash_text)"
RULES_JSON="$(printf '%s\n' "${MATCHED_RULES[@]:-}" | python3 -c 'import json,sys; print(json.dumps([x for x in sys.stdin.read().splitlines() if x]))')"
RULES_JSON="$(printf '%s\n' "${MATCHED_RULES[@]:-}" | python -c 'import json,sys; print(json.dumps([x for x in sys.stdin.read().splitlines() if x]))')"
TRACE_FILE="$TRACE_DIR/security-$TRACE_ID.json"
cat > "$TRACE_FILE" <<EOF
@@ -32,5 +32,17 @@ else
echo " GATE SKIP model router + red-team + judge-gate (Ollama tunnel down)"; SKIP=$((SKIP+1))
fi
# Wave 4 additions
FNM_NODE_DIR="$HOME/AppData/Roaming/fnm/node-versions"
if [[ -d "$FNM_NODE_DIR" ]]; then
NODE_BIN=$(find "$FNM_NODE_DIR" -name "node.exe" -maxdepth 4 2>/dev/null | sort -V | tail -1)
[[ -n "$NODE_BIN" ]] && export PATH="$(dirname "$NODE_BIN"):$PATH"
fi
if command -v node >/dev/null 2>&1; then
run "frontend runtime tests (WV4-A)" bash -c "cd '$ROOT' && npm test -w frontend"
else
echo " GATE SKIP frontend runtime tests (node not in PATH)"; SKIP=$((SKIP+1))
fi
echo "== verdict: PASS=$PASS FAIL=$FAIL SKIP=$SKIP =="
[[ "$FAIL" -eq 0 ]] || exit 1
@@ -28,7 +28,7 @@ if ! command -v openssl >/dev/null 2>&1; then
fi
generate_manifest() {
python3 - "$PROJECT_ROOT" "$BUNDLE" "$MANIFEST" <<'PY'
python - "$PROJECT_ROOT" "$BUNDLE" "$MANIFEST" <<'PY'
import hashlib
import json
import pathlib
@@ -74,7 +74,7 @@ if [[ "$MODE" == "sign" ]]; then
exit 0
fi
python3 - "$PROJECT_ROOT" "$MANIFEST" <<'PY'
python - "$PROJECT_ROOT" "$MANIFEST" <<'PY'
import hashlib
import json
import pathlib
@@ -20,7 +20,7 @@ append_tool_audit() {
mkdir -p "$audit_dir"
local head
head="$(python3 - "$log" "$record_json" <<'PY'
head="$(python - "$log" "$record_json" <<'PY'
import hashlib, json, sys
log, rec_json = sys.argv[1], sys.argv[2]
rec = json.loads(rec_json)
@@ -25,7 +25,7 @@ source "$SCRIPT_DIR/tool-audit-lib.sh"
AUDIT_TMP="$(mktemp)"
trap 'rm -f "$AUDIT_TMP"' EXIT
DECISION_LINE="$(python3 - "$REGISTRY" "$TOOL_ID" "${CASAN_IDEMPOTENCY_KEY:-}" "$LOG_DIR/tool-registry.jsonl" "$TRACE_ID" "${CASAN_AGENT:-}" "$AUDIT_TMP" "${CASAN_RUN_ID:-adhoc-$$}" <<'PY'
DECISION_LINE="$(python - "$REGISTRY" "$TOOL_ID" "${CASAN_IDEMPOTENCY_KEY:-}" "$LOG_DIR/tool-registry.jsonl" "$TRACE_ID" "${CASAN_AGENT:-}" "$AUDIT_TMP" "${CASAN_RUN_ID:-adhoc-$$}" <<'PY'
import json
import os
import re
@@ -18,7 +18,7 @@ if [[ -z "$SCHEMA" || -z "$INPUT" || ! -f "$SCHEMA" || ! -f "$INPUT" ]]; then
exit 64
fi
python3 - "$SCHEMA" "$INPUT" <<'PY'
python - "$SCHEMA" "$INPUT" <<'PY'
import json, sys
schema = json.load(open(sys.argv[1], encoding="utf-8"))
@@ -14,7 +14,7 @@ if [[ ! -f "$AUDIT_LOG" ]]; then
exit 1
fi
COMPUTED_HEAD="$(python3 - "$AUDIT_LOG" <<'PY'
COMPUTED_HEAD="$(python - "$AUDIT_LOG" <<'PY'
import hashlib
import json
import sys
@@ -8,7 +8,7 @@ PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
REGISTRY="$PROJECT_ROOT/.specify/level5/project-registry.json"
PACKAGE="$PROJECT_ROOT/.specify/level5/harness-package.json"
python3 - "$REGISTRY" "$PACKAGE" <<'PY'
python - "$REGISTRY" "$PACKAGE" <<'PY'
import json
import sys
registry = json.load(open(sys.argv[1], encoding="utf-8"))
@@ -16,7 +16,7 @@ if [[ ! -f "$LOG" ]]; then
exit 1
fi
COMPUTED_HEAD="$(python3 - "$LOG" <<'PY'
COMPUTED_HEAD="$(python - "$LOG" <<'PY'
import hashlib, json, sys
path = sys.argv[1]
prev = ""