feat: add chat replay verification

2026-07-08 22:45:12 +09:00
parent 49d0363c6a
commit 36af576f15
15 changed files with 350 additions and 14 deletions
@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+set -uo pipefail
+
+# Plan-18 MVP-2 Track 8: chat replay verifies chat chain, evidence artifact hash,
+# and Plan-17 loop trace replay for OPERATOR turns.
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$SCRIPT_DIR/../scripts/bash/casan-paths.sh"
+TURN="$CASAN_HARNESS_ROOT/scripts/bash/chat-turn.py"
+REPLAY="$CASAN_HARNESS_ROOT/scripts/bash/chat-replay.py"
+WORK="$(mktemp -d)"
+trap 'rm -rf "$WORK"' EXIT
+export CASAN_STATE_ROOT="$WORK/state"
+export CASAN_LOOP_STATE_ROOT="$CASAN_STATE_ROOT/logs/chat/loop-state"
+
+PASS=0; FAIL=0
+pass() { echo "PASS: $1"; PASS=$((PASS + 1)); }
+fail() { echo "FAIL: $1"; FAIL=$((FAIL + 1)); }
+
+echo "===== Plan-18 MVP-2 chat replay ====="
+
+python3 "$TURN" ask --message "run tests" --actor bob --role operator --chat-id replay-chat > "$WORK/turn.json"
+
+python3 "$REPLAY" verify-chain > "$WORK/verify.json" \
+  && grep -q '"decision": "OK"' "$WORK/verify.json" \
+  && pass "chat replay verifies chat audit chain" || fail "chat replay chain verify failed"
+
+python3 "$REPLAY" replay --chat-id replay-chat > "$WORK/replay.json"
+python3 - "$WORK/replay.json" <<'PY' \
+  && pass "chat replay matches clean operator turn" || fail "chat replay did not match clean turn"
+import json, sys
+d = json.load(open(sys.argv[1]))
+assert d["decision"] == "MATCH"
+assert d["records"] == 1
+assert d["loop_replayed"] == 1
+assert d["diffs"] == []
+PY
+
+ARTIFACT="$(python3 - "$WORK/turn.json" <<'PY'
+import json, os, sys
+d = json.load(open(sys.argv[1]))
+print(d["sources"][0]["path"])
+PY
+)"
+case "$ARTIFACT" in
+  /*) ARTIFACT_PATH="$ARTIFACT" ;;
+  *) ARTIFACT_PATH="$CASAN_APP_ROOT/$ARTIFACT" ;;
+esac
+printf '\nTAMPERED\n' >> "$ARTIFACT_PATH"
+set +e
+python3 "$REPLAY" replay --chat-id replay-chat > "$WORK/tamper-artifact.json"
+RC=$?
+set -e 2>/dev/null || true
+python3 - "$WORK/tamper-artifact.json" "$RC" <<'PY' \
+  && pass "chat replay detects evidence artifact tamper" || fail "chat replay missed artifact tamper"
+import json, sys
+d = json.load(open(sys.argv[1]))
+assert int(sys.argv[2]) == 3
+assert d["decision"] == "DRIFT"
+assert any(x["kind"] == "artifact_hash_mismatch" for x in d["diffs"])
+PY
+
+# Fresh state for chain tamper so artifact drift above does not mask the chain test.
+export CASAN_STATE_ROOT="$WORK/state-chain"
+export CASAN_LOOP_STATE_ROOT="$CASAN_STATE_ROOT/logs/chat/loop-state"
+python3 "$TURN" ask --message "run tests" --actor bob --role operator --chat-id chain-chat >/dev/null
+AUDIT="$CASAN_STATE_ROOT/logs/chat/chat-turns.jsonl"
+python3 - "$AUDIT" <<'PY'
+import json, sys
+rows = [json.loads(l) for l in open(sys.argv[1]) if l.strip()]
+rows[0]["decision"] = "TAMPERED"
+open(sys.argv[1], "w").write("\n".join(json.dumps(r) for r in rows) + "\n")
+PY
+set +e
+python3 "$REPLAY" verify-chain > "$WORK/tamper-chain.json"
+RC=$?
+set -e 2>/dev/null || true
+python3 - "$WORK/tamper-chain.json" "$RC" <<'PY' \
+  && pass "chat replay verify-chain detects audit tamper" || fail "chat replay missed audit tamper"
+import json, sys
+d = json.load(open(sys.argv[1]))
+assert int(sys.argv[2]) == 3
+assert d["decision"] == "BREAK"
+PY
+
+echo ""
+echo "===== CHAT REPLAY SUMMARY: PASS=$PASS FAIL=$FAIL ====="
+[[ "$FAIL" -eq 0 ]] || exit 1