feat: add chat replay verification

packages/casan-harness/scripts/bash/chat-operator.py

@@ -69,6 +69,14 @@ def sha(text: str) -> str:
     return hashlib.sha256(text.encode("utf-8")).hexdigest()
 
 
+def sha_file(path: str) -> str:
+    h = hashlib.sha256()
+    with open(path, "rb") as fh:
+        for chunk in iter(lambda: fh.read(65536), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
 def git_commit() -> str:
     try:
         r = subprocess.run(["git", "rev-parse", "HEAD"], cwd=ROOT, capture_output=True, text=True, timeout=5)
@@ -204,6 +212,12 @@ def record_metrics(trace_id: str, message: str, answer: str, status: str, latenc
 def finish(started, trace_id, chat_id, turn_id, tenant_id, actor, message, router, decision, answer,
            action=None, action_gate=None, artifact_path="", safe_message=""):
     elapsed = int((datetime.now(timezone.utc) - started).total_seconds() * 1000)
+    loop_run = {}
+    if os.environ.get("CASAN_CHAT_LOOP_RUN_JSON"):
+        try:
+            loop_run = json.loads(os.environ["CASAN_CHAT_LOOP_RUN_JSON"])
+        except ValueError:
+            loop_run = {"success": False, "decision": "DENIED", "reason": "loop_run_json_invalid"}
     source = []
     if artifact_path:
         source.append({
@@ -211,6 +225,7 @@ def finish(started, trace_id, chat_id, turn_id, tenant_id, actor, message, route
             "line": 1,
             "excerpt": answer[:260],
             "score": 10 if decision == "ACTION_COMPLETED" else 1,
+            "hash": sha_file(artifact_path),
             "envelope": provenance("chat-operator-artifact", artifact_path, decision == "ACTION_COMPLETED"),
         })
     rec = record_turn({
@@ -229,7 +244,8 @@ def finish(started, trace_id, chat_id, turn_id, tenant_id, actor, message, route
         "user_msg_ref": sha(message),
         "safe_preview": (safe_message or "")[:180],
         "answer_ref": sha(answer),
-        "sources": [{"path": s.get("path"), "line": s.get("line")} for s in source],
+        "sources": [{"path": s.get("path"), "line": s.get("line"), "hash": s.get("hash")} for s in source],
+        "loop_run": loop_run,
     })
     record_metrics(trace_id, safe_message or message, answer, "success" if decision == "ACTION_COMPLETED" else "failed", elapsed, (action or {}).get("id", "none"))
     return {
@@ -247,6 +263,7 @@ def finish(started, trace_id, chat_id, turn_id, tenant_id, actor, message, route
         "router": router,
         "action": {k: action.get(k) for k in ("id", "label", "description")} if action else None,
         "action_gate": action_gate or {},
+        "loop_run": loop_run,
     }
 
 

packages/casan-harness/scripts/bash/chat-replay.py

@@ -0,0 +1,187 @@
+#!/usr/bin/env python3
+"""Plan-18 Track 8 chat replay / verify-chain.
+
+Verifies chat history after the fact:
+  * H5 chat turn hash-chain is intact,
+  * recorded evidence artifact hashes still match,
+  * OPERATOR loop-run traces still verify and replay through Plan-17 loop-trace.
+
+This complements chat-readonly.py verify-audit by validating artifact state, not
+only the append-only JSONL chain.
+"""
+import argparse
+import hashlib
+import json
+import os
+import subprocess
+import sys
+
+
+GENESIS_HASH = "0" * 64
+
+
+def project_root() -> str:
+    d = os.path.abspath(os.path.dirname(__file__))
+    p = d
+    while p != os.path.dirname(p):
+        if os.path.isdir(os.path.join(p, ".specify")) or os.path.isdir(os.path.join(p, "packages/casan-harness")):
+            return p
+        p = os.path.dirname(p)
+    return os.path.abspath(os.path.join(d, "..", "..", ".."))
+
+
+ROOT = project_root()
+BIN = os.path.join(ROOT, "packages", "casan-harness", "scripts", "bash")
+LOOP_TRACE = os.path.join(BIN, "loop-trace.py")
+
+
+def state_root() -> str:
+    return os.environ.get("CASAN_STATE_ROOT") or os.path.join(ROOT, ".specify")
+
+
+def audit_path() -> str:
+    return os.environ.get("CASAN_CHAT_AUDIT_LOG") or os.path.join(state_root(), "logs", "chat", "chat-turns.jsonl")
+
+
+def sha_text(text: str) -> str:
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+def sha_file(path: str) -> str:
+    h = hashlib.sha256()
+    with open(path, "rb") as fh:
+        for chunk in iter(lambda: fh.read(65536), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def load_records():
+    records = []
+    try:
+        with open(audit_path(), encoding="utf-8") as fh:
+            for line in fh:
+                if line.strip():
+                    records.append(json.loads(line))
+    except OSError:
+        return []
+    return records
+
+
+def verify_chain(records):
+    prev = GENESIS_HASH
+    for idx, rec in enumerate(records, start=1):
+        got = rec.get("record_hash")
+        rest = {k: v for k, v in rec.items() if k != "record_hash"}
+        if rest.get("prev_hash") != prev or sha_text(json.dumps(rest, sort_keys=True, ensure_ascii=False)) != got:
+            return False, idx
+        prev = got
+    return True, None
+
+
+def resolve_path(path: str) -> str:
+    if os.path.isabs(path):
+        return path
+    return os.path.join(ROOT, path)
+
+
+def loop_state_root(loop_run):
+    explicit = os.environ.get("CASAN_LOOP_STATE_ROOT")
+    if explicit:
+        return explicit
+    artifact = loop_run.get("artifact") or ""
+    marker = os.sep + "logs" + os.sep + "chat" + os.sep + "loop-runs" + os.sep
+    if marker in artifact:
+        return artifact.split(marker, 1)[0] + os.sep + "logs" + os.sep + "chat" + os.sep + "loop-state"
+    return os.path.join(state_root(), "logs", "chat", "loop-state")
+
+
+def run_loop_trace(loop_run, cmd: str):
+    env = {**os.environ, "CASAN_LOOP_STATE_ROOT": loop_state_root(loop_run)}
+    run_id = loop_run.get("run_id")
+    if not run_id:
+        return False, "missing_loop_run_id"
+    args = ["python3", LOOP_TRACE, cmd, "--run-id", run_id]
+    if cmd == "replay":
+        args += ["--profile", os.environ.get("CASAN_PROFILE", "dev")]
+    r = subprocess.run(args, cwd=ROOT, capture_output=True, text=True, env=env)
+    return r.returncode == 0, (r.stdout + r.stderr).strip()
+
+
+def filter_records(records, chat_id="", turn_id=""):
+    out = records
+    if chat_id:
+        out = [r for r in out if r.get("chat_id") == chat_id]
+    if turn_id:
+        out = [r for r in out if r.get("turn_id") == turn_id]
+    return out
+
+
+def replay(args) -> int:
+    records = load_records()
+    ok, broken_at = verify_chain(records)
+    if not ok:
+        print(json.dumps({"decision": "BREAK", "reason": "chat_chain_broken", "broken_at": broken_at}, ensure_ascii=False))
+        return 3
+
+    selected = filter_records(records, args.chat_id, args.turn_id)
+    diffs = []
+    replayed = 0
+    for rec in selected:
+        for src in rec.get("sources", []):
+            expected = src.get("hash")
+            if not expected:
+                continue
+            path = resolve_path(src.get("path", ""))
+            if not os.path.isfile(path):
+                diffs.append({"seq": rec.get("seq"), "kind": "artifact_missing", "path": src.get("path")})
+                continue
+            got = sha_file(path)
+            if got != expected:
+                diffs.append({"seq": rec.get("seq"), "kind": "artifact_hash_mismatch", "path": src.get("path"), "expected": expected, "got": got})
+
+        loop_run = rec.get("loop_run") or {}
+        if loop_run.get("run_id"):
+            replayed += 1
+            trace_ok, trace_out = run_loop_trace(loop_run, "verify-chain")
+            replay_ok, replay_out = run_loop_trace(loop_run, "replay")
+            if not trace_ok:
+                diffs.append({"seq": rec.get("seq"), "kind": "loop_trace_break", "run_id": loop_run.get("run_id"), "output": trace_out[:400]})
+            if not replay_ok:
+                diffs.append({"seq": rec.get("seq"), "kind": "loop_replay_drift", "run_id": loop_run.get("run_id"), "output": replay_out[:400]})
+
+    payload = {
+        "decision": "DRIFT" if diffs else "MATCH",
+        "records": len(selected),
+        "loop_replayed": replayed,
+        "diffs": diffs,
+        "audit_path": audit_path(),
+    }
+    print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True))
+    return 3 if diffs else 0
+
+
+def verify(args) -> int:
+    records = load_records()
+    ok, broken_at = verify_chain(records)
+    payload = {"decision": "OK" if ok else "BREAK", "records": len(records), "audit_path": audit_path()}
+    if broken_at:
+        payload["broken_at"] = broken_at
+    print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True))
+    return 0 if ok else 3
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    sub = ap.add_subparsers(dest="cmd", required=True)
+    r = sub.add_parser("replay")
+    r.add_argument("--chat-id", default="")
+    r.add_argument("--turn-id", default="")
+    r.set_defaults(func=replay)
+    v = sub.add_parser("verify-chain")
+    v.set_defaults(func=verify)
+    args = ap.parse_args()
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

packages/casan-harness/scripts/bash/chat-turn.py

@@ -63,7 +63,10 @@ def scan_tool_output(text: str, label: str):
 
 
 def run_mode(args, binding, loop_run=None, scan_output_label=""):
-    r = subprocess.run(args, cwd=ROOT, capture_output=True, text=True)
+    env = os.environ.copy()
+    if loop_run is not None:
+        env["CASAN_CHAT_LOOP_RUN_JSON"] = json.dumps(loop_run, ensure_ascii=False, sort_keys=True)
+    r = subprocess.run(args, cwd=ROOT, capture_output=True, text=True, env=env)
     if scan_output_label:
         scan_rc, scan_msg = scan_tool_output(r.stdout + "\n" + r.stderr, scan_output_label)
         if scan_rc != 0:

packages/casan-harness/scripts/bash/ci-harness-gate.sh

@@ -152,6 +152,7 @@ run "phase-chat-operator"        bash "$TESTS/phase-chat-operator-tests.sh"
 run "phase-chat-agent-select"    bash "$TESTS/phase-chat-agent-select-tests.sh"
 run "phase-chat-pipeline"        bash "$TESTS/phase-chat-pipeline-tests.sh"
 run "phase-chat-stream-hold"     bash "$TESTS/phase-chat-stream-hold-tests.sh"
+run "phase-chat-replay"          bash "$TESTS/phase-chat-replay-tests.sh"
 
 # ARCH-02: coverage cannot silently drop; ARCH-01: harness/policy cannot silently
 # drift. Both SKIP cleanly when no manifest is provisioned (non-strict dev/CI).