thanhnvandClaude Opus 4.8
664bd1f00c
Physically move the pure-code subtrees out of .specify into the package, leaving compat symlinks at the old .specify/<dir> paths so every existing reference (internal CASAN_HARNESS_ROOT + external CI/docker/mjs) keeps resolving. Runtime state stays put. Moved (git mv): scripts/ tests/ security/ templates/ config/ governance/ memory/ .specify/<dir> -> packages/casan-harness/<dir> (+ .specify/<dir> symlink) Stays in .specify (state/governance/domain, handled later): logs/ agentops/ level5/ init-options.json traceability-map.json Python `.resolve()` self-location followed the compat symlink into packages and lost the app root; generate-casan-demo-context.py, generate-agentops-dashboard.py and dashboard-server.py now walk UP for the `.specify` state marker instead of a fixed parent depth (fixes "missing trace files" in run-casan4). Full gate: PASS=64 FAIL=0 SKIP=3 (CASAN_CI_STEP_TIMEOUT_SEC=1200 — track-a ~450s runs close to the 600s default and can tip over under load; this is timing variance, not a regression — it passed cleanly with headroom). Runtime log/audit artifacts kept unstaged. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
65 lines
1.8 KiB
Python
Executable File
65 lines
1.8 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""CASAN H6 hallucination signal detector.
|
|
|
|
Reads the quoted keyword markers from hallucination-tracking.yaml plus a set of
|
|
generic uncertainty markers, scans the agent output, and reports how many
|
|
hallucination signals were found. This turns hallucination-tracking.yaml from
|
|
dead config into a real, populated metric written to metrics.jsonl.
|
|
|
|
Usage: hallucination-scan.py <hallucination-tracking.yaml> <output-file>
|
|
Output (stdout): line 1 = integer signal count, line 2 = JSON list of matches.
|
|
"""
|
|
import json
|
|
import re
|
|
import sys
|
|
|
|
GENERIC_MARKERS = [
|
|
"maybe", "might be incorrect", "i am not sure", "uncertain",
|
|
"i think", "probably", "as far as i know",
|
|
]
|
|
|
|
|
|
def load_keywords(path):
|
|
keywords = []
|
|
try:
|
|
with open(path, encoding="utf-8") as fh:
|
|
for line in fh:
|
|
# Quoted list items are the hallucination keyword markers
|
|
# (unquoted list items are structured signal names, not text).
|
|
m = re.match(r'\s*-\s*"(.+)"\s*$', line)
|
|
if m:
|
|
keywords.append(m.group(1))
|
|
except OSError:
|
|
pass
|
|
return keywords
|
|
|
|
|
|
def main():
|
|
if len(sys.argv) < 3:
|
|
print(0)
|
|
print("[]")
|
|
return
|
|
keywords = load_keywords(sys.argv[1]) + GENERIC_MARKERS
|
|
try:
|
|
with open(sys.argv[2], encoding="utf-8") as fh:
|
|
text = fh.read().lower()
|
|
except OSError:
|
|
print(0)
|
|
print("[]")
|
|
return
|
|
matched = []
|
|
for kw in keywords:
|
|
k = kw.lower()
|
|
if not k:
|
|
continue
|
|
count = text.count(k)
|
|
if count:
|
|
matched.append({"marker": kw, "count": count})
|
|
total = sum(m["count"] for m in matched)
|
|
print(total)
|
|
print(json.dumps(matched))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|