65 lines
1.8 KiB
Python
Executable File
65 lines
1.8 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""CASAN H6 hallucination signal detector.
|
|
|
|
Reads the quoted keyword markers from hallucination-tracking.yaml plus a set of
|
|
generic uncertainty markers, scans the agent output, and reports how many
|
|
hallucination signals were found. This turns hallucination-tracking.yaml from
|
|
dead config into a real, populated metric written to metrics.jsonl.
|
|
|
|
Usage: hallucination-scan.py <hallucination-tracking.yaml> <output-file>
|
|
Output (stdout): line 1 = integer signal count, line 2 = JSON list of matches.
|
|
"""
|
|
import json
|
|
import re
|
|
import sys
|
|
|
|
GENERIC_MARKERS = [
|
|
"maybe", "might be incorrect", "i am not sure", "uncertain",
|
|
"i think", "probably", "as far as i know",
|
|
]
|
|
|
|
|
|
def load_keywords(path):
|
|
keywords = []
|
|
try:
|
|
with open(path, encoding="utf-8") as fh:
|
|
for line in fh:
|
|
# Quoted list items are the hallucination keyword markers
|
|
# (unquoted list items are structured signal names, not text).
|
|
m = re.match(r'\s*-\s*"(.+)"\s*$', line)
|
|
if m:
|
|
keywords.append(m.group(1))
|
|
except OSError:
|
|
pass
|
|
return keywords
|
|
|
|
|
|
def main():
|
|
if len(sys.argv) < 3:
|
|
print(0)
|
|
print("[]")
|
|
return
|
|
keywords = load_keywords(sys.argv[1]) + GENERIC_MARKERS
|
|
try:
|
|
with open(sys.argv[2], encoding="utf-8") as fh:
|
|
text = fh.read().lower()
|
|
except OSError:
|
|
print(0)
|
|
print("[]")
|
|
return
|
|
matched = []
|
|
for kw in keywords:
|
|
k = kw.lower()
|
|
if not k:
|
|
continue
|
|
count = text.count(k)
|
|
if count:
|
|
matched.append({"marker": kw, "count": count})
|
|
total = sum(m["count"] for m in matched)
|
|
print(total)
|
|
print(json.dumps(matched))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|