#!/usr/bin/env python3 """CASAN H6 hallucination signal detector. Reads the quoted keyword markers from hallucination-tracking.yaml plus a set of generic uncertainty markers, scans the agent output, and reports how many hallucination signals were found. This turns hallucination-tracking.yaml from dead config into a real, populated metric written to metrics.jsonl. Usage: hallucination-scan.py Output (stdout): line 1 = integer signal count, line 2 = JSON list of matches. """ import json import re import sys GENERIC_MARKERS = [ "maybe", "might be incorrect", "i am not sure", "uncertain", "i think", "probably", "as far as i know", ] def load_keywords(path): keywords = [] try: with open(path, encoding="utf-8") as fh: for line in fh: # Quoted list items are the hallucination keyword markers # (unquoted list items are structured signal names, not text). m = re.match(r'\s*-\s*"(.+)"\s*$', line) if m: keywords.append(m.group(1)) except OSError: pass return keywords def main(): if len(sys.argv) < 3: print(0) print("[]") return keywords = load_keywords(sys.argv[1]) + GENERIC_MARKERS try: with open(sys.argv[2], encoding="utf-8") as fh: text = fh.read().lower() except OSError: print(0) print("[]") return matched = [] for kw in keywords: k = kw.lower() if not k: continue count = text.count(k) if count: matched.append({"marker": kw, "count": count}) total = sum(m["count"] for m in matched) print(total) print(json.dumps(matched)) if __name__ == "__main__": main()