196 lines
6.7 KiB
Python
196 lines
6.7 KiB
Python
#!/usr/bin/env python3
|
|
"""CASAN-native token-killer (Plan-08 Track 3).
|
|
|
|
A deterministic tool-output compressor written for CASAN — NOT a wrapper around
|
|
RTK. It reduces token count of long command/tool output before it enters model
|
|
context, while (a) always preserving must-keep lines, (b) never compressing on
|
|
failure (raw passthrough for debugging, RTK-style tee), and (c) reporting the
|
|
token savings for H6 telemetry.
|
|
|
|
Modes:
|
|
dedup collapse consecutive duplicate lines with an (xN) counter
|
|
extractive keep only important lines (errors/failures/warnings) + must-keep
|
|
structural dedup + keep summary/important/must-keep lines (for test/log output)
|
|
|
|
Governance note: this runs AFTER `H4 scan raw` + `H5 hash raw` and BEFORE
|
|
`H4 scan compressed` in the Plan-08 pipeline; it is deterministic and needs no
|
|
model, so it cannot be used as a path to evade H4.
|
|
"""
|
|
import argparse
|
|
import json
|
|
import os
|
|
|
|
|
|
def _casan_app_root():
|
|
# Plan-01: walk UP for the `.specify` state marker (harness code lives in
|
|
# packages/casan-harness/, so a fixed __file__ depth would mis-root).
|
|
_d = os.path.abspath(os.path.dirname(__file__))
|
|
_p = _d
|
|
while _p != os.path.dirname(_p):
|
|
if os.path.isdir(os.path.join(_p, ".specify")) or os.path.isdir(os.path.join(_p, "packages/casan-harness")):
|
|
return _p
|
|
_p = os.path.dirname(_p)
|
|
return os.path.abspath(os.path.join(_d, "..", "..", ".."))
|
|
|
|
import re
|
|
import sys
|
|
|
|
|
|
def compression_enabled() -> bool:
|
|
"""Read the effective `compression.enabled` from the control-plane settings
|
|
store (the harness-owned governed settings). Absent/invalid ⇒ enabled (default).
|
|
This is how a Control Plane setting change actually governs the harness."""
|
|
store_file = os.environ.get(
|
|
"CASAN_CP_STORE_FILE",
|
|
os.path.join(
|
|
_casan_app_root(),
|
|
".specify/level5/control-plane-settings.json",
|
|
),
|
|
)
|
|
if not os.path.isfile(store_file):
|
|
return True
|
|
try:
|
|
data = json.load(open(store_file, encoding="utf-8"))
|
|
setting = data.get("settings", {}).get("compression.enabled")
|
|
return True if setting is None else bool(setting["value"])
|
|
except (OSError, ValueError, KeyError, TypeError):
|
|
return True
|
|
|
|
|
|
IMPORTANT_RE = re.compile(
|
|
r"\b(error|errors|fail|failed|failure|failing|exception|panic|denied|blocked|warn|warning)\b",
|
|
re.IGNORECASE,
|
|
)
|
|
SUMMARY_RE = re.compile(r"\b(\d+)\s+(pass|passed|fail|failed|tests?|errors?|warnings?)\b", re.IGNORECASE)
|
|
|
|
|
|
def estimate_tokens(text: str) -> int:
|
|
return len(text.split())
|
|
|
|
|
|
# SEC-09 (M-10): bound input size (DoS) and read fail-closed. Non-UTF8 degrades via
|
|
# errors="replace" instead of crashing; oversize/unreadable input exits non-zero and
|
|
# emits nothing (never a crash traceback, never silent truncation).
|
|
MAX_BYTES = int(os.environ.get("CASAN_MAX_INPUT_BYTES", str(2 * 1024 * 1024)))
|
|
|
|
|
|
def read_capped(src: str) -> str:
|
|
try:
|
|
if src == "-":
|
|
data = sys.stdin.buffer.read(MAX_BYTES + 1)
|
|
else:
|
|
with open(src, "rb") as fh:
|
|
data = fh.read(MAX_BYTES + 1)
|
|
except OSError as exc:
|
|
print(f"COMPRESS_FAIL unreadable_input: {exc}", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
if len(data) > MAX_BYTES:
|
|
print(f"COMPRESS_FAIL input_exceeds_cap({MAX_BYTES}B) fail-closed", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
return data.decode("utf-8", errors="replace")
|
|
|
|
|
|
def load_patterns(path: str):
|
|
if not path:
|
|
return []
|
|
try:
|
|
with open(path, encoding="utf-8", errors="replace") as fh:
|
|
return [line.strip() for line in fh if line.strip()]
|
|
except OSError as exc:
|
|
print(f"COMPRESS_FAIL must_keep_file_unreadable: {exc}", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
|
|
|
|
def is_must_keep(line: str, patterns) -> bool:
|
|
return any(re.search(p, line) for p in patterns)
|
|
|
|
|
|
def dedup(lines):
|
|
out = []
|
|
i = 0
|
|
n = len(lines)
|
|
while i < n:
|
|
j = i
|
|
while j + 1 < n and lines[j + 1] == lines[i]:
|
|
j += 1
|
|
count = j - i + 1
|
|
out.append(lines[i] if count == 1 else f"{lines[i]} (x{count})")
|
|
i = j + 1
|
|
return out
|
|
|
|
|
|
def compress(text: str, mode: str, must):
|
|
lines = text.split("\n")
|
|
if mode == "dedup":
|
|
return dedup(lines)
|
|
if mode == "extractive":
|
|
return [ln for ln in lines if IMPORTANT_RE.search(ln) or is_must_keep(ln, must)]
|
|
if mode == "structural":
|
|
kept = [
|
|
ln
|
|
for ln in lines
|
|
if IMPORTANT_RE.search(ln) or SUMMARY_RE.search(ln) or is_must_keep(ln, must)
|
|
]
|
|
return dedup(kept)
|
|
raise ValueError(f"unknown mode: {mode}")
|
|
|
|
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--mode", choices=["dedup", "extractive", "structural"], default="structural")
|
|
ap.add_argument("--input", default="-", help="input file or - for stdin")
|
|
ap.add_argument("--must-keep-file", default="", help="file with one must-keep regex per line")
|
|
ap.add_argument("--failed", action="store_true", help="raw passthrough (tee) when the command failed")
|
|
ap.add_argument(
|
|
"--respect-policy",
|
|
action="store_true",
|
|
help="honor control-plane `compression.enabled`; if disabled, pass raw through",
|
|
)
|
|
ap.add_argument(
|
|
"--require-must-keep-file",
|
|
default="",
|
|
help="verify every pattern in this file still appears; exit 1 (gate) if any is missing",
|
|
)
|
|
args = ap.parse_args()
|
|
|
|
raw = read_capped(args.input)
|
|
must = load_patterns(args.must_keep_file)
|
|
|
|
if args.failed:
|
|
# RTK-style tee: never compress failing output; keep raw for debugging.
|
|
out_text = raw
|
|
mode_used = "passthrough"
|
|
elif args.respect_policy and not compression_enabled():
|
|
# Control-plane setting governs the harness: compression disabled ⇒ raw.
|
|
out_text = raw
|
|
mode_used = "policy-disabled"
|
|
else:
|
|
out_lines = compress(raw, args.mode, must)
|
|
out_text = "\n".join(out_lines)
|
|
mode_used = args.mode
|
|
|
|
in_tokens = estimate_tokens(raw)
|
|
out_tokens = estimate_tokens(out_text)
|
|
saved = in_tokens - out_tokens
|
|
ratio = round(out_tokens / in_tokens, 4) if in_tokens else 1.0
|
|
|
|
verify_patterns = load_patterns(args.require_must_keep_file)
|
|
missing = [p for p in verify_patterns if not re.search(p, out_text)]
|
|
|
|
sys.stdout.write(out_text)
|
|
if not out_text.endswith("\n"):
|
|
sys.stdout.write("\n")
|
|
print(
|
|
f"COMPRESS mode={mode_used} in_tokens={in_tokens} out_tokens={out_tokens} "
|
|
f"saved={saved} ratio={ratio} must_keep_missing={len(missing)}",
|
|
file=sys.stderr,
|
|
)
|
|
if missing:
|
|
print(f"COMPRESS_MUST_KEEP_DROPPED {','.join(missing)}", file=sys.stderr)
|
|
return 1
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|