CASAN/AINative_OKR_CASAN5/packages/casan-harness/scripts/bash/context-compress.py

#!/usr/bin/env python3
"""CASAN-native token-killer (Plan-08 Track 3).

A deterministic tool-output compressor written for CASAN — NOT a wrapper around
RTK. It reduces token count of long command/tool output before it enters model
context, while (a) always preserving must-keep lines, (b) never compressing on
failure (raw passthrough for debugging, RTK-style tee), and (c) reporting the
token savings for H6 telemetry.

Modes:
  dedup       collapse consecutive duplicate lines with an (xN) counter
  extractive  keep only important lines (errors/failures/warnings) + must-keep
  structural  dedup + keep summary/important/must-keep lines (for test/log output)

Governance note: this runs AFTER `H4 scan raw` + `H5 hash raw` and BEFORE
`H4 scan compressed` in the Plan-08 pipeline; it is deterministic and needs no
model, so it cannot be used as a path to evade H4.
"""
import argparse
import json
import os
import re
import sys


def compression_enabled() -> bool:
    """Read the effective `compression.enabled` from the control-plane settings
    store (the harness-owned governed settings). Absent/invalid ⇒ enabled (default).
    This is how a Control Plane setting change actually governs the harness."""
    store_file = os.environ.get(
        "CASAN_CP_STORE_FILE",
        os.path.join(
            os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")),
            ".specify/level5/control-plane-settings.json",
        ),
    )
    if not os.path.isfile(store_file):
        return True
    try:
        data = json.load(open(store_file, encoding="utf-8"))
        setting = data.get("settings", {}).get("compression.enabled")
        return True if setting is None else bool(setting["value"])
    except (OSError, ValueError, KeyError, TypeError):
        return True


IMPORTANT_RE = re.compile(
    r"\b(error|errors|fail|failed|failure|failing|exception|panic|denied|blocked|warn|warning)\b",
    re.IGNORECASE,
)
SUMMARY_RE = re.compile(r"\b(\d+)\s+(pass|passed|fail|failed|tests?|errors?|warnings?)\b", re.IGNORECASE)


def estimate_tokens(text: str) -> int:
    return len(text.split())


# SEC-09 (M-10): bound input size (DoS) and read fail-closed. Non-UTF8 degrades via
# errors="replace" instead of crashing; oversize/unreadable input exits non-zero and
# emits nothing (never a crash traceback, never silent truncation).
MAX_BYTES = int(os.environ.get("CASAN_MAX_INPUT_BYTES", str(2 * 1024 * 1024)))


def read_capped(src: str) -> str:
    try:
        if src == "-":
            data = sys.stdin.buffer.read(MAX_BYTES + 1)
        else:
            with open(src, "rb") as fh:
                data = fh.read(MAX_BYTES + 1)
    except OSError as exc:
        print(f"COMPRESS_FAIL unreadable_input: {exc}", file=sys.stderr)
        raise SystemExit(1)
    if len(data) > MAX_BYTES:
        print(f"COMPRESS_FAIL input_exceeds_cap({MAX_BYTES}B) fail-closed", file=sys.stderr)
        raise SystemExit(1)
    return data.decode("utf-8", errors="replace")


def load_patterns(path: str):
    if not path:
        return []
    try:
        with open(path, encoding="utf-8", errors="replace") as fh:
            return [line.strip() for line in fh if line.strip()]
    except OSError as exc:
        print(f"COMPRESS_FAIL must_keep_file_unreadable: {exc}", file=sys.stderr)
        raise SystemExit(1)


def is_must_keep(line: str, patterns) -> bool:
    return any(re.search(p, line) for p in patterns)


def dedup(lines):
    out = []
    i = 0
    n = len(lines)
    while i < n:
        j = i
        while j + 1 < n and lines[j + 1] == lines[i]:
            j += 1
        count = j - i + 1
        out.append(lines[i] if count == 1 else f"{lines[i]}  (x{count})")
        i = j + 1
    return out


def compress(text: str, mode: str, must):
    lines = text.split("\n")
    if mode == "dedup":
        return dedup(lines)
    if mode == "extractive":
        return [ln for ln in lines if IMPORTANT_RE.search(ln) or is_must_keep(ln, must)]
    if mode == "structural":
        kept = [
            ln
            for ln in lines
            if IMPORTANT_RE.search(ln) or SUMMARY_RE.search(ln) or is_must_keep(ln, must)
        ]
        return dedup(kept)
    raise ValueError(f"unknown mode: {mode}")


def main() -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("--mode", choices=["dedup", "extractive", "structural"], default="structural")
    ap.add_argument("--input", default="-", help="input file or - for stdin")
    ap.add_argument("--must-keep-file", default="", help="file with one must-keep regex per line")
    ap.add_argument("--failed", action="store_true", help="raw passthrough (tee) when the command failed")
    ap.add_argument(
        "--respect-policy",
        action="store_true",
        help="honor control-plane `compression.enabled`; if disabled, pass raw through",
    )
    ap.add_argument(
        "--require-must-keep-file",
        default="",
        help="verify every pattern in this file still appears; exit 1 (gate) if any is missing",
    )
    args = ap.parse_args()

    raw = read_capped(args.input)
    must = load_patterns(args.must_keep_file)

    if args.failed:
        # RTK-style tee: never compress failing output; keep raw for debugging.
        out_text = raw
        mode_used = "passthrough"
    elif args.respect_policy and not compression_enabled():
        # Control-plane setting governs the harness: compression disabled ⇒ raw.
        out_text = raw
        mode_used = "policy-disabled"
    else:
        out_lines = compress(raw, args.mode, must)
        out_text = "\n".join(out_lines)
        mode_used = args.mode

    in_tokens = estimate_tokens(raw)
    out_tokens = estimate_tokens(out_text)
    saved = in_tokens - out_tokens
    ratio = round(out_tokens / in_tokens, 4) if in_tokens else 1.0

    verify_patterns = load_patterns(args.require_must_keep_file)
    missing = [p for p in verify_patterns if not re.search(p, out_text)]

    sys.stdout.write(out_text)
    if not out_text.endswith("\n"):
        sys.stdout.write("\n")
    print(
        f"COMPRESS mode={mode_used} in_tokens={in_tokens} out_tokens={out_tokens} "
        f"saved={saved} ratio={ratio} must_keep_missing={len(missing)}",
        file=sys.stderr,
    )
    if missing:
        print(f"COMPRESS_MUST_KEEP_DROPPED {','.join(missing)}", file=sys.stderr)
        return 1
    return 0


if __name__ == "__main__":
    raise SystemExit(main())