261 lines
11 KiB
Python
Executable File
261 lines
11 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""CASAN Plan-09 — Evidence Pack builder (MVP).
|
|
|
|
Assembles a per-run proof pack from REAL on-disk logs/reports (summaries only —
|
|
never raw secret/PII content) plus verifier statuses passed in by the bash
|
|
wrapper, then writes a hash manifest binding every file in the pack.
|
|
|
|
Argv: <project_root> <run_id> <pack_dir>
|
|
Env (verifier statuses from the wrapper):
|
|
CASAN_EP_AUDIT, CASAN_EP_TOOLAUDIT, CASAN_EP_TELEMETRY — "<text>|<rc>"
|
|
CASAN_EP_COST_RC — cost-spike rc
|
|
CASAN_EP_FP_JSON — path to benign-fp-report.json (optional)
|
|
|
|
Prints "CERTIFIED|<true|false>|<reason>" on stdout.
|
|
A run is CERTIFIED only when the required gates PASS and none was silently
|
|
skipped (missing evidence => not certified, with the reason recorded).
|
|
"""
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
|
|
from project_manifest import load as load_project_manifest
|
|
|
|
# Plan-01: this script lives at <harness>/scripts/bash/; the harness root is two levels up.
|
|
_HARNESS = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", ".."))
|
|
|
|
|
|
def read_jsonl(path):
|
|
rows = []
|
|
try:
|
|
for line in open(path, encoding="utf-8"):
|
|
line = line.strip()
|
|
if line:
|
|
try:
|
|
rows.append(json.loads(line))
|
|
except ValueError:
|
|
pass
|
|
except OSError:
|
|
pass
|
|
return rows
|
|
|
|
|
|
def status_rc(env_key):
|
|
raw = os.environ.get(env_key, "|1")
|
|
text, _, rc = raw.rpartition("|")
|
|
try:
|
|
return text, int(rc)
|
|
except ValueError:
|
|
return text, 1
|
|
|
|
|
|
def main():
|
|
root, run_id, pack_dir = sys.argv[1:4]
|
|
try:
|
|
project = load_project_manifest(root)
|
|
domain_root = os.path.join(root, project["domain_root"])
|
|
requirements_path = os.path.join(root, project["requirements"])
|
|
feature_id = project["feature"]["id"]
|
|
except (OSError, ValueError):
|
|
domain_root = os.environ.get("CASAN_DOMAIN_ROOT", os.path.join(root, "apps/okr/domain"))
|
|
requirements_path = os.path.join(domain_root, "input", "requirement.md")
|
|
if not os.path.isfile(requirements_path):
|
|
requirements_path = os.path.join(domain_root, "input", "okr-requirement.md")
|
|
feature_id = "casan-demo"
|
|
os.makedirs(pack_dir, exist_ok=True)
|
|
logs = os.path.join(root, ".specify", "logs")
|
|
|
|
reports = {}
|
|
|
|
# H1 context
|
|
reports["h1-context-report.json"] = {
|
|
"harness": "H1-context", "run_id": run_id,
|
|
"context_yaml": os.path.exists(os.path.join(root, "docs/output/output_logs", feature_id, "pipeline-context.yaml")),
|
|
"note": "path/artifact validation performed by context-validate.sh at run time",
|
|
}
|
|
|
|
# H2 tool audit
|
|
tool_rows = read_jsonl(os.path.join(logs, "audit", "tool-calls.jsonl"))
|
|
ta_text, ta_rc = status_rc("CASAN_EP_TOOLAUDIT")
|
|
reports["h2-tool-audit.json"] = {
|
|
"harness": "H2-tool", "run_id": run_id, "records": len(tool_rows),
|
|
"denied": sum(1 for r in tool_rows if r.get("decision") == "denied"),
|
|
"approved": sum(1 for r in tool_rows if r.get("decision") == "approved"),
|
|
"chain_status": ta_text, "chain_ok": ta_rc == 0,
|
|
}
|
|
|
|
# H3 eval scorecard (best effort — reference known evidence)
|
|
reports["h3-eval-scorecard.json"] = {
|
|
"harness": "H3-eval", "run_id": run_id,
|
|
"judge_gate_tests": os.path.exists(os.path.join(_HARNESS, "tests/phase3-judge-gate-tests.sh")),
|
|
"note": "judge-gate fail-before/fix cycle proven by phase3-judge-gate-tests.sh",
|
|
}
|
|
traceability_out = os.path.join(pack_dir, "traceability-matrix.json")
|
|
traceability_script = os.path.join(os.path.dirname(os.path.abspath(__file__)), "traceability-matrix.py")
|
|
traceability_rc = 1
|
|
if os.path.isfile(traceability_script):
|
|
traceability_rc = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
traceability_script,
|
|
"--requirements",
|
|
requirements_path,
|
|
"--map",
|
|
os.path.join(domain_root, "traceability-map.json"),
|
|
"--out",
|
|
traceability_out,
|
|
"--gate",
|
|
],
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
).returncode
|
|
|
|
# H4 security
|
|
sec_rows = read_jsonl(os.path.join(logs, "audit", "security.jsonl"))
|
|
rule_types = {}
|
|
for r in sec_rows:
|
|
for k in ("status",):
|
|
rule_types[r.get(k, "?")] = rule_types.get(r.get(k, "?"), 0) + 1
|
|
reports["h4-security-report.json"] = {
|
|
"harness": "H4-security", "run_id": run_id, "records": len(sec_rows),
|
|
"by_status": rule_types,
|
|
"blocked": sum(1 for r in sec_rows if r.get("status") == "blocked"),
|
|
}
|
|
|
|
# H5 audit chain proof
|
|
au_text, au_rc = status_rc("CASAN_EP_AUDIT")
|
|
tel_text, tel_rc = status_rc("CASAN_EP_TELEMETRY")
|
|
reports["h5-audit-chain-proof.json"] = {
|
|
"harness": "H5-governance", "run_id": run_id,
|
|
"audit_chain": au_text, "audit_chain_ok": au_rc == 0,
|
|
"telemetry_integrity": tel_text, "telemetry_ok": tel_rc == 0,
|
|
}
|
|
|
|
# H6 telemetry — use the same canonical sources as the Control Panel/report API.
|
|
provider_path = os.environ.get(
|
|
"CASAN_TELEMETRY_PROVIDER_LOG",
|
|
os.environ.get("CASAN_CP_PROVIDER_USAGE", os.path.join(logs, "level5", "provider-usage.jsonl")),
|
|
)
|
|
metrics_path = os.environ.get(
|
|
"CASAN_TELEMETRY_METRICS_LOG",
|
|
os.environ.get("CASAN_DASHBOARD_METRICS", os.path.join(logs, "cost", "metrics.jsonl")),
|
|
)
|
|
prov = read_jsonl(provider_path)
|
|
metrics = read_jsonl(metrics_path)
|
|
cost_rc = int(os.environ.get("CASAN_EP_COST_RC", "3") or "3")
|
|
total_tokens = sum(int(r.get("total_tokens", 0)) for r in prov if str(r.get("total_tokens", "")).isdigit())
|
|
reports["h6-cost-telemetry.json"] = {
|
|
"harness": "H6-agentops", "run_id": run_id,
|
|
"provider_records": len(prov), "metric_records": len(metrics),
|
|
"total_provider_tokens": total_tokens,
|
|
"cost_spike_rc": cost_rc,
|
|
"cost_spike_status": {0: "none", 2: "spike_detected", 3: "insufficient_data"}.get(cost_rc, "unknown"),
|
|
}
|
|
|
|
# H7 orchestration (best effort)
|
|
reports["h7-orchestration-report.json"] = {
|
|
"harness": "H7-orchestration", "run_id": run_id,
|
|
"rollback_log": os.path.exists(os.path.join(logs, "level5", "rollback-transactions.jsonl")),
|
|
"note": "rollback/fallback/drift proven by adversarial + run-casan4 suites",
|
|
}
|
|
|
|
# Red-team + benign/FP results
|
|
fp_json = os.environ.get("CASAN_EP_FP_JSON", "")
|
|
fp = None
|
|
if fp_json and os.path.isfile(fp_json):
|
|
try:
|
|
fp = json.load(open(fp_json, encoding="utf-8"))
|
|
except ValueError:
|
|
fp = None
|
|
vectors_path = os.path.join(domain_root, "corpus", "redteam-vectors.jsonl")
|
|
vectors = read_jsonl(vectors_path)
|
|
reports["redteam-result.json"] = {
|
|
"run_id": run_id, "vectors_defined": len(vectors),
|
|
"critical_vectors": sum(1 for v in vectors if v.get("severity") == "critical"),
|
|
"adversarial_block_rate_pct": (fp or {}).get("adversarial", {}).get("block_rate_pct"),
|
|
"critical_block_rate_pct": (fp or {}).get("critical", {}).get("block_rate_pct"),
|
|
"note": "block rates from benign-fp-report (deterministic layer); full suites: phase1-track-a + phase2-track-c",
|
|
}
|
|
reports["benign-fp-report.json"] = fp or {"note": "benign-fp-report not present; run benign-fp-report.sh"}
|
|
|
|
# Write the hN reports.
|
|
for name, data in reports.items():
|
|
with open(os.path.join(pack_dir, name), "w", encoding="utf-8") as f:
|
|
json.dump(data, f, indent=2, ensure_ascii=False)
|
|
|
|
# ---- Certification decision ----
|
|
reasons = []
|
|
if not (au_rc == 0):
|
|
reasons.append("audit_chain_not_valid")
|
|
if not (tel_rc == 0):
|
|
reasons.append("telemetry_integrity_not_verified")
|
|
if cost_rc == 2:
|
|
reasons.append("unresolved_cost_spike")
|
|
if len(sec_rows) == 0:
|
|
reasons.append("h4_security_not_exercised")
|
|
if traceability_rc != 0:
|
|
reasons.append("traceability_gate_failed")
|
|
fp_ok = bool(fp) and fp.get("within_budget") is True
|
|
if fp is None:
|
|
reasons.append("benign_fp_report_missing(gate_skipped)")
|
|
elif not fp_ok:
|
|
reasons.append("benign_fp_budget_breached")
|
|
certified = len(reasons) == 0
|
|
|
|
# ---- run summary ----
|
|
run_summary = {
|
|
"run_id": run_id, "pack_version": "1.0-mvp",
|
|
"certified": certified, "certification_reasons": reasons or ["all_required_gates_passed"],
|
|
"required_gates": ["H3-traceability", "H4-security", "H5-audit-chain", "H5-telemetry", "H6-cost", "benign-fp-budget"],
|
|
"harness_reports": sorted(reports.keys()),
|
|
}
|
|
with open(os.path.join(pack_dir, "run-summary.json"), "w", encoding="utf-8") as f:
|
|
json.dump(run_summary, f, indent=2, ensure_ascii=False)
|
|
|
|
# ---- decision log (human-readable) ----
|
|
dl = [
|
|
f"# CASAN Evidence Pack — Decision Log",
|
|
f"", f"Run: `{run_id}` ", f"Certified: **{certified}** ",
|
|
f"Reasons: {', '.join(run_summary['certification_reasons'])}", "",
|
|
"## Gate outcomes", "",
|
|
f"- H4 security: {reports['h4-security-report.json']['records']} records, "
|
|
f"{reports['h4-security-report.json']['blocked']} blocked",
|
|
f"- H5 audit chain: {au_text} (ok={au_rc == 0})",
|
|
f"- H5 telemetry integrity: {tel_text} (ok={tel_rc == 0})",
|
|
f"- H6 cost: {reports['h6-cost-telemetry.json']['cost_spike_status']}, "
|
|
f"{total_tokens} provider tokens",
|
|
f"- H2 tool audit: {reports['h2-tool-audit.json']['records']} records, "
|
|
f"chain_ok={reports['h2-tool-audit.json']['chain_ok']}",
|
|
f"- H3 traceability: ok={traceability_rc == 0}",
|
|
f"- Red-team: {reports['redteam-result.json']['vectors_defined']} vectors "
|
|
f"(block_rate={reports['redteam-result.json']['adversarial_block_rate_pct']}%)",
|
|
"",
|
|
"_Summaries only — no raw secret/PII content is copied into the pack._",
|
|
]
|
|
with open(os.path.join(pack_dir, "decision-log.md"), "w", encoding="utf-8") as f:
|
|
f.write("\n".join(dl) + "\n")
|
|
|
|
# ---- artifact manifest: sha256 of every pack file (except the signature) ----
|
|
manifest = {}
|
|
for fn in sorted(os.listdir(pack_dir)):
|
|
if fn in ("artifact-manifest.json", "evidence-pack.sig", "manifest-head.txt"):
|
|
continue
|
|
fp_path = os.path.join(pack_dir, fn)
|
|
if os.path.isfile(fp_path):
|
|
with open(fp_path, "rb") as f:
|
|
manifest[fn] = hashlib.sha256(f.read()).hexdigest()
|
|
canonical = json.dumps(manifest, sort_keys=True, separators=(",", ":"))
|
|
head = hashlib.sha256(canonical.encode()).hexdigest()
|
|
with open(os.path.join(pack_dir, "artifact-manifest.json"), "w", encoding="utf-8") as f:
|
|
json.dump({"files": manifest, "manifest_head": head}, f, indent=2)
|
|
with open(os.path.join(pack_dir, "manifest-head.txt"), "w", encoding="utf-8") as f:
|
|
f.write(head)
|
|
|
|
print(f"CERTIFIED|{str(certified).lower()}|{','.join(run_summary['certification_reasons'])}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|