feat(plan-01): Phase 1 — relocate harness code to packages/casan-harness (symlink facade)
Physically move the pure-code subtrees out of .specify into the package, leaving compat symlinks at the old .specify/<dir> paths so every existing reference (internal CASAN_HARNESS_ROOT + external CI/docker/mjs) keeps resolving. Runtime state stays put. Moved (git mv): scripts/ tests/ security/ templates/ config/ governance/ memory/ .specify/<dir> -> packages/casan-harness/<dir> (+ .specify/<dir> symlink) Stays in .specify (state/governance/domain, handled later): logs/ agentops/ level5/ init-options.json traceability-map.json Python `.resolve()` self-location followed the compat symlink into packages and lost the app root; generate-casan-demo-context.py, generate-agentops-dashboard.py and dashboard-server.py now walk UP for the `.specify` state marker instead of a fixed parent depth (fixes "missing trace files" in run-casan4). Full gate: PASS=64 FAIL=0 SKIP=3 (CASAN_CI_STEP_TIMEOUT_SEC=1200 — track-a ~450s runs close to the 600s default and can tip over under load; this is timing variance, not a regression — it passed cleanly with headroom). Runtime log/audit artifacts kept unstaged. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
thanhnv
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
2c765c9a45
commit
664bd1f00c
+236
@@ -0,0 +1,236 @@
|
||||
#!/usr/bin/env python3
|
||||
"""CASAN Plan-09 — Evidence Pack builder (MVP).
|
||||
|
||||
Assembles a per-run proof pack from REAL on-disk logs/reports (summaries only —
|
||||
never raw secret/PII content) plus verifier statuses passed in by the bash
|
||||
wrapper, then writes a hash manifest binding every file in the pack.
|
||||
|
||||
Argv: <project_root> <run_id> <pack_dir>
|
||||
Env (verifier statuses from the wrapper):
|
||||
CASAN_EP_AUDIT, CASAN_EP_TOOLAUDIT, CASAN_EP_TELEMETRY — "<text>|<rc>"
|
||||
CASAN_EP_COST_RC — cost-spike rc
|
||||
CASAN_EP_FP_JSON — path to benign-fp-report.json (optional)
|
||||
|
||||
Prints "CERTIFIED|<true|false>|<reason>" on stdout.
|
||||
A run is CERTIFIED only when the required gates PASS and none was silently
|
||||
skipped (missing evidence => not certified, with the reason recorded).
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
def read_jsonl(path):
|
||||
rows = []
|
||||
try:
|
||||
for line in open(path, encoding="utf-8"):
|
||||
line = line.strip()
|
||||
if line:
|
||||
try:
|
||||
rows.append(json.loads(line))
|
||||
except ValueError:
|
||||
pass
|
||||
except OSError:
|
||||
pass
|
||||
return rows
|
||||
|
||||
|
||||
def status_rc(env_key):
|
||||
raw = os.environ.get(env_key, "|1")
|
||||
text, _, rc = raw.rpartition("|")
|
||||
try:
|
||||
return text, int(rc)
|
||||
except ValueError:
|
||||
return text, 1
|
||||
|
||||
|
||||
def main():
|
||||
root, run_id, pack_dir = sys.argv[1:4]
|
||||
os.makedirs(pack_dir, exist_ok=True)
|
||||
logs = os.path.join(root, ".specify", "logs")
|
||||
|
||||
reports = {}
|
||||
|
||||
# H1 context
|
||||
reports["h1-context-report.json"] = {
|
||||
"harness": "H1-context", "run_id": run_id,
|
||||
"context_yaml": os.path.exists(os.path.join(root, "docs/output/output_logs/casan-demo/pipeline-context.yaml")),
|
||||
"note": "path/artifact validation performed by context-validate.sh at run time",
|
||||
}
|
||||
|
||||
# H2 tool audit
|
||||
tool_rows = read_jsonl(os.path.join(logs, "audit", "tool-calls.jsonl"))
|
||||
ta_text, ta_rc = status_rc("CASAN_EP_TOOLAUDIT")
|
||||
reports["h2-tool-audit.json"] = {
|
||||
"harness": "H2-tool", "run_id": run_id, "records": len(tool_rows),
|
||||
"denied": sum(1 for r in tool_rows if r.get("decision") == "denied"),
|
||||
"approved": sum(1 for r in tool_rows if r.get("decision") == "approved"),
|
||||
"chain_status": ta_text, "chain_ok": ta_rc == 0,
|
||||
}
|
||||
|
||||
# H3 eval scorecard (best effort — reference known evidence)
|
||||
reports["h3-eval-scorecard.json"] = {
|
||||
"harness": "H3-eval", "run_id": run_id,
|
||||
"judge_gate_tests": os.path.exists(os.path.join(root, ".specify/tests/phase3-judge-gate-tests.sh")),
|
||||
"note": "judge-gate fail-before/fix cycle proven by phase3-judge-gate-tests.sh",
|
||||
}
|
||||
traceability_out = os.path.join(pack_dir, "traceability-matrix.json")
|
||||
traceability_script = os.path.join(root, ".specify/scripts/bash/traceability-matrix.py")
|
||||
traceability_rc = 1
|
||||
if os.path.isfile(traceability_script):
|
||||
traceability_rc = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
traceability_script,
|
||||
"--requirements",
|
||||
os.path.join(root, "docs/input/okr-requirement.md"),
|
||||
"--map",
|
||||
os.path.join(root, ".specify/traceability-map.json"),
|
||||
"--out",
|
||||
traceability_out,
|
||||
"--gate",
|
||||
],
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
).returncode
|
||||
|
||||
# H4 security
|
||||
sec_rows = read_jsonl(os.path.join(logs, "audit", "security.jsonl"))
|
||||
rule_types = {}
|
||||
for r in sec_rows:
|
||||
for k in ("status",):
|
||||
rule_types[r.get(k, "?")] = rule_types.get(r.get(k, "?"), 0) + 1
|
||||
reports["h4-security-report.json"] = {
|
||||
"harness": "H4-security", "run_id": run_id, "records": len(sec_rows),
|
||||
"by_status": rule_types,
|
||||
"blocked": sum(1 for r in sec_rows if r.get("status") == "blocked"),
|
||||
}
|
||||
|
||||
# H5 audit chain proof
|
||||
au_text, au_rc = status_rc("CASAN_EP_AUDIT")
|
||||
tel_text, tel_rc = status_rc("CASAN_EP_TELEMETRY")
|
||||
reports["h5-audit-chain-proof.json"] = {
|
||||
"harness": "H5-governance", "run_id": run_id,
|
||||
"audit_chain": au_text, "audit_chain_ok": au_rc == 0,
|
||||
"telemetry_integrity": tel_text, "telemetry_ok": tel_rc == 0,
|
||||
}
|
||||
|
||||
# H6 cost telemetry
|
||||
prov = read_jsonl(os.path.join(logs, "level5", "provider-usage.jsonl"))
|
||||
metrics = read_jsonl(os.path.join(logs, "cost", "metrics.jsonl"))
|
||||
cost_rc = int(os.environ.get("CASAN_EP_COST_RC", "3") or "3")
|
||||
total_tokens = sum(int(r.get("total_tokens", 0)) for r in prov if str(r.get("total_tokens", "")).isdigit())
|
||||
reports["h6-cost-telemetry.json"] = {
|
||||
"harness": "H6-agentops", "run_id": run_id,
|
||||
"provider_records": len(prov), "metric_records": len(metrics),
|
||||
"total_provider_tokens": total_tokens,
|
||||
"cost_spike_rc": cost_rc,
|
||||
"cost_spike_status": {0: "none", 2: "spike_detected", 3: "insufficient_data"}.get(cost_rc, "unknown"),
|
||||
}
|
||||
|
||||
# H7 orchestration (best effort)
|
||||
reports["h7-orchestration-report.json"] = {
|
||||
"harness": "H7-orchestration", "run_id": run_id,
|
||||
"rollback_log": os.path.exists(os.path.join(logs, "level5", "rollback-transactions.jsonl")),
|
||||
"note": "rollback/fallback/drift proven by adversarial + run-casan4 suites",
|
||||
}
|
||||
|
||||
# Red-team + benign/FP results
|
||||
fp_json = os.environ.get("CASAN_EP_FP_JSON", "")
|
||||
fp = None
|
||||
if fp_json and os.path.isfile(fp_json):
|
||||
try:
|
||||
fp = json.load(open(fp_json, encoding="utf-8"))
|
||||
except ValueError:
|
||||
fp = None
|
||||
vectors_path = os.path.join(root, ".specify/security/redteam-vectors.jsonl")
|
||||
vectors = read_jsonl(vectors_path)
|
||||
reports["redteam-result.json"] = {
|
||||
"run_id": run_id, "vectors_defined": len(vectors),
|
||||
"critical_vectors": sum(1 for v in vectors if v.get("severity") == "critical"),
|
||||
"adversarial_block_rate_pct": (fp or {}).get("adversarial", {}).get("block_rate_pct"),
|
||||
"critical_block_rate_pct": (fp or {}).get("critical", {}).get("block_rate_pct"),
|
||||
"note": "block rates from benign-fp-report (deterministic layer); full suites: phase1-track-a + phase2-track-c",
|
||||
}
|
||||
reports["benign-fp-report.json"] = fp or {"note": "benign-fp-report not present; run benign-fp-report.sh"}
|
||||
|
||||
# Write the hN reports.
|
||||
for name, data in reports.items():
|
||||
with open(os.path.join(pack_dir, name), "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
||||
|
||||
# ---- Certification decision ----
|
||||
reasons = []
|
||||
if not (au_rc == 0):
|
||||
reasons.append("audit_chain_not_valid")
|
||||
if not (tel_rc == 0):
|
||||
reasons.append("telemetry_integrity_not_verified")
|
||||
if cost_rc == 2:
|
||||
reasons.append("unresolved_cost_spike")
|
||||
if len(sec_rows) == 0:
|
||||
reasons.append("h4_security_not_exercised")
|
||||
if traceability_rc != 0:
|
||||
reasons.append("traceability_gate_failed")
|
||||
fp_ok = bool(fp) and fp.get("within_budget") is True
|
||||
if fp is None:
|
||||
reasons.append("benign_fp_report_missing(gate_skipped)")
|
||||
elif not fp_ok:
|
||||
reasons.append("benign_fp_budget_breached")
|
||||
certified = len(reasons) == 0
|
||||
|
||||
# ---- run summary ----
|
||||
run_summary = {
|
||||
"run_id": run_id, "pack_version": "1.0-mvp",
|
||||
"certified": certified, "certification_reasons": reasons or ["all_required_gates_passed"],
|
||||
"required_gates": ["H3-traceability", "H4-security", "H5-audit-chain", "H5-telemetry", "H6-cost", "benign-fp-budget"],
|
||||
"harness_reports": sorted(reports.keys()),
|
||||
}
|
||||
with open(os.path.join(pack_dir, "run-summary.json"), "w", encoding="utf-8") as f:
|
||||
json.dump(run_summary, f, indent=2, ensure_ascii=False)
|
||||
|
||||
# ---- decision log (human-readable) ----
|
||||
dl = [
|
||||
f"# CASAN Evidence Pack — Decision Log",
|
||||
f"", f"Run: `{run_id}` ", f"Certified: **{certified}** ",
|
||||
f"Reasons: {', '.join(run_summary['certification_reasons'])}", "",
|
||||
"## Gate outcomes", "",
|
||||
f"- H4 security: {reports['h4-security-report.json']['records']} records, "
|
||||
f"{reports['h4-security-report.json']['blocked']} blocked",
|
||||
f"- H5 audit chain: {au_text} (ok={au_rc == 0})",
|
||||
f"- H5 telemetry integrity: {tel_text} (ok={tel_rc == 0})",
|
||||
f"- H6 cost: {reports['h6-cost-telemetry.json']['cost_spike_status']}, "
|
||||
f"{total_tokens} provider tokens",
|
||||
f"- H2 tool audit: {reports['h2-tool-audit.json']['records']} records, "
|
||||
f"chain_ok={reports['h2-tool-audit.json']['chain_ok']}",
|
||||
f"- H3 traceability: ok={traceability_rc == 0}",
|
||||
f"- Red-team: {reports['redteam-result.json']['vectors_defined']} vectors "
|
||||
f"(block_rate={reports['redteam-result.json']['adversarial_block_rate_pct']}%)",
|
||||
"",
|
||||
"_Summaries only — no raw secret/PII content is copied into the pack._",
|
||||
]
|
||||
with open(os.path.join(pack_dir, "decision-log.md"), "w", encoding="utf-8") as f:
|
||||
f.write("\n".join(dl) + "\n")
|
||||
|
||||
# ---- artifact manifest: sha256 of every pack file (except the signature) ----
|
||||
manifest = {}
|
||||
for fn in sorted(os.listdir(pack_dir)):
|
||||
if fn in ("artifact-manifest.json", "evidence-pack.sig", "manifest-head.txt"):
|
||||
continue
|
||||
fp_path = os.path.join(pack_dir, fn)
|
||||
if os.path.isfile(fp_path):
|
||||
with open(fp_path, "rb") as f:
|
||||
manifest[fn] = hashlib.sha256(f.read()).hexdigest()
|
||||
canonical = json.dumps(manifest, sort_keys=True, separators=(",", ":"))
|
||||
head = hashlib.sha256(canonical.encode()).hexdigest()
|
||||
with open(os.path.join(pack_dir, "artifact-manifest.json"), "w", encoding="utf-8") as f:
|
||||
json.dump({"files": manifest, "manifest_head": head}, f, indent=2)
|
||||
with open(os.path.join(pack_dir, "manifest-head.txt"), "w", encoding="utf-8") as f:
|
||||
f.write(head)
|
||||
|
||||
print(f"CERTIFIED|{str(certified).lower()}|{','.join(run_summary['certification_reasons'])}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user