#!/usr/bin/env bash set -uo pipefail # CASAN Plan-09 — Evidence Pack MVP tests. # # Proves: a pack is created from real run evidence; verification is tamper- # evident (changing ANY packed file fails); a signed pack cannot be re-forged # without the key; and a "Certified run" is only asserted when the required # gates pass and none was silently skipped. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../scripts/bash/casan-paths.sh" PROJECT_ROOT="$CASAN_APP_ROOT" S="$CASAN_HARNESS_ROOT/scripts/bash" EP="$S/evidence-pack.sh" WORK="$(mktemp -d)" trap 'rm -rf "$WORK"' EXIT PASS=0; FAIL=0 pass() { echo "PASS: $1"; PASS=$((PASS + 1)); } fail() { echo "FAIL: $1"; FAIL=$((FAIL + 1)); } # Self-contained signing key so the signed-pack test never depends on the # off-repo production key. openssl genrsa -out "$WORK/priv.pem" 2048 2>/dev/null openssl rsa -in "$WORK/priv.pem" -pubout -out "$WORK/pub.pem" 2>/dev/null export CASAN_AUDIT_PRIV="$WORK/priv.pem" CASAN_AUDIT_PUB="$WORK/pub.pem" RID="ep-test-$$" PACKDIR="$PROJECT_ROOT/docs/output/casan/evidence-packs/$RID" cleanup_pack() { rm -rf "$PACKDIR"; } trap 'rm -rf "$WORK"; cleanup_pack' EXIT echo "===== Evidence Pack: create + intact verify (signed) =====" if bash "$EP" pack "$RID" > "$WORK/pack.out" 2>&1; then pass "pack created" else cat "$WORK/pack.out"; fail "pack creation failed" fi # Standard files present MISSING=0 for f in run-summary.json h1-context-report.json h2-tool-audit.json h3-eval-scorecard.json \ h4-security-report.json h5-audit-chain-proof.json h6-cost-telemetry.json \ h7-orchestration-report.json redteam-result.json benign-fp-report.json \ artifact-manifest.json traceability-matrix.json decision-log.md; do [[ -f "$PACKDIR/$f" ]] || { echo " missing $f"; MISSING=$((MISSING+1)); } done [[ "$MISSING" -eq 0 ]] && pass "pack contains all 13 standard evidence files" || fail "pack missing $MISSING files" [[ -f "$PACKDIR/evidence-pack.sig" ]] && pass "pack is signed (evidence-pack.sig present)" || fail "pack signature missing" rc=0; bash "$EP" verify-pack "$RID" >/dev/null 2>&1 || rc=$? [[ "$rc" -eq 0 ]] && pass "verify-pack: intact signed pack is VALID" || fail "verify-pack rejected an intact pack (rc=$rc)" echo "===== Evidence Pack: tamper detection =====" # 1. change a report file only python3 -c "import json;p='$PACKDIR/h6-cost-telemetry.json';d=json.load(open(p));d['total_provider_tokens']=1;json.dump(d,open(p,'w'))" rc=0; bash "$EP" verify-pack "$RID" >/dev/null 2>&1 || rc=$? [[ "$rc" -eq 1 ]] && pass "verify-pack detects a changed report file" || fail "verify-pack missed a changed file (rc=$rc)" # 2. sophisticated attacker: change file AND rewrite manifest+head to match, keep old sig python3 - "$PACKDIR" <<'PY' import hashlib, json, os, sys d = sys.argv[1] man = json.load(open(os.path.join(d, "artifact-manifest.json"))) # recompute the (tampered) file hash and rewrite the manifest + head to match files = {} for fn in man["files"]: with open(os.path.join(d, fn), "rb") as f: files[fn] = hashlib.sha256(f.read()).hexdigest() canonical = json.dumps(files, sort_keys=True, separators=(",", ":")) head = hashlib.sha256(canonical.encode()).hexdigest() json.dump({"files": files, "manifest_head": head}, open(os.path.join(d, "artifact-manifest.json"), "w"), indent=2) open(os.path.join(d, "manifest-head.txt"), "w").write(head) # attacker rewrites head; cannot re-sign PY rc=0; bash "$EP" verify-pack "$RID" >/dev/null 2>&1 || rc=$? [[ "$rc" -eq 1 ]] && pass "verify-pack rejects manifest re-forge (signature over head fails)" || fail "verify-pack accepted a re-forged manifest (rc=$rc)" echo "===== Evidence Pack: certified-run gate =====" cleanup_pack # Make the required gates pass: fresh telemetry signature + benign-FP report present. bash "$S/telemetry-integrity.sh" sign >/dev/null 2>&1 || true RID2="ep-cert-$$" PACKDIR2="$PROJECT_ROOT/docs/output/casan/evidence-packs/$RID2" bash "$EP" pack "$RID2" > "$WORK/pack2.out" 2>&1 CERT="$(python3 -c "import json;print(json.load(open('$PACKDIR2/run-summary.json'))['certified'])" 2>/dev/null)" REASONS="$(python3 -c "import json;print(','.join(json.load(open('$PACKDIR2/run-summary.json'))['certification_reasons']))" 2>/dev/null)" if [[ "$CERT" == "True" ]]; then pass "certified run asserted only when required gates pass ($REASONS)" else echo " certification_reasons: $REASONS" # Not a hard failure IF the reason is an honest, real gap — but the mechanism # must at least NOT certify. Assert it declines to certify with reasons. [[ -n "$REASONS" ]] && pass "uncertified run records honest reasons (no false certification): $REASONS" \ || fail "certification produced neither a pass nor a reason" fi rm -rf "$PACKDIR2" echo "" echo "===== EVIDENCE PACK SUMMARY: PASS=$PASS FAIL=$FAIL =====" [[ "$FAIL" -eq 0 ]] || exit 1