149 lines
7.3 KiB
Bash
149 lines
7.3 KiB
Bash
#!/usr/bin/env bash
|
|
set -uo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
source "$SCRIPT_DIR/../scripts/bash/casan-paths.sh"
|
|
S="$CASAN_HARNESS_ROOT/scripts/bash"
|
|
WORK="$(mktemp -d)"
|
|
trap 'rm -rf "$WORK"' EXIT
|
|
PASS=0; FAIL=0
|
|
pass() { echo "PASS: $1"; PASS=$((PASS + 1)); }
|
|
fail() { echo "FAIL: $1"; FAIL=$((FAIL + 1)); }
|
|
|
|
echo "===== Assurance Kernel unit + cross-runtime conformance ====="
|
|
if CASAN_STATE_ROOT="$WORK/unit-state" python3 "$SCRIPT_DIR/assurance-kernel-tests.py" >/dev/null; then
|
|
pass "kernel policy and two-adapter conformance tests"
|
|
else
|
|
fail "kernel policy/conformance tests"
|
|
fi
|
|
|
|
echo "===== Agentic bridge integration ====="
|
|
if CASAN_STATE_ROOT="$WORK/bridge-state" python3 "$SCRIPT_DIR/assurance-upgrade-integration-tests.py" >/dev/null; then
|
|
pass "agentic H2, risk floor, failure outcome, trust-root integration"
|
|
else
|
|
fail "agentic upgrade integration tests"
|
|
fi
|
|
|
|
echo "===== Sandbox fail-closed selection ====="
|
|
SBX_STATE="$WORK/sandbox-state"
|
|
SBX_OUT="$(CASAN_STATE_ROOT="$SBX_STATE" CASAN_ENFORCEMENT_MODE=enforce \
|
|
CASAN_SANDBOX_MODE=container CASAN_SANDBOX_TEST_FORCE_UNAVAILABLE=1 \
|
|
bash "$S/sandbox-run.sh" --workspace "$WORK" -- sh -c 'printf should-not-run' 2>&1)"
|
|
SBX_RC=$?
|
|
if [[ "$SBX_RC" -eq 2 && "$SBX_OUT" == *"SANDBOX_ISOLATION_REQUIRED"* ]] \
|
|
&& grep -q '"reason_code":"sandbox_isolation_backend_unavailable"' "$SBX_STATE/logs/sandbox/decisions.jsonl"; then
|
|
pass "enforce mode denies unavailable isolation backend with structured evidence"
|
|
else
|
|
fail "strict sandbox unavailable behavior rc=$SBX_RC out=$SBX_OUT"
|
|
fi
|
|
|
|
DEV_OUT="$(CASAN_STATE_ROOT="$SBX_STATE" CASAN_PROFILE=test CASAN_SANDBOX_MODE=container \
|
|
CASAN_SANDBOX_TEST_FORCE_UNAVAILABLE=1 CASAN_SANDBOX_ALLOW_STATIC_FALLBACK=1 \
|
|
bash "$S/sandbox-run.sh" --workspace "$WORK" -- sh -c 'printf dev-ok' 2>/dev/null)"
|
|
[[ "$DEV_OUT" == "dev-ok" ]] && pass "explicit test-only static fallback remains available" || fail "explicit test fallback failed"
|
|
|
|
echo "===== H5 action-class risk floor ====="
|
|
GOV_STATE="$WORK/governance-state"
|
|
mkdir -p "$GOV_STATE" "$WORK/keys"
|
|
printf 'deploy a harmless documentation-only change\n' > "$WORK/deploy.txt"
|
|
GOV_OUT="$(CASAN_STATE_ROOT="$GOV_STATE" CASAN_AUDIT_KEY_DIR="$WORK/keys" CASAN_ACTOR=alice \
|
|
bash "$S/governance-check.sh" "$WORK/deploy.txt" "$WORK/deploy.out" deploy 2>&1)"
|
|
GOV_RC=$?
|
|
TRACE_FILE="$(find "$GOV_STATE/logs/trace" -name 'governance-*.json' -print -quit 2>/dev/null)"
|
|
if [[ "$GOV_RC" -eq 2 && -n "$TRACE_FILE" ]] \
|
|
&& python3 - "$TRACE_FILE" <<'PY'
|
|
import json, sys
|
|
r=json.load(open(sys.argv[1], encoding="utf-8"))
|
|
raise SystemExit(0 if r.get("action_class") == "deployment" and r.get("effective_risk") == "high" and r.get("decision") == "denied" else 1)
|
|
PY
|
|
then
|
|
pass "benign deploy text cannot lower deployment risk or bypass approval"
|
|
else
|
|
fail "deploy risk-floor regression rc=$GOV_RC out=$GOV_OUT"
|
|
fi
|
|
|
|
MISSING_ACTOR_OUT="$(CASAN_STATE_ROOT="$WORK/missing-actor-state" CASAN_AUDIT_KEY_DIR="$WORK/missing-actor-keys" \
|
|
CASAN_APPROVAL_DECISION=approve CASAN_APPROVER=bob \
|
|
bash "$S/governance-check.sh" "$WORK/deploy.txt" "$WORK/missing-actor.out" deploy 2>&1)"
|
|
MISSING_ACTOR_RC=$?
|
|
if [[ "$MISSING_ACTOR_RC" -eq 2 && "$MISSING_ACTOR_OUT" == *"actor_identity_required"* ]]; then
|
|
pass "approval cannot replace required actor identity for a high-impact action"
|
|
else
|
|
fail "missing actor was not denied rc=$MISSING_ACTOR_RC out=$MISSING_ACTOR_OUT"
|
|
fi
|
|
|
|
echo "===== Native failed side-effect completion semantics ====="
|
|
NATIVE_STATE="$WORK/native-failure-state"
|
|
printf 'safe write request\n' > "$WORK/native-input.txt"
|
|
|
|
OBSERVE_STATE="$WORK/native-observe-state"
|
|
OBSERVE_OUT="$(CASAN_STATE_ROOT="$OBSERVE_STATE" CASAN_GOVERNANCE_ROOT="$WORK/observe-governance" \
|
|
CASAN_AUDIT_KEY_DIR="$WORK/observe-keys" CASAN_ENFORCEMENT_MODE=observe CASAN_ACTOR=alice \
|
|
bash "$S/casan-harness.sh" "$WORK/native-input.txt" "$WORK/native-observe-output.txt" agent_step 2>&1)"
|
|
OBSERVE_RC=$?
|
|
if [[ "$OBSERVE_RC" -eq 0 && "$OBSERVE_OUT" == *"assurance=degraded certification=non_certified"* ]] \
|
|
&& grep -q '"assurance_result":"degraded"' "$OBSERVE_STATE/logs/kernel/"*.json; then
|
|
pass "native observe-only success is explicitly degraded and non-certified"
|
|
else
|
|
fail "native observe-only outcome was overstated rc=$OBSERVE_RC out=$OBSERVE_OUT"
|
|
fi
|
|
|
|
NATIVE_OUT="$(CASAN_STATE_ROOT="$NATIVE_STATE" CASAN_AUDIT_KEY_DIR="$WORK/native-keys" \
|
|
CASAN_ENFORCEMENT_MODE=observe CASAN_AGENT=boss CASAN_ACTOR=boss \
|
|
bash "$S/casan-harness.sh" "$WORK/native-input.txt" "$WORK/native-output.txt" write_file -- \
|
|
bash -c 'exit 7' 2>&1)"
|
|
NATIVE_RC=$?
|
|
if [[ "$NATIVE_RC" -eq 7 && "$NATIVE_OUT" != *"CASAN_HARNESS_COMPLETE"* ]] \
|
|
&& grep -q '"status":"failed"' "$NATIVE_STATE/logs/cost/metrics.jsonl" \
|
|
&& grep -q '"execution_result":"failed"' "$NATIVE_STATE/logs/kernel/"*.json; then
|
|
pass "failed native side effect records telemetry but cannot emit successful completion"
|
|
else
|
|
fail "native failed-outcome semantics rc=$NATIVE_RC out=$NATIVE_OUT"
|
|
fi
|
|
|
|
NO_OUTPUT_STATE="$WORK/native-no-output-state"
|
|
NO_OUTPUT_OUT="$(CASAN_STATE_ROOT="$NO_OUTPUT_STATE" CASAN_GOVERNANCE_ROOT="$WORK/no-output-governance" \
|
|
CASAN_AUDIT_KEY_DIR="$WORK/no-output-keys" CASAN_ENFORCEMENT_MODE=observe CASAN_ACTOR=alice \
|
|
bash "$S/casan-harness.sh" "$WORK/native-input.txt" "$WORK/native-no-output.txt" agent_step -- \
|
|
bash -c 'true' 2>&1)"
|
|
NO_OUTPUT_RC=$?
|
|
if [[ "$NO_OUTPUT_RC" -eq 1 && "$NO_OUTPUT_OUT" != *"CASAN_HARNESS_COMPLETE"* ]] \
|
|
&& grep -q '"status":"failed"' "$NO_OUTPUT_STATE/logs/cost/metrics.jsonl" \
|
|
&& grep -q '"execution_result":"failed"' "$NO_OUTPUT_STATE/logs/kernel/"*.json; then
|
|
pass "zero-exit command missing its required output remains a failed execution"
|
|
else
|
|
fail "missing required output was presented as success rc=$NO_OUTPUT_RC out=$NO_OUTPUT_OUT"
|
|
fi
|
|
|
|
echo "===== Native H2 missing dependency ====="
|
|
NATIVE_H2_STATE="$WORK/native-h2-state"
|
|
NATIVE_H2_OUT="$(CASAN_STATE_ROOT="$NATIVE_H2_STATE" CASAN_GOVERNANCE_ROOT="$WORK/native-h2-governance" \
|
|
CASAN_AUDIT_KEY_DIR="$WORK/native-h2-keys" CASAN_ENFORCEMENT_MODE=enforce CASAN_PROFILE=test \
|
|
CASAN_ACTOR=alice CASAN_APPROVAL_DECISION=approve CASAN_APPROVER=bob \
|
|
CASAN_H2_GATE_PATH="$WORK/missing-native-h2.sh" \
|
|
bash "$S/casan-harness.sh" "$WORK/native-input.txt" "$WORK/native-h2-output.txt" write_file -- \
|
|
bash -c 'printf should-not-run' 2>&1)"
|
|
NATIVE_H2_RC=$?
|
|
if [[ "$NATIVE_H2_RC" -eq 2 ]] \
|
|
&& grep -q '"reason_code":"h2_gate_unavailable"' "$NATIVE_H2_STATE/logs/policy/h2-decisions.jsonl" \
|
|
&& grep -q '"execution_result":"failed"' "$NATIVE_H2_STATE/logs/kernel/"*.json; then
|
|
pass "native enforce mode denies a missing H2 gate and emits failed canonical evidence"
|
|
else
|
|
fail "native missing H2 gate did not fail closed rc=$NATIVE_H2_RC out=$NATIVE_H2_OUT"
|
|
fi
|
|
|
|
echo "===== Production trust-root refusal ====="
|
|
printf 'read status\n' > "$WORK/read.txt"
|
|
TRUST_OUT="$(CASAN_STATE_ROOT="$WORK/trust-state" CASAN_AUDIT_KEY_DIR="$WORK/trust-keys" \
|
|
CASAN_PROFILE=production bash "$S/governance-check.sh" "$WORK/read.txt" "$WORK/read.out" agent_step 2>&1)"
|
|
TRUST_RC=$?
|
|
if [[ "$TRUST_RC" -eq 2 && "$TRUST_OUT" == *"production_trust_root_unavailable"* ]]; then
|
|
pass "production refuses local signing and local hash-chain fallback"
|
|
else
|
|
fail "production trust-root refusal rc=$TRUST_RC out=$TRUST_OUT"
|
|
fi
|
|
|
|
echo
|
|
echo "===== ASSURANCE UPGRADE SUMMARY: PASS=$PASS FAIL=$FAIL ====="
|
|
[[ "$FAIL" -eq 0 ]]
|