202 lines
9.1 KiB
Bash
Executable File
202 lines
9.1 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -uo pipefail
|
|
|
|
# CASAN CI harness gate.
|
|
# Runs the reproducible harness suites in a safe order. `run-casan4` must run
|
|
# first because it rewrites `.specify/logs`.
|
|
#
|
|
# Env:
|
|
# CASAN_CI_RUN_FRONTEND=0|1 default 1
|
|
# CASAN_CI_RUN_INFRA_LAB=0|1 default 0 (Docker Compose lab is optional in CI)
|
|
# CASAN_CI_STEP_TIMEOUT_SEC default 600
|
|
# CASAN_CI_SUITE_FILTER optional regex; run matching suite names only
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
source "$SCRIPT_DIR/casan-paths.sh"
|
|
ROOT="$CASAN_APP_ROOT"
|
|
TESTS="$CASAN_HARNESS_ROOT/tests"
|
|
|
|
# Plan-01: the integrity Python self-locates via __file__ and would mis-root when the
|
|
# package is invoked by its real path (no .specify facade). Export the harness-relative
|
|
# locations so test-integrity.py / bundle-integrity.py resolve correctly whether the gate
|
|
# is reached via .specify/... (compat symlink) or packages/casan-harness/... directly.
|
|
export CASAN_TESTS_DIR="$CASAN_HARNESS_ROOT/tests"
|
|
export CASAN_TEST_MANIFEST="$CASAN_HARNESS_ROOT/tests/test-integrity-manifest.json"
|
|
export CASAN_BUNDLE_ROOT="$CASAN_HARNESS_ROOT"
|
|
|
|
PASS=0
|
|
FAIL=0
|
|
SKIP=0
|
|
|
|
run() {
|
|
local name="$1"
|
|
shift
|
|
if [[ -n "${CASAN_CI_SUITE_FILTER:-}" && ! "$name" =~ $CASAN_CI_SUITE_FILTER ]]; then
|
|
skip "$name (filtered)"
|
|
return 0
|
|
fi
|
|
echo "==> $name"
|
|
if run_with_timeout "$@"; then
|
|
echo "CI_GATE_PASS $name"
|
|
PASS=$((PASS + 1))
|
|
else
|
|
echo "CI_GATE_FAIL $name" >&2
|
|
FAIL=$((FAIL + 1))
|
|
fi
|
|
}
|
|
|
|
skip() {
|
|
echo "CI_GATE_SKIP $1"
|
|
SKIP=$((SKIP + 1))
|
|
}
|
|
|
|
run_with_timeout() {
|
|
if ! command -v python3 >/dev/null 2>&1; then
|
|
"$@"
|
|
return $?
|
|
fi
|
|
CASAN_CI_STEP_TIMEOUT_SEC="${CASAN_CI_STEP_TIMEOUT_SEC:-600}" python3 - "$@" <<'PY'
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
|
|
timeout = int(os.environ.get("CASAN_CI_STEP_TIMEOUT_SEC", "600"))
|
|
cmd = sys.argv[1:]
|
|
try:
|
|
raise SystemExit(subprocess.run(cmd, timeout=timeout).returncode)
|
|
except subprocess.TimeoutExpired:
|
|
print(f"CI_GATE_TIMEOUT seconds={timeout} command={' '.join(cmd)}", file=sys.stderr)
|
|
raise SystemExit(124)
|
|
PY
|
|
}
|
|
|
|
cd "$ROOT" || exit 1
|
|
|
|
run "run-casan4-harness" bash "$TESTS/run-casan4-harness-tests.sh"
|
|
run "adversarial-harness" bash "$TESTS/adversarial-harness-tests.sh"
|
|
run "phase1-track-a" bash "$TESTS/phase1-track-a-tests.sh"
|
|
run "phase2-track-c" bash "$TESTS/phase2-track-c-tests.sh"
|
|
run "phase2-sourcegen" bash "$TESTS/phase2-sourcegen-tests.sh"
|
|
run "phase3-evidence-pack" bash "$TESTS/phase3-evidence-pack-tests.sh"
|
|
run "phase3-model-router" bash "$TESTS/phase3-model-router-tests.sh"
|
|
run "phase-h5-approval" bash "$TESTS/phase-h5-approval-tests.sh"
|
|
run "phase-h5-infra" bash "$TESTS/phase-h5-infra-tests.sh"
|
|
run "phase-h6-agentops" bash "$TESTS/phase-h6-agentops-tests.sh"
|
|
run "phase-c7-incident" bash "$TESTS/phase-c7-incident-tests.sh"
|
|
run "phase-h4-multilingual" bash "$TESTS/phase-h4-multilingual-tests.sh"
|
|
run "phase-c6-sandbox" bash "$TESTS/phase-c6-sandbox-tests.sh"
|
|
run "phase-h4-split-inject" bash "$TESTS/phase-h4-split-inject-tests.sh"
|
|
run "phase10-traceability" bash "$TESTS/phase10-traceability-tests.sh"
|
|
run "phase08-compression" bash "$TESTS/phase08-compression-tests.sh"
|
|
run "phase-control-plane" bash "$TESTS/phase-control-plane-tests.sh"
|
|
run "phase-control-plane-hitl" bash "$TESTS/phase-control-plane-hitl-tests.sh"
|
|
run "phase-rbac" bash "$TESTS/phase-rbac-tests.sh"
|
|
run "phase-rbac-audit" bash "$TESTS/phase-rbac-audit-tests.sh"
|
|
run "phase-rai" bash "$TESTS/phase-rai-tests.sh"
|
|
run "phase-selfimprove" bash "$TESTS/phase-selfimprove-tests.sh"
|
|
run "phase-governance-report" bash "$TESTS/phase-governance-report-tests.sh"
|
|
run "phase-preflight" bash "$TESTS/phase-preflight-tests.sh"
|
|
|
|
# Plan-16 security-audit remediation (P0) — each control has a fail-able test.
|
|
run "phase-sec01-unsigned-fail" bash "$TESTS/phase-sec01-tests.sh"
|
|
run "phase-sec02-no-local-key" bash "$TESTS/phase-sec02-tests.sh"
|
|
run "phase-sec03-rollback-rce" bash "$TESTS/phase-sec03-tests.sh"
|
|
run "phase-sec04-action-gate" bash "$TESTS/phase-sec04-tests.sh"
|
|
run "phase-sec05-json-safe" bash "$TESTS/phase-sec05-tests.sh"
|
|
run "phase-sec06-cp-signed" bash "$TESTS/phase-sec06-tests.sh"
|
|
run "phase-sec16-bundle" bash "$TESTS/phase-sec16-tests.sh"
|
|
run "phase-sec17-prod-profile" bash "$TESTS/phase-sec17-tests.sh"
|
|
run "phase-sec18-test-integrity" bash "$TESTS/phase-sec18-tests.sh"
|
|
# Plan-16 P1 (authz / fail-open / DoS)
|
|
run "phase-sec08-pii-failclosed" bash "$TESTS/phase-sec08-tests.sh"
|
|
run "phase-sec09-input-caps" bash "$TESTS/phase-sec09-tests.sh"
|
|
run "phase-sec19-atomic-store" bash "$TESTS/phase-sec19-tests.sh"
|
|
run "phase-sec20-toolchain" bash "$TESTS/phase-sec20-tests.sh"
|
|
run "phase-sec21-model-budget" bash "$TESTS/phase-sec21-tests.sh"
|
|
run "phase-sec07-approval" bash "$TESTS/phase-sec07-tests.sh"
|
|
run "phase-sec10-agent-identity" bash "$TESTS/phase-sec10-tests.sh"
|
|
# Plan-16 P2 (depth / hardening)
|
|
run "phase-sec13-ssrf" bash "$TESTS/phase-sec13-tests.sh"
|
|
run "phase-sec14-model-digest" bash "$TESTS/phase-sec14-tests.sh"
|
|
run "phase-sec27-log-controlchar" bash "$TESTS/phase-sec27-tests.sh"
|
|
run "phase-sec28-path-traversal" bash "$TESTS/phase-sec28-tests.sh"
|
|
run "phase-sec26-stored-inject" bash "$TESTS/phase-sec26-tests.sh"
|
|
run "phase-sec22-trusted-time" bash "$TESTS/phase-sec22-tests.sh"
|
|
run "phase-sec12-drift-invariant" bash "$TESTS/phase-sec12-tests.sh"
|
|
run "phase-sec29-audit-failclosed" bash "$TESTS/phase-sec29-tests.sh"
|
|
run "phase-sec30-approval-replay" bash "$TESTS/phase-sec30-tests.sh"
|
|
run "phase-sec15-low-cluster" bash "$TESTS/phase-sec15-tests.sh"
|
|
# Plan-16 SEC-23 (multi-tenant partition) — phased
|
|
run "phase-sec23-tenant-store" bash "$TESTS/phase-sec23-tenant-store-tests.sh"
|
|
run "phase-sec23-state-isolation" bash "$TESTS/phase-sec23-state-isolation-tests.sh"
|
|
run "phase-sec23-rbac-tenant" bash "$TESTS/phase-sec23-rbac-tenant-tests.sh"
|
|
run "phase-sec23-scope" bash "$TESTS/phase-sec23-scope-tests.sh"
|
|
run "phase-sec23-registry-crypt" bash "$TESTS/phase-sec23-registry-crypt-tests.sh"
|
|
# Plan-16 SEC-24/25 supply-chain (offline slice)
|
|
run "phase-sec24-supplychain" bash "$TESTS/phase-sec24-tests.sh"
|
|
run "phase-sec25-attestation" bash "$TESTS/phase-sec25-tests.sh"
|
|
run "phase-production-handoff" bash "$TESTS/phase-production-preflight-tests.sh"
|
|
run "phase-release-provenance" bash "$TESTS/phase-release-provenance-tests.sh"
|
|
run "phase-service-desk-onboard" bash "$TESTS/phase-service-desk-onboard-tests.sh"
|
|
|
|
# Plan-17 loop engineering (Agentic Loop Governance) — each primitive fail-closed.
|
|
run "phase-loop-governor" bash "$TESTS/phase-loop-governor-tests.sh"
|
|
run "phase-loop-convergence" bash "$TESTS/phase-loop-convergence-tests.sh"
|
|
run "phase-loop-gate" bash "$TESTS/phase-loop-gate-tests.sh"
|
|
run "phase-loop-trace" bash "$TESTS/phase-loop-trace-tests.sh"
|
|
run "phase-loop-metaloop" bash "$TESTS/phase-loop-metaloop-tests.sh"
|
|
run "phase-loop-run" bash "$TESTS/phase-loop-run-tests.sh"
|
|
|
|
# Plan-18 Governed Chat Console MVP-0 (Ask CASAN read-only).
|
|
run "phase-chat-prompt-router" bash "$TESTS/phase-chat-prompt-router-tests.sh"
|
|
run "phase-chat-readonly" bash "$TESTS/phase-chat-readonly-tests.sh"
|
|
run "phase-chat-model-synthesis" bash "$TESTS/phase-chat-model-synthesis-tests.sh"
|
|
run "phase-chat-advanced" bash "$TESTS/phase-chat-advanced-tests.sh"
|
|
run "phase-chat-session-audit" bash "$TESTS/phase-chat-session-audit-tests.sh"
|
|
run "phase-chat-operator" bash "$TESTS/phase-chat-operator-tests.sh"
|
|
run "phase-chat-agent-select" bash "$TESTS/phase-chat-agent-select-tests.sh"
|
|
run "phase-chat-pipeline" bash "$TESTS/phase-chat-pipeline-tests.sh"
|
|
run "phase-chat-stream-hold" bash "$TESTS/phase-chat-stream-hold-tests.sh"
|
|
run "phase-chat-replay" bash "$TESTS/phase-chat-replay-tests.sh"
|
|
run "phase-chat-approval" bash "$TESTS/phase-chat-approval-tests.sh"
|
|
run "phase-chat-codegen" bash "$TESTS/phase-chat-codegen-tests.sh"
|
|
run "phase-chat-tenant" bash "$TESTS/phase-chat-tenant-tests.sh"
|
|
|
|
# ARCH-02: coverage cannot silently drop; ARCH-01: harness/policy cannot silently
|
|
# drift. Both SKIP cleanly when no manifest is provisioned (non-strict dev/CI).
|
|
run "test-integrity" python3 "$SCRIPT_DIR/test-integrity.py" verify
|
|
run "bundle-integrity" python3 "$SCRIPT_DIR/bundle-integrity.py" verify
|
|
|
|
if [[ "${CASAN_CI_RUN_BACKEND:-1}" == "1" ]]; then
|
|
if command -v npm >/dev/null 2>&1; then
|
|
run "backend-tests" npm test -w backend
|
|
else
|
|
skip "backend-tests (npm unavailable)"
|
|
fi
|
|
else
|
|
skip "backend-tests (CASAN_CI_RUN_BACKEND=0)"
|
|
fi
|
|
|
|
if [[ "${CASAN_CI_RUN_FRONTEND:-1}" == "1" ]]; then
|
|
if command -v npm >/dev/null 2>&1; then
|
|
run "frontend-vitest" npm test -w frontend
|
|
else
|
|
skip "frontend-vitest (npm unavailable)"
|
|
fi
|
|
else
|
|
skip "frontend-vitest (CASAN_CI_RUN_FRONTEND=0)"
|
|
fi
|
|
|
|
if [[ "${CASAN_CI_RUN_INFRA_LAB:-0}" == "1" ]]; then
|
|
if command -v docker >/dev/null 2>&1 && docker compose version >/dev/null 2>&1; then
|
|
run "local-prod-infra-lab" bash "$TESTS/phase-prod-infra-lab-tests.sh"
|
|
else
|
|
skip "local-prod-infra-lab (docker compose unavailable)"
|
|
fi
|
|
else
|
|
skip "local-prod-infra-lab (CASAN_CI_RUN_INFRA_LAB=0)"
|
|
fi
|
|
|
|
echo "CI_GATE_SUMMARY PASS=$PASS FAIL=$FAIL SKIP=$SKIP"
|
|
[[ "$FAIL" -eq 0 ]] || exit 1
|