feat(c7): incident response — severity classifier + scoped kill-switch + runbook (V23)
Closes the last fully-[planned] Track-C dimension (was scored 1). - incident.sh raise <event>: classify severity via incident-severity.map (LOW/MED/HIGH/CRIT), record a structured entry (owner routing), and for HIGH/CRIT auto-engage the scoped kill-switch + dispatch an alert (reuses H6 alert-dispatch.sh). Exit 2 on HIGH/CRIT so a pipeline gate goes red. - kill-switch.sh engage/clear/check/status, scoped by project/model/provider (+ global). `check` exits 2 when engaged so gates honor it. - casan-harness.sh honors an engaged kill-switch before running (opt-in CASAN_KILLSWITCH_ENFORCE=1, default OFF → baseline unchanged). - incident-runbook.md: severity→owner→response + postmortem template + prod TODO. - phase-c7-incident-tests.sh: 15 checks — severity grading, auto kill-switch on HIGH/CRIT, MED-only records, lifecycle, global scope, structured record, and the production wrapper refusing to run under an engaged switch. Baselines: run-casan4 35/35, adversarial 44/44. New suite total: 175 → 190. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
thanhnv
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
571f5d8cc3
commit
f24ed21324
@@ -82,6 +82,20 @@ APPROVED_INPUT="$TMP_DIR/governance-approved-$TRACE_SUFFIX.txt"
|
||||
RAW_OUTPUT="$TMP_DIR/raw-output-$TRACE_SUFFIX.txt"
|
||||
|
||||
casan_log debug harness "action=$ACTION_NAME input=$INPUT_FILE output=$FINAL_OUTPUT key=${IDEMPOTENCY_KEY:0:12}…"
|
||||
|
||||
# C7: honor an engaged kill-switch before doing any work (incident containment).
|
||||
# Opt-in (default off) so the baseline is unchanged; production sets it on.
|
||||
if [[ "${CASAN_KILLSWITCH_ENFORCE:-0}" == "1" ]]; then
|
||||
KS_SCOPE="${CASAN_KILLSWITCH_SCOPE:-project}"
|
||||
KS_ID="${CASAN_KILLSWITCH_ID:-${CASAN_PROJECT:-current}}"
|
||||
if ! bash "$SCRIPT_DIR/kill-switch.sh" check "$KS_SCOPE" "$KS_ID" >/dev/null 2>&1; then
|
||||
casan_log error harness "KILL_SWITCH_ACTIVE scope=$KS_SCOPE id=$KS_ID — refusing to run $ACTION_NAME"
|
||||
: > "$FINAL_OUTPUT"
|
||||
echo "KILL_SWITCH_ACTIVE scope=$KS_SCOPE id=$KS_ID action=$ACTION_NAME" >&2
|
||||
exit 2
|
||||
fi
|
||||
fi
|
||||
|
||||
run_phase "H4-in" "$SCRIPT_DIR/security-check.sh" "$INPUT_FILE" "$SAFE_INPUT" input
|
||||
run_phase "H5" "$SCRIPT_DIR/governance-check.sh" "$SAFE_INPUT" "$APPROVED_INPUT" "$ACTION_NAME"
|
||||
|
||||
|
||||
+89
@@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
# CASAN — Incident response (C7 / V23).
|
||||
#
|
||||
# Turns a detected security/ops event into a graded incident: classify severity,
|
||||
# record a tamper-visible incident entry, and for HIGH/CRIT auto-engage the
|
||||
# scoped kill-switch + fire an alert (reuses alert-dispatch.sh from H6 if present).
|
||||
# Answers "when a gate catches an attack/spike/tamper — who is paged and what
|
||||
# stops?" — severity, owner, kill-switch, runbook.
|
||||
#
|
||||
# Usage:
|
||||
# incident.sh raise <event-type> [detail] [--scope <s>] [--id <id>]
|
||||
# incident.sh status
|
||||
# Exit: 0 recorded (LOW/MED) · 2 kill-switch engaged (HIGH/CRIT) · 64 usage.
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
|
||||
SEC_DIR="$PROJECT_ROOT/.specify/security"
|
||||
LOG="$PROJECT_ROOT/.specify/logs/level5/incidents.jsonl"
|
||||
RUNBOOK="$SEC_DIR/incident-runbook.md"
|
||||
SEVMAP="$SEC_DIR/incident-severity.map"
|
||||
mkdir -p "$(dirname "$LOG")"
|
||||
# shellcheck source=casan-log.sh
|
||||
source "$SCRIPT_DIR/casan-log.sh"
|
||||
|
||||
CMD="${1:-}"
|
||||
ts() { date -u +"%Y-%m-%dT%H:%M:%SZ"; }
|
||||
|
||||
# owner routing by severity (production: on-call rota / IdP group).
|
||||
owner_for() { case "$1" in CRIT) echo "security-oncall" ;; HIGH) echo "ops-oncall" ;; MED) echo "tech-lead" ;; *) echo "triage" ;; esac; }
|
||||
|
||||
if [[ "$CMD" == "status" ]]; then
|
||||
n=$(grep -c . "$LOG" 2>/dev/null || echo 0)
|
||||
echo "INCIDENTS total=$n log=$LOG"
|
||||
[[ -f "$LOG" ]] && tail -5 "$LOG"
|
||||
exit 0
|
||||
fi
|
||||
[[ "$CMD" == "raise" ]] || { echo "Usage: incident.sh raise <event-type> [detail] [--scope <s>] [--id <id>]" >&2; exit 64; }
|
||||
|
||||
EVENT="${2:-}"; DETAIL="${3:-}"
|
||||
[[ -n "$EVENT" ]] || { echo "usage: incident.sh raise <event-type> [detail]" >&2; exit 64; }
|
||||
SCOPE="project"; ID="${CASAN_PROJECT:-current}"
|
||||
shift 2 2>/dev/null || true
|
||||
while [[ "$#" -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--scope) SCOPE="${2:-project}"; shift 2 ;;
|
||||
--id) ID="${2:-current}"; shift 2 ;;
|
||||
*) shift ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Classify severity from the map (fallback to default).
|
||||
SEV="$(awk -v e="$EVENT" '$1==e {print $2; exit}' "$SEVMAP" 2>/dev/null)"
|
||||
[[ -n "$SEV" ]] || SEV="$(awk '$1=="default" {print $2; exit}' "$SEVMAP" 2>/dev/null)"
|
||||
[[ -n "$SEV" ]] || SEV="MED"
|
||||
OWNER="$(owner_for "$SEV")"
|
||||
TS="$(ts)"
|
||||
ACTION="recorded"
|
||||
|
||||
# HIGH/CRIT → engage the scoped kill-switch (stop the blast radius).
|
||||
if [[ "$SEV" == "CRIT" || "$SEV" == "HIGH" ]]; then
|
||||
bash "$SCRIPT_DIR/kill-switch.sh" engage "$SCOPE" "$ID" "incident:$EVENT" >/dev/null 2>&1 || true
|
||||
ACTION="kill_switch_engaged"
|
||||
# Fire an alert through the H6 dispatcher if it is wired up.
|
||||
if [[ -x "$SCRIPT_DIR/alert-dispatch.sh" ]]; then
|
||||
bash "$SCRIPT_DIR/alert-dispatch.sh" "$SEV" "incident:$EVENT" "$DETAIL" >/dev/null 2>&1 || true
|
||||
fi
|
||||
casan_log error incident "INCIDENT sev=$SEV event=$EVENT scope=$SCOPE id=$ID → kill-switch ENGAGED owner=$OWNER"
|
||||
else
|
||||
casan_log warn incident "INCIDENT sev=$SEV event=$EVENT scope=$SCOPE id=$ID owner=$OWNER"
|
||||
fi
|
||||
|
||||
# Record a structured incident entry.
|
||||
python - "$LOG" "$TS" "$EVENT" "$SEV" "$OWNER" "$SCOPE" "$ID" "$ACTION" "$DETAIL" "$RUNBOOK" <<'PY' 2>/dev/null || \
|
||||
printf '{"timestamp":"%s","event":"%s","severity":"%s","owner":"%s","scope":"%s","id":"%s","action":"%s"}\n' \
|
||||
"$TS" "$EVENT" "$SEV" "$OWNER" "$SCOPE" "$ID" "$ACTION" >> "$LOG"
|
||||
import json, sys
|
||||
log, ts, event, sev, owner, scope, iid, action, detail, runbook = sys.argv[1:11]
|
||||
with open(log, "a", encoding="utf-8") as f:
|
||||
f.write(json.dumps({
|
||||
"timestamp": ts, "event": event, "severity": sev, "owner": owner,
|
||||
"scope": scope, "id": iid, "action": action, "detail": detail[:300],
|
||||
"runbook": runbook,
|
||||
}) + "\n")
|
||||
PY
|
||||
|
||||
echo "INCIDENT_RAISED sev=$SEV event=$EVENT owner=$OWNER scope=$SCOPE id=$ID action=$ACTION runbook=$RUNBOOK"
|
||||
[[ "$SEV" == "CRIT" || "$SEV" == "HIGH" ]] && exit 2 || exit 0
|
||||
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
# CASAN H6/H7 — Kill-switch (Incident response · C7 / V23).
|
||||
#
|
||||
# A scoped emergency stop: engage a switch for a project / model / provider and
|
||||
# any gate that honors it refuses to run further work in that scope. Engaging is
|
||||
# recorded; clearing requires an explicit reason (production: reviewer approval).
|
||||
#
|
||||
# Usage:
|
||||
# kill-switch.sh engage <scope> <id> [reason] # turn the switch ON
|
||||
# kill-switch.sh clear <scope> <id> [reason] # turn it OFF (audited)
|
||||
# kill-switch.sh check <scope> <id> # exit 2 if engaged, 0 if clear
|
||||
# kill-switch.sh status # list engaged switches
|
||||
# scope ∈ {project, model, provider, global}. A `global` switch stops everything.
|
||||
# Env: CASAN_KILLSWITCH_DIR (default .specify/logs/level5/kill-switch)
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
|
||||
KS_DIR="${CASAN_KILLSWITCH_DIR:-$PROJECT_ROOT/.specify/logs/level5/kill-switch}"
|
||||
mkdir -p "$KS_DIR"
|
||||
|
||||
CMD="${1:-}"; SCOPE="${2:-}"; ID="${3:-}"; REASON="${4:-unspecified}"
|
||||
ts() { date -u +"%Y-%m-%dT%H:%M:%SZ"; }
|
||||
safe() { printf '%s' "$1" | tr '/ :' '___'; }
|
||||
|
||||
case "$CMD" in
|
||||
engage)
|
||||
[[ -n "$SCOPE" && -n "$ID" ]] || { echo "usage: kill-switch.sh engage <scope> <id> [reason]" >&2; exit 64; }
|
||||
f="$KS_DIR/$(safe "$SCOPE")-$(safe "$ID").on"
|
||||
printf '{"scope":"%s","id":"%s","reason":"%s","engaged_at":"%s","actor":"%s"}\n' \
|
||||
"$SCOPE" "$ID" "$REASON" "$(ts)" "${CASAN_ACTOR:-system}" > "$f"
|
||||
echo "KILL_SWITCH_ENGAGED scope=$SCOPE id=$ID reason=$REASON"
|
||||
;;
|
||||
clear)
|
||||
[[ -n "$SCOPE" && -n "$ID" ]] || { echo "usage: kill-switch.sh clear <scope> <id> [reason]" >&2; exit 64; }
|
||||
f="$KS_DIR/$(safe "$SCOPE")-$(safe "$ID").on"
|
||||
if [[ -f "$f" ]]; then
|
||||
printf '%s cleared_by=%s reason=%s at=%s\n' "$(cat "$f")" "${CASAN_ACTOR:-system}" "$REASON" "$(ts)" \
|
||||
>> "$KS_DIR/kill-switch-history.log"
|
||||
rm -f "$f"
|
||||
echo "KILL_SWITCH_CLEARED scope=$SCOPE id=$ID"
|
||||
else
|
||||
echo "KILL_SWITCH_NOT_ENGAGED scope=$SCOPE id=$ID"
|
||||
fi
|
||||
;;
|
||||
check)
|
||||
[[ -n "$SCOPE" && -n "$ID" ]] || { echo "usage: kill-switch.sh check <scope> <id>" >&2; exit 64; }
|
||||
# A global switch, or a switch for this exact scope/id, blocks.
|
||||
if [[ -f "$KS_DIR/global-all.on" ]]; then
|
||||
echo "KILL_SWITCH_ACTIVE scope=global" >&2; exit 2
|
||||
fi
|
||||
if [[ -f "$KS_DIR/$(safe "$SCOPE")-$(safe "$ID").on" ]]; then
|
||||
echo "KILL_SWITCH_ACTIVE scope=$SCOPE id=$ID" >&2; exit 2
|
||||
fi
|
||||
echo "KILL_SWITCH_CLEAR scope=$SCOPE id=$ID"; exit 0
|
||||
;;
|
||||
status)
|
||||
n=0
|
||||
for f in "$KS_DIR"/*.on; do [[ -e "$f" ]] || continue; cat "$f"; n=$((n+1)); done
|
||||
echo "KILL_SWITCH_STATUS engaged=$n"
|
||||
;;
|
||||
*)
|
||||
echo "Usage: kill-switch.sh {engage|clear|check|status} <scope> <id> [reason]" >&2
|
||||
exit 64 ;;
|
||||
esac
|
||||
@@ -0,0 +1,39 @@
|
||||
# CASAN Incident Runbook (C7 / V23)
|
||||
|
||||
When a gate raises an incident (`incident.sh raise <event>`), it is classified,
|
||||
recorded to `logs/level5/incidents.jsonl`, and for HIGH/CRIT the scoped
|
||||
kill-switch is engaged automatically + an alert is dispatched.
|
||||
|
||||
## Severity → owner → response
|
||||
|
||||
| Severity | Owner (on-call) | Auto-action | Human step |
|
||||
|---|---|---|---|
|
||||
| **CRIT** | security-oncall | kill-switch engaged + alert | Contain now; verify blast radius; do NOT clear until root cause known |
|
||||
| **HIGH** | ops-oncall | kill-switch engaged + alert | Assess; clear switch only after fix + reviewer sign-off |
|
||||
| **MED** | tech-lead | recorded + alert | Triage within SLA; batch-fix |
|
||||
| **LOW** | triage | recorded | Review in retro |
|
||||
|
||||
## Kill-switch operations
|
||||
```bash
|
||||
kill-switch.sh status # what is engaged
|
||||
kill-switch.sh check <scope> <id> # gates honor this (exit 2 = stop)
|
||||
kill-switch.sh clear <scope> <id> <reason># turn off (production: reviewer-approved)
|
||||
```
|
||||
Scopes: `project` · `model` · `provider` · `global` (global stops everything).
|
||||
|
||||
## Event → severity
|
||||
See `incident-severity.map`. Examples: `secret-to-cloud`=CRIT, `tool-write-sensitive`=CRIT,
|
||||
`dependency-postinstall`=HIGH, `audit-chain-broken`=HIGH, `cost-budget-exceeded`=MED.
|
||||
|
||||
## Postmortem template (fill after resolution)
|
||||
- **Incident**: <id / timestamp / event / severity>
|
||||
- **Detection**: which gate fired, what signal
|
||||
- **Blast radius**: scope, what was stopped by the kill-switch
|
||||
- **Root cause**:
|
||||
- **Fix**:
|
||||
- **Prevent recurrence**: new test/gate added (link the fail-able check)
|
||||
- **Kill-switch cleared by**: <reviewer> at <time>, reason
|
||||
|
||||
## Production TODO
|
||||
Managed alert channel (Slack/PagerDuty) + on-call rota + auto issue creation;
|
||||
kill-switch clear gated by reviewer approval (tie to approval-identity C4).
|
||||
@@ -0,0 +1,17 @@
|
||||
# CASAN — Incident severity map (C7 / V23). Line format: <event-type> <severity>
|
||||
# severity ∈ LOW | MED | HIGH | CRIT. HIGH/CRIT auto-engage the kill-switch.
|
||||
# Mirrors the Plan-07 C0 severity table.
|
||||
secret-to-cloud CRIT
|
||||
tool-write-sensitive CRIT
|
||||
private-key-exposure CRIT
|
||||
dependency-postinstall HIGH
|
||||
dependency-malicious HIGH
|
||||
audit-chain-broken HIGH
|
||||
telemetry-tamper HIGH
|
||||
sandbox-escape HIGH
|
||||
evidence-pack-tampered HIGH
|
||||
cost-budget-exceeded MED
|
||||
benign-fp-exceeded MED
|
||||
drift-detected MED
|
||||
approval-forged HIGH
|
||||
default MED
|
||||
@@ -0,0 +1,81 @@
|
||||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
# CASAN C7 — Incident response + kill-switch tests (V23).
|
||||
#
|
||||
# Proves: a detected event is graded (severity map), recorded, and for HIGH/CRIT
|
||||
# the scoped kill-switch auto-engages (gates honoring it then stop); MED/LOW only
|
||||
# record. Kill-switch check/clear and global scope work. Deterministic, no infra.
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
S="$PROJECT_ROOT/.specify/scripts/bash"
|
||||
WORK="$(mktemp -d)"; trap 'rm -rf "$WORK"' EXIT
|
||||
export CASAN_KILLSWITCH_DIR="$WORK/ks" # isolate the kill-switch state
|
||||
PASS=0; FAIL=0
|
||||
pass() { echo "PASS: $1"; PASS=$((PASS + 1)); }
|
||||
fail() { echo "FAIL: $1"; FAIL=$((FAIL + 1)); }
|
||||
expect_rc() {
|
||||
local want="$1" desc="$2"; shift 2
|
||||
local got=0; { "$@" >/dev/null 2>&1; } || got=$?
|
||||
[[ "$got" -eq "$want" ]] && pass "$desc (rc=$got)" || fail "$desc (got rc=$got, want $want)"
|
||||
}
|
||||
|
||||
INC() { bash "$S/incident.sh" "$@"; }
|
||||
KS() { bash "$S/kill-switch.sh" "$@"; }
|
||||
|
||||
echo "===== C7: severity classification + auto kill-switch ====="
|
||||
# CRIT event -> exit 2 + kill-switch engaged for its scope
|
||||
OUT="$(INC raise secret-to-cloud "key in prompt" --scope model --id m1 2>/dev/null)"; RC=$?
|
||||
{ [[ "$RC" -eq 2 ]] && printf '%s' "$OUT" | grep -q "sev=CRIT" && printf '%s' "$OUT" | grep -q "kill_switch_engaged"; } \
|
||||
&& pass "CRIT event (secret-to-cloud) → exit 2 + kill-switch engaged" \
|
||||
|| fail "CRIT handling wrong (rc=$RC out=$OUT)"
|
||||
expect_rc 2 "kill-switch now blocks that scope (model/m1)" KS check model m1
|
||||
|
||||
# HIGH event also engages
|
||||
expect_rc 2 "HIGH event (audit-chain-broken) → exit 2" INC raise audit-chain-broken "line 1" --scope project --id p1
|
||||
expect_rc 2 "kill-switch blocks project/p1 after HIGH" KS check project p1
|
||||
|
||||
# MED event: recorded only, no kill-switch
|
||||
expect_rc 0 "MED event (cost-budget-exceeded) → exit 0 (recorded, no kill)" INC raise cost-budget-exceeded "3x budget" --scope model --id m2
|
||||
expect_rc 0 "kill-switch stays clear for a MED-only scope (model/m2)" KS check model m2
|
||||
|
||||
# Unknown event → default severity (MED) → recorded, no kill
|
||||
expect_rc 0 "unknown event → default MED (recorded, no kill)" INC raise some-unmapped-thing --scope model --id m3
|
||||
|
||||
echo "===== C7: kill-switch lifecycle + global scope ====="
|
||||
expect_rc 0 "clear an engaged switch" KS clear model m1 "resolved-in-test"
|
||||
expect_rc 0 "cleared scope is unblocked again" KS check model m1
|
||||
KS engage global all "org-wide freeze" >/dev/null 2>&1
|
||||
expect_rc 2 "global kill-switch blocks ANY scope" KS check model brand-new
|
||||
KS clear global all "unfreeze" >/dev/null 2>&1
|
||||
expect_rc 0 "after clearing global, scopes flow again" KS check model brand-new
|
||||
|
||||
echo "===== C7: incident record is structured (severity + owner) ====="
|
||||
REC="$(INC raise private-key-exposure "id_rsa in output" --scope provider --id prov1 2>/dev/null)" || true
|
||||
LOGF="$PROJECT_ROOT/.specify/logs/level5/incidents.jsonl"
|
||||
if tail -5 "$LOGF" 2>/dev/null | grep -qE '"severity": ?"CRIT"' && tail -5 "$LOGF" 2>/dev/null | grep -qE '"owner": ?"security-oncall"'; then
|
||||
pass "incident recorded with severity + owner (routable)"
|
||||
else
|
||||
fail "incident record missing severity/owner"
|
||||
fi
|
||||
KS clear provider prov1 "test-cleanup" >/dev/null 2>&1 || true
|
||||
|
||||
echo "===== C7: production wrapper honors the kill-switch ====="
|
||||
printf 'benign task input\n' > "$WORK/w.txt"
|
||||
# switch clear → wrapper runs normally
|
||||
expect_rc 0 "wrapper runs when kill-switch is clear (enforce on)" \
|
||||
env CASAN_KILLSWITCH_ENFORCE=1 CASAN_KILLSWITCH_SCOPE=project CASAN_KILLSWITCH_ID=wf1 \
|
||||
bash "$S/casan-harness.sh" "$WORK/w.txt" "$WORK/wo.txt" agent_step
|
||||
KS engage project wf1 "drill" >/dev/null 2>&1
|
||||
expect_rc 2 "wrapper REFUSES to run when kill-switch engaged" \
|
||||
env CASAN_KILLSWITCH_ENFORCE=1 CASAN_KILLSWITCH_SCOPE=project CASAN_KILLSWITCH_ID=wf1 \
|
||||
bash "$S/casan-harness.sh" "$WORK/w.txt" "$WORK/wo.txt" agent_step
|
||||
expect_rc 0 "wrapper ignores engaged switch when enforcement is OFF (backward compat)" \
|
||||
env CASAN_KILLSWITCH_SCOPE=project CASAN_KILLSWITCH_ID=wf1 \
|
||||
bash "$S/casan-harness.sh" "$WORK/w.txt" "$WORK/wo.txt" agent_step
|
||||
KS clear project wf1 "cleanup" >/dev/null 2>&1
|
||||
|
||||
echo ""
|
||||
echo "===== C7 INCIDENT SUMMARY: PASS=$PASS FAIL=$FAIL ====="
|
||||
[[ "$FAIL" -eq 0 ]] || exit 1
|
||||
Reference in New Issue
Block a user