From 00963119f4eebbc56078ef50db9282a52adff8c8 Mon Sep 17 00:00:00 2001 From: thanhnv Date: Sat, 18 Jul 2026 00:14:22 +0700 Subject: [PATCH] feat: add verified state backup and restore drill --- docs/plans/CASAN_BACKLOG_STATUS.md | 1 + infra/production/README.md | 21 +++ .../scripts/bash/ci-harness-gate.sh | 1 + .../scripts/bash/state-backup.sh | 128 ++++++++++++++++++ .../tests/phase-state-backup-tests.sh | 17 +++ 5 files changed, 168 insertions(+) create mode 100755 packages/casan-harness/scripts/bash/state-backup.sh create mode 100755 packages/casan-harness/tests/phase-state-backup-tests.sh diff --git a/docs/plans/CASAN_BACKLOG_STATUS.md b/docs/plans/CASAN_BACKLOG_STATUS.md index a935549..024cd2c 100644 --- a/docs/plans/CASAN_BACKLOG_STATUS.md +++ b/docs/plans/CASAN_BACKLOG_STATUS.md @@ -42,6 +42,7 @@ | T2.4 | Kênh alert managed + on-call | 🟡 Docker lab ✅ (webhook emulator) | `alert-webhook` @ `:19092` nhận live POST. Prod: trỏ `CASAN_ALERT_WEBHOOK` tới Slack/PagerDuty + rota/on-call thật. | | T2.5 | Billing-API telemetry thật | 🟡 Docker lab ✅ (billing API mock) | `billing-api` @ `:19093/usage`; `provider-usage-fetch.sh` import được provider telemetry. Prod: OpenAI/Anthropic usage API thật + key. | | T2.6 | Sandbox: rootless/nsjail + base image CI | 🟡 (container isolation live via Docker) | Thêm profile nsjail/bubblewrap cho Linux CI (không cần Docker daemon); hardened base image tối thiểu. `sandbox-container.sh` đã có bản Docker. | +| T2.7 | Backup/restore + restore drill | 🟡 automated + test | `state-backup.sh` creates manifest/hash-bound state snapshots, requires encryption key in prod, restores only to an empty explicit directory; `phase-state-backup` is in CI. Remaining: customer object-store replication, retention/RPO/RTO, and a retained production restore-drill record. | ## TIER 3 — Platform plans (INDEX 01–12, việc lớn nhiều phiên) 📋 diff --git a/infra/production/README.md b/infra/production/README.md index a5699fc..6635db8 100644 --- a/infra/production/README.md +++ b/infra/production/README.md @@ -63,3 +63,24 @@ sudo systemctl daemon-reload sudo systemctl enable --now casan-audit-anchor.timer systemctl list-timers casan-audit-anchor.timer ``` + +## Backup and restore drill + +Back up CASAN runtime state using `state-backup.sh`. In production the command +requires `CASAN_BACKUP_PASSPHRASE_FILE`; store that file via the customer secret +manager, not in the repository. Restore refuses a non-empty target directory, +so a drill is non-destructive by default. + +```bash +export CASAN_PROFILE=prod +export CASAN_BACKUP_PASSPHRASE_FILE=/run/secrets/casan-backup-passphrase +export CASAN_BACKUP_ROOT=/srv/casan/backups + +bash packages/casan-harness/scripts/bash/state-backup.sh create "$(date -u +%Y%m%dT%H%M%SZ)" +bash packages/casan-harness/scripts/bash/state-backup.sh verify "" +bash packages/casan-harness/scripts/bash/state-backup.sh restore "" --to /srv/casan/restore-drill +``` + +Perform and retain a restore drill before enabling write actions. A production +runbook must define retention, off-site replication, an owner, RPO/RTO targets, +and approval for any restore into a live state directory. diff --git a/packages/casan-harness/scripts/bash/ci-harness-gate.sh b/packages/casan-harness/scripts/bash/ci-harness-gate.sh index 4bb4e70..3e300e5 100755 --- a/packages/casan-harness/scripts/bash/ci-harness-gate.sh +++ b/packages/casan-harness/scripts/bash/ci-harness-gate.sh @@ -137,6 +137,7 @@ run "phase-sec23-registry-crypt" bash "$TESTS/phase-sec23-registry-crypt-tests.s run "phase-sec24-supplychain" bash "$TESTS/phase-sec24-tests.sh" run "phase-sec25-attestation" bash "$TESTS/phase-sec25-tests.sh" run "phase-production-handoff" bash "$TESTS/phase-production-preflight-tests.sh" +run "phase-state-backup" bash "$TESTS/phase-state-backup-tests.sh" run "phase-release-provenance" bash "$TESTS/phase-release-provenance-tests.sh" run "phase-service-desk-onboard" bash "$TESTS/phase-service-desk-onboard-tests.sh" diff --git a/packages/casan-harness/scripts/bash/state-backup.sh b/packages/casan-harness/scripts/bash/state-backup.sh new file mode 100755 index 0000000..fc81bca --- /dev/null +++ b/packages/casan-harness/scripts/bash/state-backup.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash +set -euo pipefail + +# CASAN state backup / restore. Backups are self-verifying and restore only +# into an explicit empty directory; production requires encryption at rest. +# +# state-backup.sh create [--out DIR] +# state-backup.sh verify [--out DIR] +# state-backup.sh restore --out DIR --to EMPTY_DIR +# state-backup.sh drill [--out DIR] +# +# Env: CASAN_BACKUP_PASSPHRASE_FILE is required in prod and optional elsewhere. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/casan-paths.sh" + +MODE="${1:-}"; ID="${2:-}" +shift 2 2>/dev/null || true +OUT_ROOT="${CASAN_BACKUP_ROOT:-$CASAN_APP_ROOT/backups/casan-state}" +TO="" +while [[ "$#" -gt 0 ]]; do + case "$1" in + --out) OUT_ROOT="${2:-}"; shift 2 ;; + --to) TO="${2:-}"; shift 2 ;; + *) echo "STATE_BACKUP_FAIL reason=unknown_option option=$1" >&2; exit 64 ;; + esac +done + +fail() { echo "STATE_BACKUP_FAIL $*" >&2; exit 1; } +[[ "$ID" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,80}$ ]] || { echo "STATE_BACKUP_FAIL reason=invalid_backup_id" >&2; exit 64; } +OUT_ROOT="$(cd "$(dirname "$OUT_ROOT")" 2>/dev/null && pwd)/$(basename "$OUT_ROOT")" +PACK_DIR="$OUT_ROOT/$ID" +ARCHIVE="$PACK_DIR/state.tar.gz" +ENCRYPTED="$PACK_DIR/state.tar.gz.enc" +MANIFEST="$PACK_DIR/manifest.json" + +passphrase_file() { + local key="${CASAN_BACKUP_PASSPHRASE_FILE:-}" + if [[ "${CASAN_PROFILE:-}" == "prod" && ( -z "$key" || ! -s "$key" ) ]]; then + fail "reason=prod_encryption_key_required" + fi + [[ -z "$key" || -s "$key" ]] || fail "reason=backup_passphrase_file_missing" + printf '%s' "$key" +} + +write_manifest() { + local payload="$1" encrypted="$2" files="$3" + python3 - "$MANIFEST" "$ID" "$payload" "$encrypted" "$files" <<'PY' +import hashlib, json, os, sys +out, backup_id, payload, encrypted, files = sys.argv[1:] +with open(payload, 'rb') as f: digest = hashlib.sha256(f.read()).hexdigest() +data = {"schema_version":"1.0", "backup_id":backup_id, "payload":os.path.basename(payload), + "encrypted":encrypted == "true", "sha256":digest, "state_files":int(files)} +with open(out, 'w', encoding='utf8') as f: json.dump(data, f, sort_keys=True, indent=2); f.write('\n') +PY +} + +verify() { + [[ -f "$MANIFEST" ]] || fail "reason=manifest_missing" + python3 - "$MANIFEST" "$PACK_DIR" <<'PY' +import hashlib, json, os, sys +manifest, root = sys.argv[1:] +data = json.load(open(manifest, encoding='utf8')) +payload = data.get('payload', '') +if not payload or os.path.basename(payload) != payload: raise SystemExit('payload_invalid') +path = os.path.join(root, payload) +if not os.path.isfile(path): raise SystemExit('payload_missing') +with open(path, 'rb') as f: actual = hashlib.sha256(f.read()).hexdigest() +if actual != data.get('sha256'): raise SystemExit('checksum_mismatch') +print(json.dumps(data, sort_keys=True)) +PY +} + +case "$MODE" in + create) + [[ -d "$CASAN_STATE_ROOT" ]] || fail "reason=state_root_missing" + [[ ! -e "$PACK_DIR" ]] || fail "reason=backup_already_exists" + mkdir -p "$PACK_DIR" || fail "reason=backup_dir_create_failed" + TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT + # State is copied by relative path only. The archive cannot contain an + # arbitrary absolute filesystem path even when CASAN_STATE_ROOT is overridden. + tar -czf "$TMP/state.tar.gz" -C "$(dirname "$CASAN_STATE_ROOT")" "$(basename "$CASAN_STATE_ROOT")" + COUNT="$(tar -tzf "$TMP/state.tar.gz" | wc -l | tr -d '[:space:]')" + KEY="$(passphrase_file)" + if [[ -n "$KEY" ]]; then + command -v openssl >/dev/null 2>&1 || fail "reason=openssl_required_for_encryption" + openssl enc -aes-256-cbc -pbkdf2 -salt -in "$TMP/state.tar.gz" -out "$ENCRYPTED" -pass "file:$KEY" + write_manifest "$ENCRYPTED" true "$COUNT" + rm -f "$TMP/state.tar.gz" + else + mv "$TMP/state.tar.gz" "$ARCHIVE" + write_manifest "$ARCHIVE" false "$COUNT" + fi + chmod 700 "$PACK_DIR"; chmod 600 "$MANIFEST" "$PACK_DIR"/state.tar.gz* 2>/dev/null || true + echo "STATE_BACKUP_CREATED id=$ID files=$COUNT encrypted=$([[ -n "$KEY" ]] && echo true || echo false) dir=$PACK_DIR" + ;; + verify) + RESULT="$(verify)" || fail "reason=manifest_verification_failed" + echo "STATE_BACKUP_VALID id=$ID $RESULT" + ;; + restore) + [[ -n "$TO" ]] || { echo "STATE_BACKUP_FAIL reason=restore_target_required" >&2; exit 64; } + RESULT="$(verify)" || fail "reason=manifest_verification_failed" + [[ ! -e "$TO" || ( -d "$TO" && -z "$(find "$TO" -mindepth 1 -maxdepth 1 -print -quit)" ) ]] || fail "reason=restore_target_not_empty" + mkdir -p "$TO" + TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT + ENCRYPTED_FLAG="$(python3 -c 'import json,sys; print(str(json.loads(sys.argv[1])["encrypted"]).lower())' "$RESULT")" + if [[ "$ENCRYPTED_FLAG" == true ]]; then + KEY="$(passphrase_file)"; [[ -n "$KEY" ]] || fail "reason=decrypt_key_required" + openssl enc -d -aes-256-cbc -pbkdf2 -in "$ENCRYPTED" -out "$TMP/state.tar.gz" -pass "file:$KEY" || fail "reason=decrypt_failed" + else + cp "$ARCHIVE" "$TMP/state.tar.gz" + fi + tar -xzf "$TMP/state.tar.gz" -C "$TO" --no-same-owner + echo "STATE_BACKUP_RESTORED id=$ID target=$TO" + ;; + drill) + "$0" create "$ID" --out "$OUT_ROOT" + "$0" verify "$ID" --out "$OUT_ROOT" + TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT + "$0" restore "$ID" --out "$OUT_ROOT" --to "$TMP/restore" + ORIGINAL="$(tar -tzf "${ARCHIVE:-$PACK_DIR/state.tar.gz}" 2>/dev/null || true)" + [[ -n "$ORIGINAL" || -f "$ENCRYPTED" ]] || fail "reason=drill_archive_missing" + [[ -d "$TMP/restore/$(basename "$CASAN_STATE_ROOT")" ]] || fail "reason=drill_restore_shape_invalid" + echo "STATE_BACKUP_DRILL_PASS id=$ID restored=true" + ;; + *) echo "Usage: state-backup.sh {create|verify|restore|drill} [--out DIR] [--to EMPTY_DIR]" >&2; exit 64 ;; +esac diff --git a/packages/casan-harness/tests/phase-state-backup-tests.sh b/packages/casan-harness/tests/phase-state-backup-tests.sh new file mode 100755 index 0000000..2532b37 --- /dev/null +++ b/packages/casan-harness/tests/phase-state-backup-tests.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -uo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +S="$SCRIPT_DIR/../scripts/bash/state-backup.sh" +WORK="$(mktemp -d)"; trap 'rm -rf "$WORK"' EXIT +PASS=0; FAIL=0 +ok() { "$@" >/dev/null 2>&1 && { echo "PASS: $2"; PASS=$((PASS+1)); } || { echo "FAIL: $2"; FAIL=$((FAIL+1)); }; } +STATE="$WORK/state"; mkdir -p "$STATE/logs/audit"; printf 'audit\n' > "$STATE/logs/audit/audit.jsonl"; printf 'head\n' > "$STATE/logs/audit/audit-head.txt" +CASAN_APP_ROOT="$WORK/app" CASAN_STATE_ROOT="$STATE" CASAN_BACKUP_ROOT="$WORK/backups" bash "$S" create smoke >/dev/null && { echo "PASS: create plaintext backup"; PASS=$((PASS+1)); } || { echo "FAIL: create plaintext backup"; FAIL=$((FAIL+1)); } +CASAN_APP_ROOT="$WORK/app" CASAN_STATE_ROOT="$STATE" CASAN_BACKUP_ROOT="$WORK/backups" bash "$S" verify smoke >/dev/null && { echo "PASS: verify backup manifest"; PASS=$((PASS+1)); } || { echo "FAIL: verify backup manifest"; FAIL=$((FAIL+1)); } +CASAN_APP_ROOT="$WORK/app" CASAN_STATE_ROOT="$STATE" CASAN_BACKUP_ROOT="$WORK/backups" bash "$S" restore smoke --to "$WORK/restore" >/dev/null && cmp "$STATE/logs/audit/audit.jsonl" "$WORK/restore/state/logs/audit/audit.jsonl" >/dev/null && { echo "PASS: restore preserves state"; PASS=$((PASS+1)); } || { echo "FAIL: restore preserves state"; FAIL=$((FAIL+1)); } +mkdir -p "$WORK/not-empty"; touch "$WORK/not-empty/keep" +if CASAN_APP_ROOT="$WORK/app" CASAN_STATE_ROOT="$STATE" CASAN_BACKUP_ROOT="$WORK/backups" bash "$S" restore smoke --to "$WORK/not-empty" >/dev/null 2>&1; then echo "FAIL: non-empty restore target denied"; FAIL=$((FAIL+1)); else echo "PASS: non-empty restore target denied"; PASS=$((PASS+1)); fi +printf 'passphrase\n' > "$WORK/key"; chmod 600 "$WORK/key" +CASAN_PROFILE=prod CASAN_BACKUP_PASSPHRASE_FILE="$WORK/key" CASAN_APP_ROOT="$WORK/app" CASAN_STATE_ROOT="$STATE" CASAN_BACKUP_ROOT="$WORK/backups" bash "$S" create prod >/dev/null && { echo "PASS: prod backup encrypted"; PASS=$((PASS+1)); } || { echo "FAIL: prod backup encrypted"; FAIL=$((FAIL+1)); } +if CASAN_PROFILE=prod CASAN_APP_ROOT="$WORK/app" CASAN_STATE_ROOT="$STATE" CASAN_BACKUP_ROOT="$WORK/backups" bash "$S" create no-key >/dev/null 2>&1; then echo "FAIL: prod backup without key denied"; FAIL=$((FAIL+1)); else echo "PASS: prod backup without key denied"; PASS=$((PASS+1)); fi +echo "===== STATE BACKUP SUMMARY: PASS=$PASS FAIL=$FAIL ====="; [[ "$FAIL" -eq 0 ]]