Add local production infra lab
This commit is contained in:
thanhnv
@@ -21,6 +21,7 @@ set -uo pipefail
|
||||
# CASAN_REVIEWERS_DIR (default governance/reviewers) — base dir for pubkey-file
|
||||
# CASAN_APPROVAL_JWT (optional RS256 IdP token)
|
||||
# CASAN_IDP_PUBLIC_KEY (default central-governance/idp-public.pem)
|
||||
# CASAN_IDP_JWKS_URL (optional OIDC JWKS endpoint; overrides public key)
|
||||
# Exit: 0 ok (prints "APPROVAL_OK role=<role>"), 3 deny (reason on stderr), 64 usage.
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
@@ -54,7 +55,11 @@ ROLES="$(awk -v a="$ACTION" '$1=="action" && $2==a {print $3; exit}' "$REVIEWERS
|
||||
|
||||
if [[ -n "${CASAN_APPROVAL_JWT:-}" ]]; then
|
||||
IDP_PUB="${CASAN_IDP_PUBLIC_KEY:-$GOV_DIR/idp-public.pem}"
|
||||
[[ -f "$IDP_PUB" ]] || deny "idp_pubkey_missing($IDP_PUB)"
|
||||
if [[ -n "${CASAN_IDP_JWKS_URL:-}" ]]; then
|
||||
IDP_PUB="jwks:${CASAN_IDP_JWKS_URL}"
|
||||
else
|
||||
[[ -f "$IDP_PUB" ]] || deny "idp_pubkey_missing($IDP_PUB)"
|
||||
fi
|
||||
JWT_OUT="$(
|
||||
python3 - "$CASAN_APPROVAL_JWT" "$IDP_PUB" "$APPROVER" "$ROLES" "$ACTION" "$ACTOR" "$INPUT_SHA" <<'PY'
|
||||
import base64
|
||||
@@ -64,6 +69,7 @@ import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
jwt, pub, approver, roles_csv, action, actor, input_sha = sys.argv[1:8]
|
||||
|
||||
@@ -101,13 +107,54 @@ if claims.get("input_sha256") != input_sha:
|
||||
|
||||
sig = b64u_decode(parts[2])
|
||||
signing_input = ".".join(parts[:2]).encode()
|
||||
|
||||
def der_len(n):
|
||||
if n < 128:
|
||||
return bytes([n])
|
||||
raw = n.to_bytes((n.bit_length() + 7) // 8, "big")
|
||||
return bytes([0x80 | len(raw)]) + raw
|
||||
|
||||
def der_tlv(tag, body):
|
||||
return bytes([tag]) + der_len(len(body)) + body
|
||||
|
||||
def der_int(n):
|
||||
raw = n.to_bytes((n.bit_length() + 7) // 8, "big") or b"\x00"
|
||||
if raw[0] & 0x80:
|
||||
raw = b"\x00" + raw
|
||||
return der_tlv(0x02, raw)
|
||||
|
||||
def jwk_to_pem(jwk):
|
||||
n = int.from_bytes(b64u_decode(jwk["n"]), "big")
|
||||
e = int.from_bytes(b64u_decode(jwk["e"]), "big")
|
||||
rsa_pub = der_tlv(0x30, der_int(n) + der_int(e))
|
||||
alg_id = der_tlv(
|
||||
0x30,
|
||||
der_tlv(0x06, b"\x2a\x86\x48\x86\xf7\x0d\x01\x01\x01") + der_tlv(0x05, b""),
|
||||
)
|
||||
spki = der_tlv(0x30, alg_id + der_tlv(0x03, b"\x00" + rsa_pub))
|
||||
b64 = base64.encodebytes(spki).decode().replace("\n", "")
|
||||
lines = [b64[i:i+64] for i in range(0, len(b64), 64)]
|
||||
return "-----BEGIN PUBLIC KEY-----\n" + "\n".join(lines) + "\n-----END PUBLIC KEY-----\n"
|
||||
|
||||
pub_file = pub
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
sig_path = os.path.join(td, "sig.bin")
|
||||
msg_path = os.path.join(td, "msg.txt")
|
||||
open(sig_path, "wb").write(sig)
|
||||
open(msg_path, "wb").write(signing_input)
|
||||
if pub.startswith("jwks:"):
|
||||
try:
|
||||
jwks = json.loads(urllib.request.urlopen(pub[len("jwks:"):], timeout=5).read().decode())
|
||||
except Exception:
|
||||
die("jwks_fetch_failed")
|
||||
kid = header.get("kid")
|
||||
keys = [k for k in jwks.get("keys", []) if k.get("kty") == "RSA" and (not kid or k.get("kid") == kid)]
|
||||
if not keys:
|
||||
die("jwks_key_not_found")
|
||||
pub_file = os.path.join(td, "idp-public.pem")
|
||||
open(pub_file, "w", encoding="utf-8").write(jwk_to_pem(keys[0]))
|
||||
rc = subprocess.run(
|
||||
["openssl", "dgst", "-sha256", "-verify", pub, "-signature", sig_path, msg_path],
|
||||
["openssl", "dgst", "-sha256", "-verify", pub_file, "-signature", sig_path, msg_path],
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
).returncode
|
||||
|
||||
@@ -73,6 +73,8 @@ class Handler(BaseHTTPRequestHandler):
|
||||
def log_message(self, *args): # silence per-request stderr noise
|
||||
pass
|
||||
|
||||
BIND = os.environ.get("CASAN_DASHBOARD_BIND", "127.0.0.1")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
HTTPServer(("127.0.0.1", PORT), Handler).serve_forever()
|
||||
HTTPServer((BIND, PORT), Handler).serve_forever()
|
||||
|
||||
+143
@@ -0,0 +1,143 @@
|
||||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
# CASAN local production-like infra lab.
|
||||
#
|
||||
# Usage:
|
||||
# infra-lab.sh start|stop|status|verify|env
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
|
||||
COMPOSE="$PROJECT_ROOT/infra/local-prod/docker-compose.yml"
|
||||
CMD="${1:-status}"
|
||||
|
||||
compose() {
|
||||
docker compose -f "$COMPOSE" "$@"
|
||||
}
|
||||
|
||||
need_docker() {
|
||||
command -v docker >/dev/null 2>&1 || { echo "INFRA_LAB_DOCKER_MISSING" >&2; exit 1; }
|
||||
docker compose version >/dev/null 2>&1 || { echo "INFRA_LAB_COMPOSE_MISSING" >&2; exit 1; }
|
||||
}
|
||||
|
||||
hash_file() {
|
||||
if command -v sha256sum >/dev/null 2>&1; then sha256sum "$1" | awk '{print $1}'
|
||||
else shasum -a 256 "$1" | awk '{print $1}'; fi
|
||||
}
|
||||
|
||||
wait_url() {
|
||||
local url="$1"
|
||||
for _ in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24; do
|
||||
curl -fsS -m 3 "$url" >/dev/null 2>&1 && return 0
|
||||
sleep 2
|
||||
done
|
||||
echo "INFRA_LAB_WAIT_TIMEOUT url=$url" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
case "$CMD" in
|
||||
start)
|
||||
need_docker
|
||||
compose up -d --build
|
||||
wait_url "http://127.0.0.1:18200/v1/sys/health" || exit 1
|
||||
wait_url "http://127.0.0.1:18081/healthz" || exit 1
|
||||
wait_url "http://127.0.0.1:19090/minio/health/live" || exit 1
|
||||
wait_url "http://127.0.0.1:19092/healthz" || exit 1
|
||||
wait_url "http://127.0.0.1:19093/healthz" || exit 1
|
||||
wait_url "http://127.0.0.1:18080/healthz" || exit 1
|
||||
echo "INFRA_LAB_STARTED compose=$COMPOSE"
|
||||
;;
|
||||
|
||||
stop)
|
||||
need_docker
|
||||
compose down
|
||||
echo "INFRA_LAB_STOPPED"
|
||||
;;
|
||||
|
||||
status)
|
||||
need_docker
|
||||
compose ps
|
||||
;;
|
||||
|
||||
env)
|
||||
cat <<'EOF'
|
||||
export VAULT_ADDR=http://127.0.0.1:18200
|
||||
export VAULT_TOKEN=root
|
||||
export CASAN_IDP_JWKS_URL=http://127.0.0.1:18081/.well-known/jwks.json
|
||||
export CASAN_ALERT_WEBHOOK=http://127.0.0.1:19092/alert
|
||||
export CASAN_PROVIDER_USAGE_API=http://127.0.0.1:19093/usage
|
||||
export CASAN_DASHBOARD_URL=http://127.0.0.1:18080
|
||||
export CASAN_DASHBOARD_AUTH=casan:casan
|
||||
export CASAN_MINIO_ENDPOINT=http://127.0.0.1:19090
|
||||
export CASAN_MINIO_BUCKET=casan-worm
|
||||
EOF
|
||||
;;
|
||||
|
||||
verify)
|
||||
need_docker
|
||||
PASS=0; FAIL=0
|
||||
pass() { echo "PASS: $1"; PASS=$((PASS + 1)); }
|
||||
fail() { echo "FAIL: $1"; FAIL=$((FAIL + 1)); }
|
||||
|
||||
wait_url "http://127.0.0.1:18200/v1/sys/health" && pass "Vault dev reachable" || fail "Vault dev unreachable"
|
||||
VAULT_ADDR=http://127.0.0.1:18200 VAULT_TOKEN=root bash "$SCRIPT_DIR/vault-kms.sh" enable-transit >/dev/null 2>&1
|
||||
WORK="$(mktemp -d)"
|
||||
trap 'rm -rf "$WORK"' EXIT
|
||||
printf 'infra-lab-head\n' > "$WORK/head.txt"
|
||||
if VAULT_ADDR=http://127.0.0.1:18200 VAULT_TOKEN=root bash "$SCRIPT_DIR/vault-kms.sh" sign "$WORK/head.txt" "$WORK/head.sig" casan-infra-lab >/dev/null 2>&1 \
|
||||
&& VAULT_ADDR=http://127.0.0.1:18200 VAULT_TOKEN=root bash "$SCRIPT_DIR/vault-kms.sh" verify "$WORK/head.txt" "$WORK/head.sig" casan-infra-lab >/dev/null 2>&1; then
|
||||
pass "Vault Transit signs and verifies"
|
||||
else
|
||||
fail "Vault Transit sign/verify failed"
|
||||
fi
|
||||
|
||||
printf 'deploy to production and run database migration\n' > "$WORK/approval.txt"
|
||||
INPUT_SHA="$(hash_file "$WORK/approval.txt")"
|
||||
TOKEN="$(curl -fsS -m 5 -H 'Content-Type: application/json' \
|
||||
-d "{\"sub\":\"oidc-ops\",\"role\":\"ops\",\"action\":\"deploy\",\"actor\":\"alice\",\"input_sha256\":\"$INPUT_SHA\",\"ttl_s\":300}" \
|
||||
http://127.0.0.1:18081/token | python3 -c 'import json,sys; print(json.load(sys.stdin)["access_token"])')" || TOKEN=""
|
||||
if [[ -n "$TOKEN" ]] && CASAN_APPROVAL_JWT="$TOKEN" CASAN_IDP_JWKS_URL="http://127.0.0.1:18081/.well-known/jwks.json" \
|
||||
bash "$SCRIPT_DIR/approval-verify.sh" deploy alice "$WORK/approval.txt" oidc-ops - >/dev/null 2>&1; then
|
||||
pass "Mock OIDC IdP issues JWKS-verifiable approval JWT"
|
||||
else
|
||||
fail "Mock OIDC approval verification failed"
|
||||
fi
|
||||
|
||||
if compose run --rm minio-mc -c 'set -eu; obj="verify-$(date +%s)-$$.txt"; mc alias set local http://minio:9000 casanadmin casanadmin123 >/dev/null; printf verify >/tmp/verify.txt; mc cp /tmp/verify.txt "local/casan-worm/$obj" >/dev/null; out="$(mc retention info "local/casan-worm/$obj")"; case "$out" in *COMPLIANCE*|*Mode*) exit 0;; *) echo "$out" >&2; exit 1;; esac' >/dev/null 2>&1; then
|
||||
pass "MinIO WORM/Object Lock bucket accepts retained object"
|
||||
else
|
||||
fail "MinIO WORM verification failed"
|
||||
fi
|
||||
|
||||
ALERT_FILE="$WORK/alert.json"
|
||||
printf '{"body":{"alert.type":"execution-failed","step.name":"infra-lab"},"resource":{"service.name":"casan"}}\n' > "$ALERT_FILE"
|
||||
if CASAN_AGENTOPS_DIR="$WORK/agentops" CASAN_ALERT_WEBHOOK=http://127.0.0.1:19092/alert bash "$SCRIPT_DIR/alert-dispatch.sh" "$ALERT_FILE" >/dev/null 2>&1 \
|
||||
&& curl -fsS http://127.0.0.1:19092/events | grep -q "infra-lab"; then
|
||||
pass "Alert webhook receives live dispatch"
|
||||
else
|
||||
fail "Alert webhook verification failed"
|
||||
fi
|
||||
|
||||
if bash "$SCRIPT_DIR/provider-usage-fetch.sh" http://127.0.0.1:19093/usage "$WORK/provider.jsonl" >/dev/null 2>&1 \
|
||||
&& grep -q "provider_api" "$WORK/provider.jsonl"; then
|
||||
pass "Billing API mock imports provider telemetry"
|
||||
else
|
||||
fail "Billing API mock verification failed"
|
||||
fi
|
||||
|
||||
if curl -fsS http://127.0.0.1:18080/healthz | grep -q '"status"' \
|
||||
&& curl -fsS -u casan:casan http://127.0.0.1:18080/ | grep -q "AgentOps Dashboard"; then
|
||||
pass "Dashboard served behind nginx basic auth"
|
||||
else
|
||||
fail "Dashboard/nginx verification failed"
|
||||
fi
|
||||
|
||||
echo "INFRA_LAB_VERIFY_SUMMARY PASS=$PASS FAIL=$FAIL"
|
||||
[[ "$FAIL" -eq 0 ]] || exit 1
|
||||
;;
|
||||
|
||||
*)
|
||||
echo "Usage: infra-lab.sh start|stop|status|verify|env" >&2
|
||||
exit 64
|
||||
;;
|
||||
esac
|
||||
@@ -0,0 +1,37 @@
|
||||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
# CASAN Tier-2 local production-like infra lab tests.
|
||||
# Starts Docker Compose if needed, verifies Vault/IdP/MinIO/dashboard/alert/billing.
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
S="$PROJECT_ROOT/.specify/scripts/bash"
|
||||
|
||||
PASS=0; FAIL=0; SKIP=0
|
||||
pass() { echo "PASS: $1"; PASS=$((PASS + 1)); }
|
||||
fail() { echo "FAIL: $1"; FAIL=$((FAIL + 1)); }
|
||||
skip() { echo "SKIP: $1"; SKIP=$((SKIP + 1)); }
|
||||
|
||||
if ! command -v docker >/dev/null 2>&1 || ! docker compose version >/dev/null 2>&1; then
|
||||
skip "Docker Compose unavailable"
|
||||
echo "===== PROD INFRA LAB SUMMARY: PASS=$PASS FAIL=$FAIL SKIP=$SKIP ====="
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if bash "$S/infra-lab.sh" start >/dev/null 2>&1; then
|
||||
pass "local-prod Docker Compose stack starts"
|
||||
else
|
||||
fail "local-prod Docker Compose stack failed to start"
|
||||
fi
|
||||
|
||||
if bash "$S/infra-lab.sh" verify > /tmp/casan-infra-lab-verify.out 2>&1; then
|
||||
pass "local-prod infra lab verification passes"
|
||||
else
|
||||
cat /tmp/casan-infra-lab-verify.out
|
||||
fail "local-prod infra lab verification failed"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "===== PROD INFRA LAB SUMMARY: PASS=$PASS FAIL=$FAIL SKIP=$SKIP ====="
|
||||
[[ "$FAIL" -eq 0 ]] || exit 1
|
||||
Reference in New Issue
Block a user