137 lines
5.4 KiB
Bash
137 lines
5.4 KiB
Bash
#!/usr/bin/env bash
|
|
set -uo pipefail
|
|
|
|
# Plan-18 MVP-2 Track 4: agent/skill selection is governed by registry + RBAC.
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
source "$SCRIPT_DIR/../scripts/bash/casan-paths.sh"
|
|
RESOLVER="$CASAN_HARNESS_ROOT/scripts/bash/chat-agent-resolver.py"
|
|
TURN="$CASAN_HARNESS_ROOT/scripts/bash/chat-turn.py"
|
|
WORK="$(mktemp -d)"
|
|
trap 'rm -rf "$WORK"' EXIT
|
|
export CASAN_STATE_ROOT="$WORK/state"
|
|
|
|
PASS=0; FAIL=0
|
|
pass() { echo "PASS: $1"; PASS=$((PASS + 1)); }
|
|
fail() { echo "FAIL: $1"; FAIL=$((FAIL + 1)); }
|
|
|
|
echo "===== Plan-18 MVP-2 agent/skill selection ====="
|
|
|
|
python3 "$RESOLVER" list-agents --role viewer > "$WORK/list.json"
|
|
python3 - "$WORK/list.json" <<'PY' \
|
|
&& pass "agent registry lists versioned agents with role visibility" || fail "agent registry listing invalid"
|
|
import json, sys
|
|
d = json.load(open(sys.argv[1]))
|
|
ids = {a["id"]: a for a in d["agents"]}
|
|
assert d["success"] is True
|
|
assert {"evidence-reader", "ops-operator", "codegen-draft"}.issubset(ids)
|
|
assert ids["evidence-reader"]["allowed_for_role"] is True
|
|
assert ids["ops-operator"]["allowed_for_role"] is False
|
|
PY
|
|
|
|
python3 "$RESOLVER" bind --agent evidence-reader --skill evidence-summary --role viewer --actor alice --project default --tenant tenant-a > "$WORK/viewer.json"
|
|
python3 - "$WORK/viewer.json" <<'PY' \
|
|
&& pass "viewer can bind read-only evidence-reader" || fail "viewer could not bind read-only agent"
|
|
import json, sys
|
|
d = json.load(open(sys.argv[1]))
|
|
assert d["success"] is True
|
|
assert d["decision"] == "BOUND"
|
|
assert d["agent_selected"] == "evidence-reader"
|
|
assert d["delegation_level"] == 0
|
|
assert d["tool_allowlist"] == []
|
|
PY
|
|
|
|
set +e
|
|
python3 "$RESOLVER" bind --agent ops-operator --role viewer --actor alice --project default --tenant tenant-a > "$WORK/viewer-op.json"
|
|
RC=$?
|
|
set -e 2>/dev/null || true
|
|
python3 - "$WORK/viewer-op.json" "$RC" <<'PY' \
|
|
&& pass "viewer cannot select operator agent" || fail "viewer selected operator agent"
|
|
import json, sys
|
|
d = json.load(open(sys.argv[1]))
|
|
assert int(sys.argv[2]) == 2
|
|
assert d["success"] is False
|
|
assert d["decision"] == "DENIED"
|
|
assert d["reason"] == "role_not_allowed_for_agent"
|
|
PY
|
|
|
|
set +e
|
|
python3 "$RESOLVER" bind --agent evidence-reader --role viewer --actor alice --project default --tenant tenant-a --target-tenant tenant-b > "$WORK/cross-tenant.json"
|
|
RC=$?
|
|
set -e 2>/dev/null || true
|
|
python3 - "$WORK/cross-tenant.json" "$RC" <<'PY' \
|
|
&& pass "cross-tenant agent binding is denied by RBAC" || fail "cross-tenant agent binding was allowed"
|
|
import json, sys
|
|
d = json.load(open(sys.argv[1]))
|
|
assert int(sys.argv[2]) == 2
|
|
assert d["success"] is False
|
|
assert d["decision"] == "DENIED"
|
|
assert "rbac_denied" in d["reason"]
|
|
PY
|
|
|
|
set +e
|
|
python3 "$RESOLVER" bind --agent evidence-reader --role viewer --actor alice --project default --tenant tenant-a --delegation-level 1 > "$WORK/delegation.json"
|
|
RC=$?
|
|
set -e 2>/dev/null || true
|
|
python3 - "$WORK/delegation.json" "$RC" <<'PY' \
|
|
&& pass "delegation above agent max requires approval" || fail "delegation escalation was not held"
|
|
import json, sys
|
|
d = json.load(open(sys.argv[1]))
|
|
assert int(sys.argv[2]) == 3
|
|
assert d["success"] is False
|
|
assert d["decision"] == "REQUIRES_APPROVAL"
|
|
assert d["requires_approval"] is True
|
|
PY
|
|
|
|
set +e
|
|
python3 "$RESOLVER" bind --agent ops-operator --role operator --actor bob --project default --tenant tenant-a --tool shell > "$WORK/tool.json"
|
|
RC=$?
|
|
set -e 2>/dev/null || true
|
|
python3 - "$WORK/tool.json" "$RC" <<'PY' \
|
|
&& pass "tool outside agent allowlist is blocked" || fail "tool outside allowlist was accepted"
|
|
import json, sys
|
|
d = json.load(open(sys.argv[1]))
|
|
assert int(sys.argv[2]) == 2
|
|
assert d["success"] is False
|
|
assert d["decision"] == "DENIED"
|
|
assert d["action_gate"]["outcome"] == "BLOCK"
|
|
PY
|
|
|
|
set +e
|
|
python3 "$TURN" ask --message "run tests" --agent ops-operator --role viewer --actor alice --chat-id agent-chat --tenant tenant-a > "$WORK/turn-deny.json"
|
|
RC=$?
|
|
set -e 2>/dev/null || true
|
|
python3 - "$WORK/turn-deny.json" "$RC" <<'PY' \
|
|
&& pass "chat-turn denies unauthorized selected agent before operator runtime" || fail "chat-turn bypassed agent resolver"
|
|
import json, sys
|
|
d = json.load(open(sys.argv[1]))
|
|
assert int(sys.argv[2]) == 2
|
|
assert d["success"] is False
|
|
assert d["decision"] == "DENIED"
|
|
assert d["agent_selected"] == "ops-operator"
|
|
PY
|
|
|
|
python3 "$RESOLVER" verify-audit > "$WORK/audit.out" \
|
|
&& grep -q "CHAT_AGENT_AUDIT ok=true" "$WORK/audit.out" \
|
|
&& pass "agent binding audit hash chain verifies" || fail "agent binding audit invalid"
|
|
|
|
set +e
|
|
python3 "$TURN" ask --message "Show project status" --role viewer --actor telemetry-test \
|
|
--project okr --chat-id project-telemetry --tenant default > "$WORK/project-turn.json"
|
|
RC=$?
|
|
set -e 2>/dev/null || true
|
|
python3 - "$WORK/project-turn.json" "$WORK/state/logs/cost/metrics.jsonl" "$RC" <<'PY' \
|
|
&& pass "chat-turn attributes H6 telemetry to the adopted project" || fail "H6 project attribution missing"
|
|
import json, sys
|
|
result = json.loads(open(sys.argv[1], encoding="utf-8").read().splitlines()[-1])
|
|
rows = [json.loads(line) for line in open(sys.argv[2], encoding="utf-8") if line.strip()]
|
|
matches = [row for row in rows if row.get("trace_id") == result.get("trace_id")]
|
|
assert int(sys.argv[3]) in (0, 2)
|
|
assert result["project_id"] == "okr"
|
|
assert matches and matches[-1]["project"] == "okr" and matches[-1]["project_id"] == "okr"
|
|
PY
|
|
|
|
echo ""
|
|
echo "===== CHAT AGENT SUMMARY: PASS=$PASS FAIL=$FAIL ====="
|
|
[[ "$FAIL" -eq 0 ]] || exit 1
|