feat: add governed chat agent selection

This commit is contained in:
thanhnv
2026-07-08 22:19:51 +09:00
parent 06d8d31b16
commit 004afa73c9
18 changed files with 677 additions and 39 deletions
@@ -0,0 +1,36 @@
{
"version": 1,
"agents": [
{
"id": "evidence-reader",
"label": "Evidence Reader",
"mode": "READ_ONLY",
"model_role": "read_only",
"roles_allowed": ["viewer", "auditor", "operator", "project-admin", "org-admin"],
"skills_allowed": ["evidence-summary"],
"tool_allowlist": [],
"max_delegation_level": 0
},
{
"id": "ops-operator",
"label": "Ops Operator",
"mode": "OPERATOR",
"model_role": "operator",
"roles_allowed": ["operator", "project-admin", "org-admin"],
"skills_allowed": ["registered-actions"],
"tool_allowlist": ["run-chat-tests", "build-evidence-pack", "verify-evidence-pack"],
"max_delegation_level": 1
},
{
"id": "codegen-draft",
"label": "Codegen Draft",
"mode": "CODEGEN",
"model_role": "codegen",
"roles_allowed": ["project-admin", "org-admin"],
"skills_allowed": ["sourcegen-draft"],
"tool_allowlist": ["artifact-scan", "tool-output-scan"],
"max_delegation_level": 2,
"requires_approval": true
}
]
}
@@ -0,0 +1,268 @@
#!/usr/bin/env python3
"""Plan-18 MVP-2 agent/skill governance resolver.
This is a harness-owned policy primitive. It binds a chat turn to an agent,
skill, delegation level, model role, and tool allowlist before any runtime is
invoked. Unknown policy, RBAC failure, role mismatch, delegation escalation, or
tool outside the agent allowlist all fail closed.
"""
import argparse
import datetime
import hashlib
import json
import os
import subprocess
import sys
def project_root() -> str:
d = os.path.abspath(os.path.dirname(__file__))
p = d
while p != os.path.dirname(p):
if os.path.isdir(os.path.join(p, ".specify")) or os.path.isdir(os.path.join(p, "packages/casan-harness")):
return p
p = os.path.dirname(p)
return os.path.abspath(os.path.join(d, "..", "..", ".."))
ROOT = project_root()
HARNESS_ROOT = os.path.join(ROOT, "packages", "casan-harness")
HARNESS_BIN = os.path.join(HARNESS_ROOT, "scripts", "bash")
RBAC = os.path.join(HARNESS_BIN, "rbac-check.py")
GENESIS_HASH = "0" * 64
def now_iso() -> str:
return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
def sha(text: str) -> str:
return hashlib.sha256(text.encode("utf-8")).hexdigest()
def state_root() -> str:
return os.environ.get("CASAN_STATE_ROOT") or os.path.join(ROOT, ".specify")
def registry_path() -> str:
return os.environ.get("CASAN_AGENT_REGISTRY_FILE") or os.path.join(HARNESS_ROOT, "config", "agent-registry.yaml")
def audit_path() -> str:
return os.environ.get("CASAN_AGENT_BIND_AUDIT_LOG") or os.path.join(state_root(), "logs", "chat", "agent-bindings.jsonl")
def head_path() -> str:
return os.environ.get("CASAN_AGENT_BIND_AUDIT_HEAD") or os.path.join(state_root(), "logs", "chat", "agent-bindings-head.txt")
def load_registry():
try:
with open(registry_path(), encoding="utf-8") as fh:
data = json.load(fh)
agents = data.get("agents")
if not isinstance(agents, list):
raise ValueError("agents_not_list")
return data
except Exception as exc:
return {"_error": f"agent_registry_unreadable:{exc}", "agents": []}
def find_agent(registry, agent_id: str):
return next((a for a in registry.get("agents", []) if a.get("id") == agent_id), None)
def load_head() -> str:
try:
return open(head_path(), encoding="utf-8").read().strip() or GENESIS_HASH
except OSError:
return GENESIS_HASH
def append_audit(base):
path = audit_path()
os.makedirs(os.path.dirname(path), exist_ok=True)
seq = 1
if os.path.isfile(path):
with open(path, encoding="utf-8") as fh:
seq = sum(1 for line in fh if line.strip()) + 1
prev = load_head()
core = {"seq": seq, **base, "prev_hash": prev}
record_hash = sha(json.dumps(core, sort_keys=True, ensure_ascii=False))
rec = {**core, "record_hash": record_hash}
with open(path, "a", encoding="utf-8") as fh:
fh.write(json.dumps(rec, ensure_ascii=False) + "\n")
os.makedirs(os.path.dirname(head_path()), exist_ok=True)
with open(head_path(), "w", encoding="utf-8") as fh:
fh.write(record_hash + "\n")
return rec
def decision_payload(args, decision: str, reason: str, agent=None, skill="", tools=None):
tools = tools or []
success = decision == "BOUND"
payload = {
"success": success,
"decision": decision,
"reason": reason,
"agent_selected": (agent or {}).get("id") or args.agent,
"skill_selected": skill or "",
"delegation_level": args.delegation_level,
"model_role": (agent or {}).get("model_role"),
"mode": (agent or {}).get("mode"),
"tool_allowlist": (agent or {}).get("tool_allowlist", []),
"requested_tools": tools,
"side_effect_allowed": bool((agent or {}).get("tool_allowlist")),
"requires_approval": bool((agent or {}).get("requires_approval")) or decision == "REQUIRES_APPROVAL",
}
rec = append_audit({
"timestamp": now_iso(),
"harness": "H5-chat-agent-bind",
"actor": args.actor,
"role": args.role,
"project": args.project,
"tenant": args.tenant,
"agent_selected": payload["agent_selected"],
"skill_selected": payload["skill_selected"],
"delegation_level": args.delegation_level,
"decision": decision,
"reason": reason,
"tool_allowlist_ref": sha(json.dumps(payload["tool_allowlist"], sort_keys=True)),
"requested_tools_ref": sha(json.dumps(tools, sort_keys=True)),
})
payload["audit"] = {"seq": rec["seq"], "record_hash": rec["record_hash"], "head": rec["record_hash"]}
return payload
def run_rbac(args):
r = subprocess.run([
"python3", RBAC, "check",
"--role", args.role,
"--resource", "chat",
"--action", "select_agent",
"--role-project", args.project,
"--target-project", args.target_project or args.project,
"--role-tenant", args.tenant,
"--target-tenant", args.target_tenant or args.tenant,
], cwd=ROOT, capture_output=True, text=True)
return r.returncode, (r.stdout + r.stderr).strip()
def bind(args) -> int:
registry = load_registry()
if registry.get("_error"):
out = decision_payload(args, "DENIED", registry["_error"])
print(json.dumps(out, ensure_ascii=False))
return 2
agent = find_agent(registry, args.agent)
if not agent:
out = decision_payload(args, "DENIED", "unknown_agent")
print(json.dumps(out, ensure_ascii=False))
return 2
rc, msg = run_rbac(args)
if rc != 0:
out = decision_payload(args, "DENIED", f"rbac_denied:{msg}", agent)
print(json.dumps(out, ensure_ascii=False))
return 2
if args.role not in agent.get("roles_allowed", []):
out = decision_payload(args, "DENIED", "role_not_allowed_for_agent", agent)
print(json.dumps(out, ensure_ascii=False))
return 2
skill = args.skill or (agent.get("skills_allowed", [""])[0] if agent.get("skills_allowed") else "")
if skill and skill not in agent.get("skills_allowed", []):
out = decision_payload(args, "DENIED", "skill_not_allowed_for_agent", agent, skill)
print(json.dumps(out, ensure_ascii=False))
return 2
max_level = int(agent.get("max_delegation_level", 0))
if args.delegation_level > max_level:
out = decision_payload(args, "REQUIRES_APPROVAL", "delegation_exceeds_agent_max", agent, skill)
print(json.dumps(out, ensure_ascii=False))
return 3
requested_tools = list(args.tool or [])
allow = set(agent.get("tool_allowlist", []))
outside = [t for t in requested_tools if t not in allow]
if outside:
out = decision_payload(args, "DENIED", "tool_not_allowlisted", agent, skill, requested_tools)
out["action_gate"] = {"outcome": "BLOCK", "reason": f"tool_not_allowlisted:{','.join(outside)}"}
print(json.dumps(out, ensure_ascii=False))
return 2
out = decision_payload(args, "BOUND", "agent_bound", agent, skill, requested_tools)
print(json.dumps(out, ensure_ascii=False))
return 0
def list_agents(args) -> int:
registry = load_registry()
if registry.get("_error"):
print(json.dumps({"success": False, "error": registry["_error"], "agents": []}, ensure_ascii=False))
return 2
agents = []
for agent in registry.get("agents", []):
item = {k: agent.get(k) for k in (
"id", "label", "mode", "model_role", "roles_allowed", "skills_allowed",
"tool_allowlist", "max_delegation_level", "requires_approval",
)}
item["allowed_for_role"] = args.role in agent.get("roles_allowed", []) if args.role else None
agents.append(item)
print(json.dumps({"success": True, "version": registry.get("version"), "agents": agents}, ensure_ascii=False))
return 0
def verify_audit() -> int:
prev = GENESIS_HASH
count = 0
try:
fh = open(audit_path(), encoding="utf-8")
except OSError:
print("CHAT_AGENT_AUDIT ok=true records=0 head=" + prev)
return 0
with fh:
for line in fh:
if not line.strip():
continue
count += 1
rec = json.loads(line)
got = rec.get("record_hash")
rest = {k: v for k, v in rec.items() if k != "record_hash"}
if rest.get("prev_hash") != prev or sha(json.dumps(rest, sort_keys=True, ensure_ascii=False)) != got:
print(f"CHAT_AGENT_AUDIT ok=false brokenAt={count}")
return 1
prev = got
print(f"CHAT_AGENT_AUDIT ok=true records={count} head={prev}")
return 0
def main() -> int:
ap = argparse.ArgumentParser()
sub = ap.add_subparsers(dest="cmd", required=True)
b = sub.add_parser("bind")
b.add_argument("--agent", required=True)
b.add_argument("--skill", default="")
b.add_argument("--actor", default="anonymous")
b.add_argument("--role", default="viewer")
b.add_argument("--project", default="default")
b.add_argument("--tenant", default="default")
b.add_argument("--target-project", default="")
b.add_argument("--target-tenant", default="")
b.add_argument("--delegation-level", type=int, default=0)
b.add_argument("--tool", action="append", default=[])
b.set_defaults(func=bind)
l = sub.add_parser("list-agents")
l.add_argument("--role", default="")
l.set_defaults(func=list_agents)
sub.add_parser("verify-audit").set_defaults(func=lambda _args: verify_audit())
args = ap.parse_args()
return args.func(args)
if __name__ == "__main__":
raise SystemExit(main())
@@ -24,6 +24,7 @@ def project_root() -> str:
ROOT = project_root()
BIN = os.path.join(ROOT, "packages", "casan-harness", "scripts", "bash")
ROUTER = os.path.join(BIN, "prompt-mode-router.py")
AGENT_RESOLVER = os.path.join(BIN, "chat-agent-resolver.py")
READONLY = os.path.join(BIN, "chat-readonly.py")
OPERATOR = os.path.join(BIN, "chat-operator.py")
@@ -36,13 +37,69 @@ def classify(message: str):
return {"mode": "BLOCK", "risk": "high", "reason": "router_invalid_json", "matched_rules": [r.stderr.strip()]}
def run_mode(args, binding):
r = subprocess.run(args, cwd=ROOT, capture_output=True, text=True)
try:
payload = json.loads(r.stdout)
payload["agent_binding"] = binding
print(json.dumps(payload, ensure_ascii=False))
except Exception:
if r.stdout:
sys.stdout.write(r.stdout)
if r.stderr:
sys.stderr.write(r.stderr)
return r.returncode
def run_and_passthrough(args):
r = subprocess.run(args, cwd=ROOT, text=True)
return r.returncode
def default_agent_for_mode(mode: str) -> str:
if mode == "OPERATOR":
return "ops-operator"
if mode == "CODEGEN":
return "codegen-draft"
return "evidence-reader"
def bind_agent(args, router):
agent = args.agent or default_agent_for_mode(router.get("mode", "READ_ONLY"))
tools = []
if router.get("mode") == "OPERATOR":
# The operator primitive resolves the exact registered action later; the
# agent bind still proves this turn is allowed to use the operator class.
tools.append("run-chat-tests")
cmd = [
"python3", AGENT_RESOLVER, "bind",
"--agent", agent,
"--actor", args.actor,
"--role", args.role,
"--project", args.project,
"--tenant", args.tenant,
"--delegation-level", str(args.delegation_level),
]
if args.skill:
cmd += ["--skill", args.skill]
for tool in tools:
cmd += ["--tool", tool]
r = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True)
if r.returncode != 0:
sys.stdout.write((r.stdout or r.stderr).strip() + "\n")
return r.returncode, None
try:
return 0, json.loads(r.stdout)
except Exception:
print(json.dumps({"success": False, "decision": "DENIED", "reason": "agent_resolver_invalid_json"}, ensure_ascii=False))
return 2, None
def ask(args) -> int:
router = classify(args.message)
bind_rc, binding = bind_agent(args, router)
if bind_rc != 0:
return bind_rc
common = [
"--message", args.message,
"--actor", args.actor,
@@ -51,8 +108,8 @@ def ask(args) -> int:
"--tenant", args.tenant,
]
if router.get("mode") == "OPERATOR":
return run_and_passthrough(["python3", OPERATOR, "run", *common])
return run_and_passthrough(["python3", READONLY, "ask", *common])
return run_mode(["python3", OPERATOR, "run", *common], binding)
return run_mode(["python3", READONLY, "ask", *common], binding)
def verify_audit() -> int:
@@ -65,9 +122,14 @@ def main() -> int:
askp = sub.add_parser("ask")
askp.add_argument("--message", required=True)
askp.add_argument("--actor", default="anonymous")
askp.add_argument("--role", default="viewer")
askp.add_argument("--project", default="default")
askp.add_argument("--chat-id", default="")
askp.add_argument("--turn-id", default="")
askp.add_argument("--tenant", default="default")
askp.add_argument("--agent", default="")
askp.add_argument("--skill", default="")
askp.add_argument("--delegation-level", type=int, default=0)
askp.set_defaults(func=ask)
sub.add_parser("verify-audit").set_defaults(func=lambda _args: verify_audit())
args = ap.parse_args()
@@ -149,6 +149,7 @@ run "phase-chat-prompt-router" bash "$TESTS/phase-chat-prompt-router-tests.sh"
run "phase-chat-readonly" bash "$TESTS/phase-chat-readonly-tests.sh"
run "phase-chat-session-audit" bash "$TESTS/phase-chat-session-audit-tests.sh"
run "phase-chat-operator" bash "$TESTS/phase-chat-operator-tests.sh"
run "phase-chat-agent-select" bash "$TESTS/phase-chat-agent-select-tests.sh"
# ARCH-02: coverage cannot silently drop; ARCH-01: harness/policy cannot silently
# drift. Both SKIP cleanly when no manifest is provisioned (non-strict dev/CI).
@@ -17,11 +17,11 @@ import sys
# scope: "org" (all projects) or "project" (must match the acted-on project)
PERMISSIONS = {
"org-admin": {"scope": "org", "allow": {"*"}},
"project-admin": {"scope": "project", "allow": {"settings:read", "settings:write", "monitoring:read", "audit:read"}},
"approver": {"scope": "project", "allow": {"settings:read", "monitoring:read", "approval:grant"}},
"operator": {"scope": "project", "allow": {"monitoring:read", "kill_switch:engage"}},
"viewer": {"scope": "project", "allow": {"settings:read", "monitoring:read"}},
"auditor": {"scope": "org", "allow": {"settings:read", "monitoring:read", "audit:read"}},
"project-admin": {"scope": "project", "allow": {"settings:read", "settings:write", "monitoring:read", "audit:read", "chat:select_agent"}},
"approver": {"scope": "project", "allow": {"settings:read", "monitoring:read", "approval:grant", "chat:select_agent"}},
"operator": {"scope": "project", "allow": {"monitoring:read", "kill_switch:engage", "chat:select_agent"}},
"viewer": {"scope": "project", "allow": {"settings:read", "monitoring:read", "chat:select_agent"}},
"auditor": {"scope": "org", "allow": {"settings:read", "monitoring:read", "audit:read", "chat:select_agent"}},
}
# Maps an IdP-issued claim (role name / group) to an RBAC role. The IdP (Plan-07
@@ -0,0 +1,120 @@
#!/usr/bin/env bash
set -uo pipefail
# Plan-18 MVP-2 Track 4: agent/skill selection is governed by registry + RBAC.
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../scripts/bash/casan-paths.sh"
RESOLVER="$CASAN_HARNESS_ROOT/scripts/bash/chat-agent-resolver.py"
TURN="$CASAN_HARNESS_ROOT/scripts/bash/chat-turn.py"
WORK="$(mktemp -d)"
trap 'rm -rf "$WORK"' EXIT
export CASAN_STATE_ROOT="$WORK/state"
PASS=0; FAIL=0
pass() { echo "PASS: $1"; PASS=$((PASS + 1)); }
fail() { echo "FAIL: $1"; FAIL=$((FAIL + 1)); }
echo "===== Plan-18 MVP-2 agent/skill selection ====="
python3 "$RESOLVER" list-agents --role viewer > "$WORK/list.json"
python3 - "$WORK/list.json" <<'PY' \
&& pass "agent registry lists versioned agents with role visibility" || fail "agent registry listing invalid"
import json, sys
d = json.load(open(sys.argv[1]))
ids = {a["id"]: a for a in d["agents"]}
assert d["success"] is True
assert {"evidence-reader", "ops-operator", "codegen-draft"}.issubset(ids)
assert ids["evidence-reader"]["allowed_for_role"] is True
assert ids["ops-operator"]["allowed_for_role"] is False
PY
python3 "$RESOLVER" bind --agent evidence-reader --skill evidence-summary --role viewer --actor alice --project default --tenant tenant-a > "$WORK/viewer.json"
python3 - "$WORK/viewer.json" <<'PY' \
&& pass "viewer can bind read-only evidence-reader" || fail "viewer could not bind read-only agent"
import json, sys
d = json.load(open(sys.argv[1]))
assert d["success"] is True
assert d["decision"] == "BOUND"
assert d["agent_selected"] == "evidence-reader"
assert d["delegation_level"] == 0
assert d["tool_allowlist"] == []
PY
set +e
python3 "$RESOLVER" bind --agent ops-operator --role viewer --actor alice --project default --tenant tenant-a > "$WORK/viewer-op.json"
RC=$?
set -e 2>/dev/null || true
python3 - "$WORK/viewer-op.json" "$RC" <<'PY' \
&& pass "viewer cannot select operator agent" || fail "viewer selected operator agent"
import json, sys
d = json.load(open(sys.argv[1]))
assert int(sys.argv[2]) == 2
assert d["success"] is False
assert d["decision"] == "DENIED"
assert d["reason"] == "role_not_allowed_for_agent"
PY
set +e
python3 "$RESOLVER" bind --agent evidence-reader --role viewer --actor alice --project default --tenant tenant-a --target-tenant tenant-b > "$WORK/cross-tenant.json"
RC=$?
set -e 2>/dev/null || true
python3 - "$WORK/cross-tenant.json" "$RC" <<'PY' \
&& pass "cross-tenant agent binding is denied by RBAC" || fail "cross-tenant agent binding was allowed"
import json, sys
d = json.load(open(sys.argv[1]))
assert int(sys.argv[2]) == 2
assert d["success"] is False
assert d["decision"] == "DENIED"
assert "rbac_denied" in d["reason"]
PY
set +e
python3 "$RESOLVER" bind --agent evidence-reader --role viewer --actor alice --project default --tenant tenant-a --delegation-level 1 > "$WORK/delegation.json"
RC=$?
set -e 2>/dev/null || true
python3 - "$WORK/delegation.json" "$RC" <<'PY' \
&& pass "delegation above agent max requires approval" || fail "delegation escalation was not held"
import json, sys
d = json.load(open(sys.argv[1]))
assert int(sys.argv[2]) == 3
assert d["success"] is False
assert d["decision"] == "REQUIRES_APPROVAL"
assert d["requires_approval"] is True
PY
set +e
python3 "$RESOLVER" bind --agent ops-operator --role operator --actor bob --project default --tenant tenant-a --tool shell > "$WORK/tool.json"
RC=$?
set -e 2>/dev/null || true
python3 - "$WORK/tool.json" "$RC" <<'PY' \
&& pass "tool outside agent allowlist is blocked" || fail "tool outside allowlist was accepted"
import json, sys
d = json.load(open(sys.argv[1]))
assert int(sys.argv[2]) == 2
assert d["success"] is False
assert d["decision"] == "DENIED"
assert d["action_gate"]["outcome"] == "BLOCK"
PY
set +e
python3 "$TURN" ask --message "run tests" --agent ops-operator --role viewer --actor alice --chat-id agent-chat --tenant tenant-a > "$WORK/turn-deny.json"
RC=$?
set -e 2>/dev/null || true
python3 - "$WORK/turn-deny.json" "$RC" <<'PY' \
&& pass "chat-turn denies unauthorized selected agent before operator runtime" || fail "chat-turn bypassed agent resolver"
import json, sys
d = json.load(open(sys.argv[1]))
assert int(sys.argv[2]) == 2
assert d["success"] is False
assert d["decision"] == "DENIED"
assert d["agent_selected"] == "ops-operator"
PY
python3 "$RESOLVER" verify-audit > "$WORK/audit.out" \
&& grep -q "CHAT_AGENT_AUDIT ok=true" "$WORK/audit.out" \
&& pass "agent binding audit hash chain verifies" || fail "agent binding audit invalid"
echo ""
echo "===== CHAT AGENT SUMMARY: PASS=$PASS FAIL=$FAIL ====="
[[ "$FAIL" -eq 0 ]] || exit 1
@@ -37,7 +37,7 @@ ids = {a["id"] for a in d["actions"]}
assert {"run-chat-tests", "build-evidence-pack", "verify-evidence-pack"}.issubset(ids)
PY
python3 "$TURN" ask --message "run tests" --actor bob --chat-id op1 > "$WORK/run.json"
python3 "$TURN" ask --message "run tests" --actor bob --role operator --chat-id op1 > "$WORK/run.json"
python3 - "$WORK/run.json" <<'PY' \
&& pass "registered run tests action executes through chat-turn" || fail "registered action did not complete"
import json, os, sys