feat: govern chat model selection by policy
This commit is contained in:
thanhnv
@@ -50,4 +50,12 @@ export class ChatController {
|
||||
agents(@Headers() headers: Record<string, string | string[] | undefined>) {
|
||||
return ok(this.svc.listAgents(actorFromHeaders(headers)));
|
||||
}
|
||||
|
||||
@Get('models')
|
||||
models(
|
||||
@Headers() headers: Record<string, string | string[] | undefined>,
|
||||
@Query('modelRole') modelRole?: string,
|
||||
) {
|
||||
return ok(this.svc.listModels(actorFromHeaders(headers), modelRole || 'read_only'));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ export interface ChatAskInput {
|
||||
chatId?: string;
|
||||
agentId?: string;
|
||||
skillId?: string;
|
||||
modelProvider?: string;
|
||||
delegationLevel?: number;
|
||||
}
|
||||
|
||||
@@ -78,6 +79,7 @@ export class ChatService {
|
||||
];
|
||||
if (input.agentId) args.push('--agent', input.agentId);
|
||||
if (input.skillId) args.push('--skill', input.skillId);
|
||||
if (input.modelProvider) args.push('--model-provider', input.modelProvider);
|
||||
if (input.delegationLevel !== undefined) args.push('--delegation-level', String(input.delegationLevel));
|
||||
const res = runPython(CHAT_CLI, args);
|
||||
const parsed = parseJson<Record<string, any>>(res.stdout);
|
||||
@@ -136,6 +138,7 @@ export class ChatService {
|
||||
];
|
||||
if (input.agentId) args.push('--agent', input.agentId);
|
||||
if (input.skillId) args.push('--skill', input.skillId);
|
||||
if (input.modelProvider) args.push('--model-provider', input.modelProvider);
|
||||
if (input.delegationLevel !== undefined) args.push('--delegation-level', String(input.delegationLevel));
|
||||
res.setHeader('Content-Type', 'application/x-ndjson; charset=utf-8');
|
||||
res.setHeader('Cache-Control', 'no-cache');
|
||||
@@ -176,6 +179,16 @@ export class ChatService {
|
||||
throw new InternalServerErrorException(res.stderr || res.stdout || 'CHAT_AGENTS_FAILED');
|
||||
}
|
||||
|
||||
listModels(actor: SettingsActor, modelRole = 'read_only') {
|
||||
this.requireRead(actor);
|
||||
const res = runPython(join(HARNESS_BIN, 'chat-model-resolver.py'), [
|
||||
'list', '--actor', actor.actor, '--role', actor.role, '--model-role', modelRole,
|
||||
]);
|
||||
const parsed = parseJson<Record<string, unknown>>(res.stdout);
|
||||
if (parsed) return parsed;
|
||||
throw new InternalServerErrorException(res.stderr || res.stdout || 'CHAT_MODELS_FAILED');
|
||||
}
|
||||
|
||||
private requireRead(actor: SettingsActor) {
|
||||
const res = runPython(RBAC_CLI, [
|
||||
'check',
|
||||
|
||||
@@ -185,6 +185,16 @@ export interface ChatAgent {
|
||||
allowed_for_role?: boolean;
|
||||
}
|
||||
|
||||
export interface ChatModelProvider {
|
||||
id: string;
|
||||
model: string;
|
||||
class: string;
|
||||
requires_preflight: boolean;
|
||||
allowed: boolean;
|
||||
configured: boolean;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
export interface ChatReplay {
|
||||
ok: boolean;
|
||||
decision: 'MATCH' | 'DRIFT' | 'BREAK' | string;
|
||||
@@ -268,11 +278,11 @@ export const api = {
|
||||
post<{ proposal: any; audit_verify: { ok: boolean; output: string } }>('approvals/submit', body, actorHeaders(actor)),
|
||||
decideApproval: (actor: SettingsActor, body: { id: string; decision: 'approve' | 'reject'; reason: string; approvalJwt?: string }) =>
|
||||
post<{ proposal: any; applied: any; audit_verify: { ok: boolean; output: string } }>('approvals/decide', body, actorHeaders(actor)),
|
||||
askChat: (actor: SettingsActor, body: { message: string; chatId?: string; agentId?: string; skillId?: string; delegationLevel?: number }) =>
|
||||
askChat: (actor: SettingsActor, body: { message: string; chatId?: string; agentId?: string; skillId?: string; modelProvider?: string; delegationLevel?: number }) =>
|
||||
post<ChatAnswer>('chat/ask', body, actorHeaders(actor)),
|
||||
askChatStream: async (
|
||||
actor: SettingsActor,
|
||||
body: { message: string; chatId?: string; agentId?: string; skillId?: string; delegationLevel?: number },
|
||||
body: { message: string; chatId?: string; agentId?: string; skillId?: string; modelProvider?: string; delegationLevel?: number },
|
||||
onPhase: (phase: ChatStreamPhase) => void,
|
||||
): Promise<void> => {
|
||||
const base = import.meta.env.VITE_API_BASE_URL ?? '/api/v1';
|
||||
@@ -314,6 +324,7 @@ export const api = {
|
||||
replayChat: (chatId = '', turnId = '', tenant = '') => get<ChatReplay>(`chat/replay?chatId=${encodeURIComponent(chatId)}&turnId=${encodeURIComponent(turnId)}&tenant=${encodeURIComponent(tenant)}`),
|
||||
chatActions: (actor: SettingsActor) => getWithHeaders<{ success: boolean; actions: ChatAction[] }>('chat/actions', actorHeaders(actor)),
|
||||
chatAgents: (actor: SettingsActor) => getWithHeaders<{ success: boolean; agents: ChatAgent[] }>('chat/agents', actorHeaders(actor)),
|
||||
chatModels: (actor: SettingsActor, modelRole: string) => getWithHeaders<{ success: boolean; default: string; providers: ChatModelProvider[] }>(`chat/models?modelRole=${encodeURIComponent(modelRole)}`, actorHeaders(actor)),
|
||||
};
|
||||
|
||||
// Health is raw (not enveloped) + carries HTTP status.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { type KeyboardEvent, useMemo, useState } from 'react';
|
||||
import { useMutation, useQuery } from '@tanstack/react-query';
|
||||
import { api, ChatAction, ChatAgent, ChatAnswer, ChatHistoryTurn, ChatStreamPhase, SettingsActor } from '../lib/api';
|
||||
import { api, ChatAction, ChatAgent, ChatAnswer, ChatHistoryTurn, ChatModelProvider, ChatStreamPhase, SettingsActor } from '../lib/api';
|
||||
import { Card, StatusBadge } from '../components/ui/Card';
|
||||
|
||||
const ROLES = ['viewer', 'auditor', 'operator', 'project-admin', 'org-admin'];
|
||||
@@ -82,6 +82,7 @@ export function Chat() {
|
||||
const [chatId, setChatId] = useState('chat-default');
|
||||
const [agentId, setAgentId] = useState('evidence-reader');
|
||||
const [skillId, setSkillId] = useState('evidence-summary');
|
||||
const [modelProvider, setModelProvider] = useState('local');
|
||||
const [delegationLevel, setDelegationLevel] = useState(0);
|
||||
const [message, setMessage] = useState('');
|
||||
const [last, setLast] = useState<ChatAnswer | null>(null);
|
||||
@@ -100,6 +101,9 @@ export function Chat() {
|
||||
const agents = agentsQuery.data?.agents ?? [];
|
||||
const selectedAgent = agents.find((agent) => agent.id === agentId) ?? agents.find((agent) => agent.allowed_for_role) ?? agents[0];
|
||||
const selectedSkill = selectedAgent?.skills_allowed.includes(skillId) ? skillId : (selectedAgent?.skills_allowed[0] ?? '');
|
||||
const modelsQuery = useQuery({ queryKey: ['chat-models', actor, selectedAgent?.model_role], queryFn: () => api.chatModels(actor, selectedAgent?.model_role ?? 'read_only'), retry: false });
|
||||
const models = modelsQuery.data?.providers ?? [];
|
||||
const selectedModel = models.find((provider) => provider.id === modelProvider) ?? models.find((provider) => provider.id === modelsQuery.data?.default) ?? models[0];
|
||||
const persistedMessages = useMemo(() => (historyQuery.data?.turns ?? []).flatMap(turnMessages), [historyQuery.data]);
|
||||
const liveAlreadyStored = Boolean(last?.turn_id && historyQuery.data?.turns.some((turn) => turn.turn_id === last.turn_id));
|
||||
const liveMessages: WorkspaceMessage[] = [];
|
||||
@@ -120,6 +124,7 @@ export function Chat() {
|
||||
chatId,
|
||||
agentId: override?.agentId ?? selectedAgent?.id ?? agentId,
|
||||
skillId: override?.skillId ?? selectedSkill,
|
||||
modelProvider: selectedModel?.id,
|
||||
delegationLevel,
|
||||
}),
|
||||
onSuccess: (answer) => {
|
||||
@@ -143,7 +148,7 @@ export function Chat() {
|
||||
setDraftText(null);
|
||||
setPendingMessage(text);
|
||||
try {
|
||||
await api.askChatStream(actor, { message: text, chatId, agentId: selectedAgent?.id ?? agentId, skillId: selectedSkill, delegationLevel }, (phase) => {
|
||||
await api.askChatStream(actor, { message: text, chatId, agentId: selectedAgent?.id ?? agentId, skillId: selectedSkill, modelProvider: selectedModel?.id, delegationLevel }, (phase) => {
|
||||
if (phase.phase === 'draft') setDraftText(phase.answer);
|
||||
if (phase.phase === 'final') {
|
||||
setDraftText(null);
|
||||
@@ -267,7 +272,13 @@ export function Chat() {
|
||||
<div className="text-[10px] font-bold uppercase tracking-[0.15em] text-slate-400">Governance context</div>
|
||||
<div className="mt-3 rounded-xl border border-slate-200 bg-white p-3.5">
|
||||
<div className="flex items-center justify-between gap-2"><div className="text-sm font-semibold text-slate-800">{selectedAgent?.label ?? 'Evidence reader'}</div><StatusBadge value={selectedAgent?.allowed_for_role ? 'allowed' : 'locked'} /></div>
|
||||
<div className="mt-1 text-xs text-slate-500">{selectedAgent?.model_role ?? 'read_only'} · {selectedSkill || 'no skill selected'}</div>
|
||||
<div className="mt-1 text-xs text-slate-500">{selectedAgent?.model_role ?? 'read_only'} · {selectedSkill || 'no skill selected'} · {selectedModel?.model ?? 'model unavailable'}</div>
|
||||
<div className="mt-3 grid gap-2 border-t border-slate-100 pt-3 text-xs text-slate-600">
|
||||
<label>Agent<select value={selectedAgent?.id ?? agentId} onChange={(event) => { const next = agents.find((agent) => agent.id === event.target.value); setAgentId(event.target.value); setSkillId(next?.skills_allowed[0] ?? ''); }} className="mt-1 w-full rounded-lg border border-slate-200 px-2 py-1.5 text-xs">{agents.map((agent: ChatAgent) => <option key={agent.id} value={agent.id} disabled={!agent.allowed_for_role}>{agent.label}{agent.allowed_for_role ? '' : ' (locked)'}</option>)}</select></label>
|
||||
<label>Skill<select value={selectedSkill} onChange={(event) => setSkillId(event.target.value)} className="mt-1 w-full rounded-lg border border-slate-200 px-2 py-1.5 text-xs">{(selectedAgent?.skills_allowed ?? []).map((skill) => <option key={skill}>{skill}</option>)}</select></label>
|
||||
<label>Model provider<select value={selectedModel?.id ?? ''} onChange={(event) => setModelProvider(event.target.value)} className="mt-1 w-full rounded-lg border border-slate-200 px-2 py-1.5 text-xs">{models.map((provider: ChatModelProvider) => <option key={provider.id} value={provider.id} disabled={!provider.allowed || !provider.configured}>{provider.id} · {provider.model}{provider.allowed && provider.configured ? '' : ` (${provider.reason || 'not allowed'})`}</option>)}</select></label>
|
||||
<p className="text-[11px] leading-4 text-slate-500">Provider is policy-bound to your role and agent; cloud/gateway routes enforce preflight.</p>
|
||||
</div>
|
||||
<details className="mt-3 border-t border-slate-100 pt-3 text-xs text-slate-600"><summary className="cursor-pointer font-medium text-slate-700">Session scope</summary><div className="mt-3 grid grid-cols-2 gap-2"><label className="col-span-2">Actor<input value={actor.actor} onChange={(event) => setActor({ ...actor, actor: event.target.value })} className="mt-1 w-full rounded-lg border border-slate-200 px-2 py-1.5 text-xs" /></label><label>Role<select value={actor.role} onChange={(event) => setActor({ ...actor, role: event.target.value })} className="mt-1 w-full rounded-lg border border-slate-200 px-2 py-1.5 text-xs">{ROLES.map((role) => <option key={role}>{role}</option>)}</select></label><label>Delegate<input type="number" min={0} max={5} value={delegationLevel} onChange={(event) => setDelegationLevel(Number(event.target.value || 0))} className="mt-1 w-full rounded-lg border border-slate-200 px-2 py-1.5 text-xs" /></label><label>Project<input value={actor.project} onChange={(event) => setActor({ ...actor, project: event.target.value })} className="mt-1 w-full rounded-lg border border-slate-200 px-2 py-1.5 text-xs" /></label><label>Tenant<input value={actor.tenant} onChange={(event) => setActor({ ...actor, tenant: event.target.value })} className="mt-1 w-full rounded-lg border border-slate-200 px-2 py-1.5 text-xs" /></label><label className="col-span-2">Agent<select value={selectedAgent?.id ?? agentId} onChange={(event) => { const next = agents.find((agent) => agent.id === event.target.value); setAgentId(event.target.value); setSkillId(next?.skills_allowed[0] ?? ''); }} className="mt-1 w-full rounded-lg border border-slate-200 px-2 py-1.5 text-xs">{agents.map((agent: ChatAgent) => <option key={agent.id} value={agent.id}>{agent.label}{agent.allowed_for_role ? '' : ' (locked)'}</option>)}</select></label><label className="col-span-2">Skill<select value={selectedSkill} onChange={(event) => setSkillId(event.target.value)} className="mt-1 w-full rounded-lg border border-slate-200 px-2 py-1.5 text-xs">{(selectedAgent?.skills_allowed ?? []).map((skill) => <option key={skill}>{skill}</option>)}</select></label></div></details>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -6,26 +6,34 @@
|
||||
"local": {
|
||||
"model": "ollama:ornith:9b",
|
||||
"class": "local",
|
||||
"requires_key": false
|
||||
"requires_key": false,
|
||||
"roles_allowed": ["viewer", "auditor", "operator", "project-admin", "org-admin"],
|
||||
"model_roles_allowed": ["read_only", "analysis", "operator", "codegen"]
|
||||
},
|
||||
"cloud-anthropic": {
|
||||
"model": "anthropic:claude-3-5-sonnet-latest",
|
||||
"class": "cloud",
|
||||
"requires_key": true,
|
||||
"key_env": "ANTHROPIC_API_KEY"
|
||||
"key_env": "ANTHROPIC_API_KEY",
|
||||
"roles_allowed": ["project-admin", "org-admin"],
|
||||
"model_roles_allowed": ["analysis", "codegen"]
|
||||
},
|
||||
"cloud-openai": {
|
||||
"model": "openai:gpt-4o-mini",
|
||||
"class": "cloud",
|
||||
"requires_key": true,
|
||||
"key_env": "OPENAI_API_KEY"
|
||||
"key_env": "OPENAI_API_KEY",
|
||||
"roles_allowed": ["project-admin", "org-admin"],
|
||||
"model_roles_allowed": ["analysis", "codegen"]
|
||||
},
|
||||
"omniroute": {
|
||||
"model": "openai-compatible:auto/best-coding",
|
||||
"class": "gateway",
|
||||
"requires_key": true,
|
||||
"key_env": "CASAN_OPENAI_COMPATIBLE_API_KEY",
|
||||
"requires_preflight": true
|
||||
"requires_preflight": true,
|
||||
"roles_allowed": ["operator", "project-admin", "org-admin"],
|
||||
"model_roles_allowed": ["read_only", "analysis", "operator", "codegen"]
|
||||
}
|
||||
},
|
||||
"role_bindings": {
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Governed model-provider selector for Chat.
|
||||
|
||||
The browser sends only a provider ID. This resolver reads the managed provider
|
||||
policy, checks actor role, agent model role and credential availability, writes
|
||||
an append-only selection audit, and returns no secret material.
|
||||
"""
|
||||
import argparse, hashlib, json, os, sys
|
||||
from datetime import datetime, timezone
|
||||
|
||||
ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", ".."))
|
||||
CONFIG = os.path.join(ROOT, "packages", "casan-harness", "config", "model-providers.yaml")
|
||||
|
||||
def now(): return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
def sha(v): return hashlib.sha256(v.encode()).hexdigest()
|
||||
def state_root(): return os.environ.get("CASAN_STATE_ROOT") or os.path.join(ROOT, ".specify")
|
||||
def audit_path(): return os.environ.get("CASAN_CHAT_MODEL_AUDIT_LOG") or os.path.join(state_root(), "logs", "chat", "model-selection.jsonl")
|
||||
def load():
|
||||
try:
|
||||
return json.load(open(os.environ.get("CASAN_MODEL_PROVIDERS_FILE") or CONFIG, encoding="utf-8"))
|
||||
except Exception as exc:
|
||||
return {"_error": f"providers_unreadable:{exc}"}
|
||||
def visible(pid, p, role, model_role):
|
||||
allowed_roles = p.get("roles_allowed", [])
|
||||
allowed_models = p.get("model_roles_allowed", [])
|
||||
enabled = not p.get("requires_key") or bool(os.environ.get(p.get("key_env", "")))
|
||||
return {"id": pid, "model": p.get("model", ""), "class": p.get("class", "local"),
|
||||
"requires_preflight": bool(p.get("requires_preflight")), "allowed": role in allowed_roles and model_role in allowed_models,
|
||||
"configured": enabled, "reason": "" if enabled else "credential_not_configured"}
|
||||
def append(rec):
|
||||
path=audit_path(); os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
prev=""
|
||||
if os.path.isfile(path):
|
||||
for line in open(path, encoding="utf-8"):
|
||||
if line.strip(): prev=json.loads(line).get("record_hash", prev)
|
||||
core={**rec,"prev_hash":prev}; core["record_hash"]=sha(json.dumps(core,sort_keys=True,ensure_ascii=False))
|
||||
with open(path,"a",encoding="utf-8") as f: f.write(json.dumps(core,ensure_ascii=False)+"\n")
|
||||
return core
|
||||
def list_models(args):
|
||||
cfg=load()
|
||||
if cfg.get("_error"): print(json.dumps({"success":False,"providers":[],"reason":cfg["_error"]})); return 2
|
||||
rows=[visible(pid,p,args.role,args.model_role) for pid,p in cfg.get("providers",{}).items()]
|
||||
print(json.dumps({"success":True,"model_role":args.model_role,"default":cfg.get("role_bindings",{}).get(args.model_role,""),"providers":rows}))
|
||||
return 0
|
||||
def bind(args):
|
||||
cfg=load()
|
||||
if cfg.get("_error"): print(json.dumps({"success":False,"decision":"DENIED","reason":cfg["_error"]})); return 2
|
||||
pid=args.provider or cfg.get("role_bindings",{}).get(args.model_role,"")
|
||||
p=cfg.get("providers",{}).get(pid)
|
||||
if not p:
|
||||
decision,reason="DENIED","unknown_provider"
|
||||
else:
|
||||
row=visible(pid,p,args.role,args.model_role)
|
||||
decision,reason=("BOUND", "provider_bound") if row["allowed"] and row["configured"] else ("DENIED", row["reason"] or "provider_not_allowed")
|
||||
rec=append({"timestamp":now(),"harness":"H5-chat-model-bind","actor":args.actor,"role":args.role,"model_role":args.model_role,"provider":pid,"decision":decision,"reason":reason})
|
||||
out={"success":decision=="BOUND","decision":decision,"reason":reason,"provider":pid,"model":(p or {}).get("model"),"class":(p or {}).get("class"),"audit":{"seq":rec.get("record_hash","")[:16],"record_hash":rec["record_hash"]}}
|
||||
print(json.dumps(out,ensure_ascii=False)); return 0 if decision=="BOUND" else 2
|
||||
def main():
|
||||
ap=argparse.ArgumentParser(); subs=ap.add_subparsers(dest="cmd",required=True)
|
||||
for name in ("list","bind"):
|
||||
p=subs.add_parser(name); p.add_argument("--role",required=True); p.add_argument("--model-role",required=True); p.add_argument("--actor",default="anonymous")
|
||||
if name=="bind": p.add_argument("--provider",default="")
|
||||
args=ap.parse_args(); return list_models(args) if args.cmd=="list" else bind(args)
|
||||
if __name__=="__main__": raise SystemExit(main())
|
||||
@@ -29,6 +29,7 @@ ROOT = project_root()
|
||||
BIN = os.path.join(ROOT, "packages", "casan-harness", "scripts", "bash")
|
||||
ROUTER = os.path.join(BIN, "prompt-mode-router.py")
|
||||
AGENT_RESOLVER = os.path.join(BIN, "chat-agent-resolver.py")
|
||||
MODEL_RESOLVER = os.path.join(BIN, "chat-model-resolver.py")
|
||||
READONLY = os.path.join(BIN, "chat-readonly.py")
|
||||
OPERATOR = os.path.join(BIN, "chat-operator.py")
|
||||
APPROVAL_INBOX = os.path.join(BIN, "approval-inbox.py")
|
||||
@@ -112,6 +113,8 @@ def run_mode(args, binding, loop_run=None, scan_output_label=""):
|
||||
env = os.environ.copy()
|
||||
if loop_run is not None:
|
||||
env["CASAN_CHAT_LOOP_RUN_JSON"] = json.dumps(loop_run, ensure_ascii=False, sort_keys=True)
|
||||
if binding and binding.get("model_binding", {}).get("decision") == "BOUND":
|
||||
env["CASAN_CHAT_MODEL_PROVIDER"] = binding["model_binding"].get("provider", "")
|
||||
r = subprocess.run(args, cwd=ROOT, capture_output=True, text=True, env=env)
|
||||
if scan_output_label:
|
||||
scan_rc, scan_msg = scan_tool_output(r.stdout + "\n" + r.stderr, scan_output_label)
|
||||
@@ -144,8 +147,8 @@ def run_mode(args, binding, loop_run=None, scan_output_label=""):
|
||||
return r.returncode
|
||||
|
||||
|
||||
def run_and_passthrough(args):
|
||||
r = subprocess.run(args, cwd=ROOT, text=True)
|
||||
def run_and_passthrough(args, env=None):
|
||||
r = subprocess.run(args, cwd=ROOT, text=True, env=env)
|
||||
return r.returncode
|
||||
|
||||
|
||||
@@ -622,6 +625,20 @@ def bind_agent(args, router):
|
||||
return 2, None
|
||||
|
||||
|
||||
def bind_model(args, binding):
|
||||
r = subprocess.run([
|
||||
"python3", MODEL_RESOLVER, "bind",
|
||||
"--provider", args.model_provider,
|
||||
"--actor", args.actor,
|
||||
"--role", args.role,
|
||||
"--model-role", binding.get("model_role") or "read_only",
|
||||
], cwd=ROOT, capture_output=True, text=True)
|
||||
try:
|
||||
return r.returncode, json.loads(r.stdout)
|
||||
except Exception:
|
||||
return 2, {"success": False, "decision": "DENIED", "reason": "model_resolver_invalid_json"}
|
||||
|
||||
|
||||
def submit_escalation(args, router, binding):
|
||||
turn_id = args.turn_id or str(uuid.uuid4())
|
||||
payload = {
|
||||
@@ -695,6 +712,12 @@ def ask(args) -> int:
|
||||
if binding:
|
||||
print(json.dumps(binding, ensure_ascii=False))
|
||||
return bind_rc
|
||||
model_rc, model_binding = bind_model(args, binding)
|
||||
if model_rc != 0:
|
||||
print(json.dumps({"success": False, "mode": router.get("mode"), "decision": "DENIED", "reason": model_binding.get("reason"), "agent_binding": binding, "model_binding": model_binding}, ensure_ascii=False))
|
||||
return model_rc
|
||||
binding["model_binding"] = model_binding
|
||||
os.environ["CASAN_CHAT_MODEL_PROVIDER"] = model_binding.get("provider", "")
|
||||
common = [
|
||||
"--message", args.message,
|
||||
"--actor", args.actor,
|
||||
@@ -712,7 +735,9 @@ def ask(args) -> int:
|
||||
loop_run, loop_rc = certify_codegen_draft(args, router, binding)
|
||||
return finish_codegen(args, router, binding, loop_run, loop_rc)
|
||||
if getattr(args, "stream", False) and router.get("mode") in ("READ_ONLY", "ANALYSIS"):
|
||||
return run_and_passthrough(["python3", READONLY, "ask", "--stream", *common])
|
||||
env = os.environ.copy()
|
||||
env["CASAN_CHAT_MODEL_PROVIDER"] = model_binding.get("provider", "")
|
||||
return run_and_passthrough(["python3", READONLY, "ask", "--stream", *common], env)
|
||||
return run_mode(["python3", READONLY, "ask", *common], binding)
|
||||
|
||||
|
||||
@@ -740,6 +765,7 @@ def main() -> int:
|
||||
askp.add_argument("--tenant", default="default")
|
||||
askp.add_argument("--agent", default="")
|
||||
askp.add_argument("--skill", default="")
|
||||
askp.add_argument("--model-provider", default="")
|
||||
askp.add_argument("--delegation-level", type=int, default=0)
|
||||
askp.add_argument("--stream", action="store_true")
|
||||
askp.set_defaults(func=ask)
|
||||
|
||||
@@ -151,6 +151,7 @@ run "phase-loop-run" bash "$TESTS/phase-loop-run-tests.sh"
|
||||
run "phase-chat-prompt-router" bash "$TESTS/phase-chat-prompt-router-tests.sh"
|
||||
run "phase-chat-readonly" bash "$TESTS/phase-chat-readonly-tests.sh"
|
||||
run "phase-chat-model-synthesis" bash "$TESTS/phase-chat-model-synthesis-tests.sh"
|
||||
run "phase-chat-model-selection" bash "$TESTS/phase-chat-model-selection-tests.sh"
|
||||
run "phase-chat-advanced" bash "$TESTS/phase-chat-advanced-tests.sh"
|
||||
run "phase-chat-session-audit" bash "$TESTS/phase-chat-session-audit-tests.sh"
|
||||
run "phase-chat-operator" bash "$TESTS/phase-chat-operator-tests.sh"
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$SCRIPT_DIR/../scripts/bash/casan-paths.sh"
|
||||
RESOLVER="$CASAN_HARNESS_ROOT/scripts/bash/chat-model-resolver.py"
|
||||
TURN="$CASAN_HARNESS_ROOT/scripts/bash/chat-turn.py"
|
||||
W="$(mktemp -d)"; trap 'rm -rf "$W"' EXIT
|
||||
export CASAN_STATE_ROOT="$W/state"
|
||||
PASS=0; FAIL=0
|
||||
pass() { echo "PASS: $1"; PASS=$((PASS+1)); }
|
||||
fail() { echo "FAIL: $1"; FAIL=$((FAIL+1)); }
|
||||
python3 "$RESOLVER" list --actor viewer-a --role viewer --model-role read_only > "$W/list.json"
|
||||
python3 - "$W/list.json" <<'PY' && pass "viewer can see only policy-allowed local read model" || fail "model list policy incorrect"
|
||||
import json, sys
|
||||
d=json.load(open(sys.argv[1])); local=next(x for x in d['providers'] if x['id']=='local')
|
||||
cloud=next(x for x in d['providers'] if x['id']=='cloud-openai')
|
||||
assert local['allowed'] and local['configured']; assert not cloud['allowed']
|
||||
PY
|
||||
python3 "$RESOLVER" bind --actor viewer-a --role viewer --model-role read_only --provider local > "$W/local.json" \
|
||||
&& grep -q '"decision": "BOUND"' "$W/local.json" && pass "local provider selection is audited and bound" || fail "local provider binding failed"
|
||||
set +e
|
||||
python3 "$RESOLVER" bind --actor viewer-a --role viewer --model-role read_only --provider cloud-openai > "$W/cloud.json"
|
||||
RC=$?
|
||||
set -e
|
||||
[[ "$RC" -eq 2 ]] && grep -q '"decision": "DENIED"' "$W/cloud.json" && pass "unauthorized or unconfigured cloud provider denied" || fail "cloud provider was not denied"
|
||||
set +e
|
||||
python3 "$TURN" ask --message 'Summarize Plan 18 status' --actor viewer-a --role viewer --project default --chat-id model-select --model-provider cloud-openai > "$W/turn.json"
|
||||
RC=$?
|
||||
set -e
|
||||
[[ "$RC" -eq 2 ]] && grep -q 'credential_not_configured\|provider_not_allowed' "$W/turn.json" && pass "chat turn cannot bypass model provider policy" || fail "chat bypassed model policy"
|
||||
echo "===== CHAT MODEL SELECTION SUMMARY: PASS=$PASS FAIL=$FAIL ====="
|
||||
[[ "$FAIL" -eq 0 ]]
|
||||
Reference in New Issue
Block a user