gitea-admin/CASAN
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: create new project added

Browse Source
This commit is contained in:
thanhnv
2026-07-11 22:42:42 +09:00
parent 193a449829
commit 159022c73f
11 changed files with 405 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/casan-harness/scripts/bash/goal-orchestrator.py
+105 -16
View File
@@ -303,7 +303,7 @@ def scan(text: str, mode: str):
return result.returncode == 0, safe
def call_model(model: str, prompt: str, cloud: bool):
def call_model(model: str, prompt: str, cloud: bool, timeout_seconds=None):
if not model:
return False, "", {}, "model_unconfigured"
with tempfile.TemporaryDirectory() as directory:
@@ -318,6 +318,10 @@ def call_model(model: str, prompt: str, cloud: bool):
# remains the hard ceiling; this only raises the router's 60s default.
environment.setdefault("CASAN_MODEL_TIMEOUT_SEC", os.environ.get("CASAN_GOAL_LOCAL_TIMEOUT_SEC", "240") if not cloud else "120")
environment.setdefault("CASAN_MODEL_GENERATE_MAX_TOKENS", os.environ.get("CASAN_GOAL_MAX_OUTPUT_TOKENS", "1400"))
timeout = max(1, int(timeout_seconds or os.environ.get("CASAN_GOAL_MODEL_TIMEOUT", "300")))
environment["CASAN_MODEL_TIMEOUT_SEC"] = str(min(
int(environment.get("CASAN_MODEL_TIMEOUT_SEC", timeout)), timeout
))
try:
result = subprocess.run(
["bash", MODEL_ROUTER, prompt_path, output_path, "--role", "generate", "--model", model],
@@ -325,7 +329,7 @@ def call_model(model: str, prompt: str, cloud: bool):
capture_output=True,
text=True,
env=environment,
timeout=int(os.environ.get("CASAN_GOAL_MODEL_TIMEOUT", "300")),
timeout=timeout,
)
except subprocess.TimeoutExpired:
return False, "", {}, "model_timeout"
@@ -349,7 +353,7 @@ def call_model(model: str, prompt: str, cloud: bool):
return True, text, metadata, "ok"
def call_account_model(provider: str, prompt: str):
def call_account_model(provider: str, prompt: str, timeout_seconds=None):
base_url = os.environ.get("CASAN_AUTH_BRIDGE_URL", "").rstrip("/")
token = os.environ.get("CASAN_AUTH_BRIDGE_TOKEN", "")
if provider not in {"codex", "claude"} or not base_url or not token:
@@ -361,7 +365,7 @@ def call_account_model(provider: str, prompt: str):
method="POST",
)
try:
with urllib.request.urlopen(request, timeout=310) as response:
with urllib.request.urlopen(request, timeout=max(1, int(timeout_seconds or 310))) as response:
payload = json.loads(response.read().decode("utf-8"))
except (urllib.error.URLError, TimeoutError, ValueError):
return False, "", {}, "account_bridge_failed"
@@ -378,6 +382,94 @@ def call_account_model(provider: str, prompt: str):
return True, text, metadata, "ok"
def reviewer_failure_retryable(reason: str) -> bool:
"""Only transient/provider failures may advance to another reviewer."""
normalized = str(reason or "").lower()
non_retryable = (
"endpoint_not_allowed", "model_unconfigured", "model_not_discovered",
"invalid_request", "content_policy", "security_blocked",
)
return not any(marker in normalized for marker in non_retryable)
def reviewer_candidates(account_provider: str, cloud_model: str, local_model: str):
"""Build a stable, de-duplicated account/cloud -> OmniRoute -> local chain."""
candidates = []
if account_provider:
candidates.append({
"kind": "account", "provider": f"{account_provider}-account",
"model": f"{account_provider}-account-default", "value": account_provider,
})
fallback_model = os.environ.get("CASAN_GOAL_CLOUD_FALLBACK_MODEL", "").strip()
if fallback_model:
fallback_provider = "omniroute" if fallback_model.startswith("openai-compatible:") else "cloud-fallback"
candidates.append({"kind": "model", "provider": fallback_provider, "model": fallback_model, "value": fallback_model})
elif cloud_model:
candidates.append({"kind": "model", "provider": os.environ.get("CASAN_GOAL_CLOUD_PROVIDER", "cloud"), "model": cloud_model, "value": cloud_model})
for model in [item.strip() for item in os.environ.get("CASAN_GOAL_CLOUD_MODELS", "").split(",") if item.strip()]:
provider = "anthropic" if model.startswith("anthropic:") else "openai" if model.startswith("openai:") else "cloud"
candidates.append({"kind": "model", "provider": provider, "model": model, "value": model})
raw_omniroute = os.environ.get("CASAN_GOAL_OMNIROUTE_MODELS", "")
omniroute_models = [item.strip() for item in raw_omniroute.split(",") if item.strip()]
for model in omniroute_models:
value = model if ":" in model else f"openai-compatible:{model}"
candidates.append({"kind": "model", "provider": "omniroute", "model": value, "value": value})
local_reviewer = os.environ.get("CASAN_GOAL_LOCAL_REVIEWER_MODEL", "").strip() or local_model
if local_reviewer:
candidates.append({"kind": "model", "provider": os.environ.get("CASAN_GOAL_LOCAL_PROVIDER", "local-policy"), "model": local_reviewer, "value": local_reviewer})
unique, seen = [], set()
for candidate in candidates:
key = (candidate["kind"], candidate["value"])
if key not in seen:
seen.add(key)
unique.append(candidate)
return unique
def run_reviewer_chain(job_path: str, prompt: str, account_provider: str, cloud_model: str, local_model: str):
max_attempts = max(1, min(int(os.environ.get("CASAN_GOAL_REVIEWER_MAX_ATTEMPTS", "5")), 10))
deadline_seconds = max(1, min(int(os.environ.get("CASAN_GOAL_REVIEWER_DEADLINE_SEC", "360")), 900))
deadline = time.monotonic() + deadline_seconds
ledger = []
update_job(job_path, reviewer_attempts=ledger)
last_reason = "reviewer_candidates_unavailable"
candidates = reviewer_candidates(account_provider, cloud_model, local_model)
if len(candidates) > max_attempts and candidates[-1].get("provider") == os.environ.get("CASAN_GOAL_LOCAL_PROVIDER", "local-policy"):
candidates = candidates[:max_attempts - 1] + [candidates[-1]] if max_attempts > 1 else [candidates[-1]]
else:
candidates = candidates[:max_attempts]
for candidate in candidates:
remaining = int(deadline - time.monotonic())
if remaining <= 0:
last_reason = "reviewer_deadline_exceeded"
break
attempt_number = len(ledger) + 1
stage(job_path, "cloud-reviewer", "running", f"Reviewer attempt {attempt_number}/{max_attempts}", candidate["provider"], candidate["model"])
started_at, started_clock = now(), time.monotonic()
if candidate["kind"] == "account":
ok, result, metadata, reason = call_account_model(candidate["value"], prompt, remaining)
else:
ok, result, metadata, reason = call_model(candidate["value"], prompt, candidate["provider"] != os.environ.get("CASAN_GOAL_LOCAL_PROVIDER", "local-policy"), remaining)
retryable = False if ok else reviewer_failure_retryable(reason)
ledger.append({
"attempt": attempt_number, "provider": candidate["provider"], "model": candidate["model"],
"status": "pass" if ok else "failed", "reason": reason, "retryable": retryable,
"started_at": started_at, "finished_at": now(),
"latency_ms": int((time.monotonic() - started_clock) * 1000),
})
update_job(job_path, reviewer_attempts=ledger)
if ok:
return True, result, metadata, "ok", candidate
last_reason = reason
if not retryable and any(marker in str(reason).lower() for marker in ("security_blocked", "content_policy")):
break
return False, "", {}, last_reason, (ledger[-1] if ledger else {})
def audit(job: dict, status: str) -> str:
path = os.path.join(STATE_ROOT, "logs", "audit", "goal-orchestrator.jsonl")
head_path = os.path.join(STATE_ROOT, "logs", "audit", "goal-orchestrator-head.txt")
@@ -510,7 +602,7 @@ def run(job_path: str) -> int:
update_job(job_path, local_draft=safe_local, local_usage=local_meta)
emit(goal_id, "H2-tool", "pass", "Local solution prepared", {"provider": job.get("local_provider", ""), "model": local_model, **local_meta})
stage(job_path, "cloud-reviewer", "running", "Cloud model is challenging and improving the local solution", job.get("cloud_provider", ""), cloud_model)
stage(job_path, "cloud-reviewer", "running", "Independent reviewer is challenging and improving the local solution", job.get("cloud_provider", ""), cloud_model)
emit(goal_id, "H3-eval", "running", "Cloud reviewer is evaluating the local solution", {"provider": job.get("cloud_provider", ""), "model": cloud_model})
review_prompt = (
"You are the cloud CASAN reviewer. Critically review the local worker's proposal "
@@ -522,26 +614,23 @@ def run(job_path: str) -> int:
f"OBJECTIVE:\n{safe_goal}\n\nWORKSPACE SNAPSHOT ({project_id}):\n{context_bundle}\n\n"
f"LOCAL WORKER PROPOSAL:\n{safe_local[:5000]}"
)
if account_provider:
cloud_ok, cloud_result, cloud_meta, cloud_reason = call_account_model(account_provider, review_prompt)
if not cloud_ok and os.environ.get("CASAN_GOAL_CLOUD_FALLBACK_MODEL"):
cloud_ok, cloud_result, cloud_meta, cloud_reason = call_model(
os.environ["CASAN_GOAL_CLOUD_FALLBACK_MODEL"], review_prompt, True
)
else:
cloud_ok, cloud_result, cloud_meta, cloud_reason = call_model(cloud_model, review_prompt, True)
cloud_ok, cloud_result, cloud_meta, cloud_reason, reviewer = run_reviewer_chain(
job_path, review_prompt, account_provider, cloud_model, local_model
)
reviewer_provider = str(reviewer.get("provider") or job.get("cloud_provider", ""))
reviewer_model = str(reviewer.get("model") or cloud_model)
if cloud_ok:
allowed, safe_result = scan(cloud_result, "output")
if not allowed:
emit(goal_id, "H4-security", "blocked", "Cloud reviewer output rejected")
raise ValueError("cloud_output_security_blocked")
stage(job_path, "cloud-reviewer", "pass", "Cloud review incorporated", job.get("cloud_provider", ""), cloud_model)
emit(goal_id, "H3-eval", "pass", "Cloud review incorporated", {"provider": job.get("cloud_provider", ""), "model": cloud_model, **cloud_meta})
stage(job_path, "cloud-reviewer", "pass", "Independent review incorporated", reviewer_provider, reviewer_model)
emit(goal_id, "H3-eval", "pass", "Independent review incorporated", {"provider": reviewer_provider, "model": reviewer_model, **cloud_meta})
final_status = "completed"
metric_status = "success"
else:
safe_result = safe_local
stage(job_path, "cloud-reviewer", "warning", cloud_reason, job.get("cloud_provider", ""), cloud_model)
stage(job_path, "cloud-reviewer", "warning", cloud_reason, reviewer_provider, reviewer_model)
emit(goal_id, "H3-eval", "warning", "Cloud reviewer unavailable; local solution retained", {"reason": cloud_reason})
final_status = "degraded"
metric_status = "degraded"
packages/casan-harness/tests/goal-orchestrator-fallback-tests.py
+120
View File
@@ -0,0 +1,120 @@
#!/usr/bin/env python3
import importlib.util
import json
import os
import tempfile
import unittest
from unittest.mock import patch
ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", ".."))
SCRIPT = os.path.join(ROOT, "packages", "casan-harness", "scripts", "bash", "goal-orchestrator.py")
SPEC = importlib.util.spec_from_file_location("goal_orchestrator", SCRIPT)
MODULE = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(MODULE)
def job_file(directory):
path = os.path.join(directory, "job.json")
with open(path, "w", encoding="utf-8") as handle:
json.dump({
"id": "11111111-1111-1111-1111-111111111111",
"updated_at": MODULE.now(),
"stages": [{"id": "cloud-reviewer", "status": "queued"}],
}, handle)
return path
class ReviewerFallbackTests(unittest.TestCase):
def test_account_then_omniroute_then_local_and_persists_ledger(self):
calls = []
def account(provider, prompt, timeout):
calls.append(("account", provider))
return False, "", {}, "account_bridge_failed"
def model(model, prompt, cloud, timeout):
calls.append(("model", model, cloud))
if model.startswith("openai-compatible:"):
return False, "", {}, "model_timeout"
return True, "reviewed result", {"output_tokens": 7}, "ok"
environment = {
"CASAN_GOAL_OMNIROUTE_MODELS": "route-a,openai-compatible:route-b",
"CASAN_GOAL_LOCAL_PROVIDER": "ollama",
"CASAN_GOAL_REVIEWER_MAX_ATTEMPTS": "4",
"CASAN_GOAL_REVIEWER_DEADLINE_SEC": "30",
}
with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, environment, clear=False), \
patch.object(MODULE, "call_account_model", account), patch.object(MODULE, "call_model", model):
path = job_file(directory)
ok, result, metadata, reason, reviewer = MODULE.run_reviewer_chain(
path, "review prompt", "claude", "openai:gpt-primary", "ollama:worker-model"
)
with open(path, encoding="utf-8") as handle:
job = json.load(handle)
self.assertTrue(ok)
self.assertEqual(result, "reviewed result")
self.assertEqual(metadata["output_tokens"], 7)
self.assertEqual(reason, "ok")
self.assertEqual(reviewer["provider"], "ollama")
self.assertEqual(calls, [
("account", "claude"),
("model", "openai-compatible:route-a", True),
("model", "openai-compatible:route-b", True),
("model", "ollama:worker-model", False),
])
self.assertEqual([row["status"] for row in job["reviewer_attempts"]], ["failed", "failed", "failed", "pass"])
self.assertEqual(job["stages"][0]["provider"], "ollama")
self.assertEqual(job["stages"][0]["model"], "ollama:worker-model")
def test_attempt_limit_is_bounded(self):
calls = []
def model(model, prompt, cloud, timeout):
calls.append(model)
return False, "", {}, "model_timeout"
environment = {
"CASAN_GOAL_OMNIROUTE_MODELS": "route-a,route-b,route-c",
"CASAN_GOAL_REVIEWER_MAX_ATTEMPTS": "2",
"CASAN_GOAL_REVIEWER_DEADLINE_SEC": "30",
}
with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, environment, clear=False), patch.object(MODULE, "call_model", model):
path = job_file(directory)
ok, _, _, reason, _ = MODULE.run_reviewer_chain(path, "prompt", "", "openai:gpt", "ollama:local")
with open(path, encoding="utf-8") as handle:
job = json.load(handle)
self.assertFalse(ok)
self.assertEqual(reason, "model_timeout")
self.assertEqual(len(calls), 2)
self.assertEqual(len(job["reviewer_attempts"]), 2)
def test_non_retryable_candidate_failure_advances_to_fallbacks(self):
calls = []
def model(model, prompt, cloud, timeout):
calls.append(model)
return False, "", {}, "endpoint_not_allowed"
environment = {
"CASAN_GOAL_OMNIROUTE_MODELS": "route-a",
"CASAN_GOAL_REVIEWER_MAX_ATTEMPTS": "5",
"CASAN_GOAL_REVIEWER_DEADLINE_SEC": "30",
}
with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, environment, clear=False), patch.object(MODULE, "call_model", model):
path = job_file(directory)
ok, _, _, reason, _ = MODULE.run_reviewer_chain(path, "prompt", "", "openai:gpt", "ollama:local")
with open(path, encoding="utf-8") as handle:
job = json.load(handle)
self.assertFalse(ok)
self.assertEqual(reason, "endpoint_not_allowed")
self.assertEqual(calls, ["openai:gpt", "openai-compatible:route-a", "ollama:local"])
self.assertFalse(job["reviewer_attempts"][0]["retryable"])
if __name__ == "__main__":
unittest.main()