202 lines
8.6 KiB
Python
202 lines
8.6 KiB
Python
#!/usr/bin/env python3
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
|
|
ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", ".."))
|
|
SCRIPT = os.path.join(ROOT, "packages", "casan-harness", "scripts", "bash", "goal-orchestrator.py")
|
|
SPEC = importlib.util.spec_from_file_location("goal_orchestrator", SCRIPT)
|
|
MODULE = importlib.util.module_from_spec(SPEC)
|
|
SPEC.loader.exec_module(MODULE)
|
|
|
|
|
|
def job_file(directory):
|
|
path = os.path.join(directory, "job.json")
|
|
with open(path, "w", encoding="utf-8") as handle:
|
|
json.dump({
|
|
"id": "11111111-1111-1111-1111-111111111111",
|
|
"updated_at": MODULE.now(),
|
|
"stages": [{"id": "cloud-reviewer", "status": "queued"}],
|
|
}, handle)
|
|
return path
|
|
|
|
|
|
class ReviewerFallbackTests(unittest.TestCase):
|
|
def setUp(self):
|
|
# Production enables route preflight. Individual tests opt in explicitly
|
|
# so cached/live provider health cannot affect deterministic unit tests.
|
|
self.environment = patch.dict(
|
|
os.environ, {"CASAN_GOAL_MODEL_PREFLIGHT": "0"}, clear=False,
|
|
)
|
|
self.environment.start()
|
|
self.addCleanup(self.environment.stop)
|
|
|
|
def test_account_then_omniroute_then_local_and_persists_ledger(self):
|
|
calls = []
|
|
|
|
def account(provider, prompt, timeout):
|
|
calls.append(("account", provider))
|
|
return False, "", {}, "account_bridge_failed"
|
|
|
|
def model(model, prompt, cloud, timeout):
|
|
calls.append(("model", model, cloud))
|
|
if model.startswith("openai-compatible:"):
|
|
return False, "", {}, "model_timeout"
|
|
return True, "reviewed result", {"output_tokens": 7}, "ok"
|
|
|
|
environment = {
|
|
"CASAN_GOAL_OMNIROUTE_MODELS": "route-a,openai-compatible:route-b",
|
|
"CASAN_GOAL_LOCAL_PROVIDER": "ollama",
|
|
"CASAN_GOAL_REVIEWER_MAX_ATTEMPTS": "4",
|
|
"CASAN_GOAL_REVIEWER_DEADLINE_SEC": "30",
|
|
}
|
|
with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, environment, clear=False), \
|
|
patch.object(MODULE, "call_account_model", account), patch.object(MODULE, "call_model", model):
|
|
path = job_file(directory)
|
|
ok, result, metadata, reason, reviewer = MODULE.run_reviewer_chain(
|
|
path, "review prompt", "claude", "openai:gpt-primary", "ollama:worker-model"
|
|
)
|
|
with open(path, encoding="utf-8") as handle:
|
|
job = json.load(handle)
|
|
|
|
self.assertTrue(ok)
|
|
self.assertEqual(result, "reviewed result")
|
|
self.assertEqual(metadata["output_tokens"], 7)
|
|
self.assertEqual(reason, "ok")
|
|
self.assertEqual(reviewer["provider"], "ollama")
|
|
self.assertEqual(calls, [
|
|
("account", "claude"),
|
|
("model", "openai-compatible:route-a", True),
|
|
("model", "openai-compatible:route-b", True),
|
|
("model", "ollama:worker-model", False),
|
|
])
|
|
self.assertEqual([row["status"] for row in job["reviewer_attempts"]], ["failed", "failed", "failed", "pass"])
|
|
self.assertEqual(job["stages"][0]["provider"], "ollama")
|
|
self.assertEqual(job["stages"][0]["model"], "ollama:worker-model")
|
|
|
|
def test_attempt_limit_is_bounded(self):
|
|
calls = []
|
|
|
|
def model(model, prompt, cloud, timeout):
|
|
calls.append(model)
|
|
return False, "", {}, "model_timeout"
|
|
|
|
environment = {
|
|
"CASAN_GOAL_OMNIROUTE_MODELS": "route-a,route-b,route-c",
|
|
"CASAN_GOAL_REVIEWER_MAX_ATTEMPTS": "2",
|
|
"CASAN_GOAL_REVIEWER_DEADLINE_SEC": "30",
|
|
}
|
|
with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, environment, clear=False), patch.object(MODULE, "call_model", model):
|
|
path = job_file(directory)
|
|
ok, _, _, reason, _ = MODULE.run_reviewer_chain(path, "prompt", "", "openai:gpt", "ollama:local")
|
|
with open(path, encoding="utf-8") as handle:
|
|
job = json.load(handle)
|
|
|
|
self.assertFalse(ok)
|
|
self.assertEqual(reason, "model_timeout")
|
|
self.assertEqual(len(calls), 2)
|
|
self.assertEqual(len(job["reviewer_attempts"]), 2)
|
|
|
|
def test_non_retryable_candidate_failure_advances_to_fallbacks(self):
|
|
calls = []
|
|
|
|
def model(model, prompt, cloud, timeout):
|
|
calls.append(model)
|
|
return False, "", {}, "endpoint_not_allowed"
|
|
|
|
environment = {
|
|
"CASAN_GOAL_OMNIROUTE_MODELS": "route-a",
|
|
"CASAN_GOAL_REVIEWER_MAX_ATTEMPTS": "5",
|
|
"CASAN_GOAL_REVIEWER_DEADLINE_SEC": "30",
|
|
}
|
|
with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, environment, clear=False), patch.object(MODULE, "call_model", model):
|
|
path = job_file(directory)
|
|
ok, _, _, reason, _ = MODULE.run_reviewer_chain(path, "prompt", "", "openai:gpt", "ollama:local")
|
|
with open(path, encoding="utf-8") as handle:
|
|
job = json.load(handle)
|
|
|
|
self.assertFalse(ok)
|
|
self.assertEqual(reason, "endpoint_not_allowed")
|
|
self.assertEqual(calls, ["openai:gpt", "openai-compatible:route-a", "ollama:local"])
|
|
self.assertFalse(job["reviewer_attempts"][0]["retryable"])
|
|
|
|
def test_patch_reviewer_excludes_h2_model_and_receives_full_output_budget(self):
|
|
calls = []
|
|
|
|
def model(model, prompt, cloud, timeout, max_output_tokens=None):
|
|
calls.append((model, max_output_tokens))
|
|
return True, "reviewed patch", {}, "ok"
|
|
|
|
environment = {
|
|
"CASAN_GOAL_OMNIROUTE_MODELS": "route-a",
|
|
"CASAN_GOAL_REVIEWER_MAX_ATTEMPTS": "3",
|
|
"CASAN_GOAL_REVIEWER_DEADLINE_SEC": "30",
|
|
}
|
|
with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, environment, clear=False), patch.object(MODULE, "call_model", model):
|
|
path = job_file(directory)
|
|
ok, _, _, _, reviewer = MODULE.run_reviewer_chain(
|
|
path, "prompt", "", "openai:gpt-worker", "ollama:local",
|
|
max_output_tokens=8192, excluded_models={"openai:gpt-worker"},
|
|
)
|
|
|
|
self.assertTrue(ok)
|
|
self.assertEqual(reviewer["model"], "openai-compatible:route-a")
|
|
self.assertEqual(calls, [("openai-compatible:route-a", 8192)])
|
|
|
|
def test_local_reviewer_can_be_disabled_when_human_approval_is_available(self):
|
|
calls = []
|
|
|
|
def model(model, prompt, cloud, timeout):
|
|
calls.append(model)
|
|
return False, "", {}, "gateway_unavailable"
|
|
|
|
environment = {
|
|
"CASAN_GOAL_OMNIROUTE_MODELS": "route-a",
|
|
"CASAN_GOAL_ENABLE_LOCAL_REVIEWER": "0",
|
|
"CASAN_GOAL_REVIEWER_MAX_ATTEMPTS": "3",
|
|
"CASAN_GOAL_REVIEWER_DEADLINE_SEC": "30",
|
|
}
|
|
with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, environment, clear=False), patch.object(MODULE, "call_model", model):
|
|
path = job_file(directory)
|
|
ok, _, _, reason, _ = MODULE.run_reviewer_chain(path, "prompt", "", "", "ollama:ornith:9b")
|
|
|
|
self.assertFalse(ok)
|
|
self.assertEqual(reason, "gateway_unavailable")
|
|
self.assertEqual(calls, ["openai-compatible:route-a"])
|
|
|
|
def test_patch_reviewer_skips_unhealthy_route_before_full_generation(self):
|
|
calls = []
|
|
|
|
def model(model, prompt, cloud, timeout, max_output_tokens=None):
|
|
calls.append(model)
|
|
return True, "reviewed", {}, "ok"
|
|
|
|
environment = {
|
|
"CASAN_GOAL_OMNIROUTE_MODELS": "route-b",
|
|
"CASAN_GOAL_ENABLE_LOCAL_REVIEWER": "0",
|
|
"CASAN_GOAL_REVIEWER_MAX_ATTEMPTS": "3",
|
|
"CASAN_GOAL_REVIEWER_DEADLINE_SEC": "30",
|
|
}
|
|
with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, environment, clear=False), \
|
|
patch.object(MODULE, "model_preflight", side_effect=[(False, "gateway_503"), (True, "ok")]), \
|
|
patch.object(MODULE, "call_model", model):
|
|
path = job_file(directory)
|
|
ok, _, _, _, reviewer = MODULE.run_reviewer_chain(
|
|
path, "prompt", "", "openai-compatible:route-a", "ollama:ornith:9b", max_output_tokens=8192,
|
|
)
|
|
with open(path, encoding="utf-8") as handle:
|
|
job = json.load(handle)
|
|
|
|
self.assertTrue(ok)
|
|
self.assertEqual(reviewer["model"], "openai-compatible:route-b")
|
|
self.assertEqual(calls, ["openai-compatible:route-b"])
|
|
self.assertIn("model_preflight_failed", job["reviewer_attempts"][0]["reason"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|