170 lines
11 KiB
Python
170 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
|
|
ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", ".."))
|
|
|
|
|
|
def load_module(name, relative):
|
|
spec = importlib.util.spec_from_file_location(name, os.path.join(ROOT, relative))
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
ORCHESTRATOR = load_module("goal_orchestrator_patch", "packages/casan-harness/scripts/bash/goal-orchestrator.py")
|
|
EXECUTOR = load_module("goal_patch_executor", "packages/casan-harness/scripts/bash/goal-patch-executor.py")
|
|
|
|
|
|
class Result:
|
|
def __init__(self, code=0, stdout="", stderr=""):
|
|
self.returncode = code
|
|
self.stdout = stdout
|
|
self.stderr = stderr
|
|
|
|
|
|
class GoalPatchWorkflowTests(unittest.TestCase):
|
|
def test_vietnamese_completion_goal_is_write_intent(self):
|
|
self.assertTrue(ORCHESTRATOR.requests_side_effect("Hoàn thành component KeyResultDetail với form update progress đầy đủ"))
|
|
|
|
def test_extract_patch_requires_unified_diff(self):
|
|
with self.assertRaisesRegex(ValueError, "goal_patch_missing"):
|
|
ORCHESTRATOR.extract_patch("implementation plan only")
|
|
value = ORCHESTRATOR.extract_patch("```diff\ndiff --git a/apps/okr/frontend/a.ts b/apps/okr/frontend/a.ts\n--- a/apps/okr/frontend/a.ts\n+++ b/apps/okr/frontend/a.ts\n@@ -1 +1 @@\n-a\n+b\n```")
|
|
self.assertTrue(value.startswith("diff --git"))
|
|
|
|
def test_write_contract_rejects_a_truncated_diff_before_h3(self):
|
|
truncated = "diff --git a/apps/okr/frontend/a.ts b/apps/okr/frontend/a.ts\n--- a/apps/okr/frontend/a.ts\n+++ b/apps/okr/frontend/a.ts\n@@ -1 +1 @@\n-a\n"
|
|
with self.assertRaisesRegex(ValueError, "goal_patch_check_failed"):
|
|
ORCHESTRATOR.validate_write_output(truncated)
|
|
|
|
def test_invalid_write_output_gets_one_bounded_repair_attempt(self):
|
|
repaired = "```diff\ndiff --git a/apps/okr/frontend/a.ts b/apps/okr/frontend/a.ts\n--- a/apps/okr/frontend/a.ts\n+++ b/apps/okr/frontend/a.ts\n@@ -1 +1 @@\n-a\n+b\n```"
|
|
with patch.dict(os.environ, {"CASAN_GOAL_PATCH_REPAIR_ATTEMPTS": "1"}, clear=False), \
|
|
patch.object(ORCHESTRATOR, "call_model", return_value=(True, repaired, {"output_tokens": 9}, "ok")) as call, \
|
|
patch.object(ORCHESTRATOR, "validate_write_output", return_value=repaired):
|
|
ok, value, usage, reason = ORCHESTRATOR.repair_write_output("ollama:test", "original prompt", "implementation plan")
|
|
self.assertTrue(ok)
|
|
self.assertEqual(reason, "ok")
|
|
self.assertEqual(usage["output_tokens"], 9)
|
|
self.assertIn("diff --git", value)
|
|
self.assertEqual(call.call_count, 1)
|
|
|
|
def test_write_output_repair_can_be_disabled(self):
|
|
with patch.dict(os.environ, {"CASAN_GOAL_PATCH_REPAIR_ATTEMPTS": "0"}, clear=False), \
|
|
patch.object(ORCHESTRATOR, "call_model") as call:
|
|
ok, _, _, reason = ORCHESTRATOR.repair_write_output("ollama:test", "original prompt", "implementation plan")
|
|
self.assertFalse(ok)
|
|
self.assertEqual(reason, "goal_patch_missing")
|
|
call.assert_not_called()
|
|
|
|
def test_repair_prefers_direct_cloud_over_gateway_and_local(self):
|
|
repaired = "diff --git a/apps/okr/frontend/a.ts b/apps/okr/frontend/a.ts\n--- a/apps/okr/frontend/a.ts\n+++ b/apps/okr/frontend/a.ts\n@@ -1 +1 @@\n-a\n+b\n"
|
|
with (
|
|
patch.dict(os.environ, {"CASAN_GOAL_PATCH_REPAIR_ATTEMPTS": "1", "CASAN_GOAL_PATCH_REPAIR_MODELS": "openai:gpt-4o-mini,anthropic:claude,openai-compatible:auto/coding,ollama:ornith"}, clear=False),
|
|
patch.object(ORCHESTRATOR, "call_model", return_value=(True, repaired, {}, "ok")) as call,
|
|
patch.object(ORCHESTRATOR, "validate_write_output", return_value=repaired),
|
|
):
|
|
ok, _, usage, reason = ORCHESTRATOR.repair_write_output("ollama:ornith", "original", "plan")
|
|
self.assertTrue(ok)
|
|
self.assertEqual(reason, "ok")
|
|
self.assertEqual(usage["repair_model"], "openai:gpt-4o-mini")
|
|
self.assertEqual(call.call_args.args[0], "openai:gpt-4o-mini")
|
|
self.assertEqual(call.call_args.kwargs["max_output_tokens"], ORCHESTRATOR.patch_output_tokens())
|
|
|
|
def test_repair_tries_next_stronger_candidate_when_first_patch_is_corrupt(self):
|
|
repaired = "diff --git a/apps/okr/frontend/a.ts b/apps/okr/frontend/a.ts\n--- a/apps/okr/frontend/a.ts\n+++ b/apps/okr/frontend/a.ts\n@@ -1 +1 @@\n-a\n+b\n"
|
|
with (
|
|
patch.dict(os.environ, {"CASAN_GOAL_PATCH_REPAIR_ATTEMPTS": "2", "CASAN_GOAL_PATCH_REPAIR_MODELS": "openai:gpt-4.1,openai-compatible:aug/claude-sonnet"}, clear=False),
|
|
patch.object(ORCHESTRATOR, "call_model", side_effect=[(True, "corrupt", {}, "ok"), (True, repaired, {}, "ok")]) as call,
|
|
patch.object(ORCHESTRATOR, "validate_write_output", side_effect=[ValueError("goal_patch_check_failed:corrupt"), repaired]),
|
|
):
|
|
ok, output, usage, reason = ORCHESTRATOR.repair_write_output("ollama:ornith", "original", "invalid")
|
|
self.assertTrue(ok)
|
|
self.assertEqual(output, repaired)
|
|
self.assertEqual(reason, "ok")
|
|
self.assertEqual(usage["repair_model"], "openai-compatible:aug/claude-sonnet")
|
|
self.assertEqual(usage["repair_attempts"], [
|
|
{"attempt": 1, "provider": "openai", "model": "openai:gpt-4.1", "status": "failed", "reason": "goal_patch_check_failed:corrupt"},
|
|
{"attempt": 2, "provider": "omniroute", "model": "openai-compatible:aug/claude-sonnet", "status": "pass", "reason": "ok"},
|
|
])
|
|
self.assertEqual([item.args[0] for item in call.call_args_list], ["openai:gpt-4.1", "openai-compatible:aug/claude-sonnet"])
|
|
|
|
def test_repair_records_each_provider_failure_without_hiding_the_primary(self):
|
|
with (
|
|
patch.dict(os.environ, {"CASAN_GOAL_PATCH_REPAIR_ATTEMPTS": "2", "CASAN_GOAL_PATCH_REPAIR_MODELS": "openai:gpt-5.3-codex,openai-compatible:gateway"}, clear=False),
|
|
patch.object(ORCHESTRATOR, "call_model", side_effect=[
|
|
(False, "", {}, "model_exit_2:MODEL_ROUTER_ERROR provider_response_empty openai-responses content_type=unknown"),
|
|
(False, "", {}, "model_exit_2:MODEL_ROUTER_ERROR provider_response_invalid openai-compatible JSONDecodeError bytes=0 content_type=unknown"),
|
|
]),
|
|
):
|
|
ok, _, usage, reason = ORCHESTRATOR.repair_write_output("ollama:ornith", "original", "plan")
|
|
self.assertFalse(ok)
|
|
self.assertIn("openai-compatible", reason)
|
|
self.assertEqual([entry["provider"] for entry in usage["repair_attempts"]], ["openai", "omniroute"])
|
|
self.assertIn("openai-responses", usage["repair_attempts"][0]["reason"])
|
|
|
|
def test_codex_gets_its_own_corrective_pass_before_gateway_fallback(self):
|
|
repaired = "diff --git a/apps/okr/frontend/a.ts b/apps/okr/frontend/a.ts\n--- a/apps/okr/frontend/a.ts\n+++ b/apps/okr/frontend/a.ts\n@@ -1 +1 @@\n-a\n+b\n"
|
|
with (
|
|
patch.dict(os.environ, {"CASAN_GOAL_PATCH_REPAIR_ATTEMPTS": "3", "CASAN_GOAL_PATCH_REPAIR_MODELS": "openai:gpt-5.3-codex,openai-compatible:gateway,ollama:ornith"}, clear=False),
|
|
patch.object(ORCHESTRATOR, "call_model", side_effect=[(True, "corrupt", {}, "ok"), (True, repaired, {}, "ok")]) as call,
|
|
patch.object(ORCHESTRATOR, "validate_write_output", side_effect=[ValueError("goal_patch_check_failed:corrupt"), repaired]),
|
|
):
|
|
ok, _, usage, reason = ORCHESTRATOR.repair_write_output("ollama:ornith", "original", "invalid")
|
|
self.assertTrue(ok)
|
|
self.assertEqual(reason, "ok")
|
|
self.assertEqual([item.args[0] for item in call.call_args_list], ["openai:gpt-5.3-codex", "openai:gpt-5.3-codex"])
|
|
self.assertEqual(usage["repair_attempts"][-1]["status"], "pass")
|
|
|
|
def test_codex_keeps_all_repair_attempts_when_its_patch_headers_remain_corrupt(self):
|
|
with (
|
|
patch.dict(os.environ, {"CASAN_GOAL_PATCH_REPAIR_ATTEMPTS": "3", "CASAN_GOAL_PATCH_REPAIR_MODELS": "openai:gpt-5.3-codex,ollama:ornith"}, clear=False),
|
|
patch.object(ORCHESTRATOR, "call_model", return_value=(True, "corrupt", {}, "ok")) as call,
|
|
patch.object(ORCHESTRATOR, "validate_write_output", side_effect=ValueError("goal_patch_check_failed:error: corrupt patch at line 125")),
|
|
):
|
|
ok, _, usage, _ = ORCHESTRATOR.repair_write_output("ollama:ornith", "original", "invalid")
|
|
self.assertFalse(ok)
|
|
self.assertEqual([item.args[0] for item in call.call_args_list], ["openai:gpt-5.3-codex"] * 3)
|
|
self.assertEqual([attempt["provider"] for attempt in usage["repair_attempts"]], ["openai"] * 3)
|
|
|
|
def test_patch_outside_workspace_is_denied(self):
|
|
job = {"workspace": {"context_roots": ["apps/okr/frontend"]}}
|
|
content = "diff --git a/package.json b/package.json\n--- a/package.json\n+++ b/package.json\n@@ -1 +1 @@\n-a\n+b\n"
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
path = os.path.join(directory, "job.json")
|
|
with self.assertRaisesRegex(ValueError, "goal_patch_outside_workspace"):
|
|
ORCHESTRATOR.validate_and_store_patch(path, job, content)
|
|
|
|
def test_patch_rename_source_outside_workspace_is_denied(self):
|
|
job = {"workspace": {"context_roots": ["apps/okr/frontend"]}}
|
|
content = "diff --git a/package.json b/apps/okr/frontend/package.json\nsimilarity index 100%\nrename from package.json\nrename to apps/okr/frontend/package.json\n"
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
with self.assertRaisesRegex(ValueError, "goal_patch_outside_workspace"):
|
|
ORCHESTRATOR.validate_and_store_patch(os.path.join(directory, "job.json"), job, content)
|
|
|
|
def test_executor_requires_approved_matching_proposal(self):
|
|
job = {"id": "goal-1", "approval": {"id": "AP-1"}, "patch_artifact": {"sha256": "abc"}}
|
|
pending = {"proposals": [{"id": "AP-1", "status": "pending", "action": "goal.workspace.execute", "payload": {"goal_id": "goal-1", "patch_sha256": "abc"}}]}
|
|
with patch.object(EXECUTOR, "run", return_value=Result(stdout=json.dumps(pending))):
|
|
with self.assertRaisesRegex(PermissionError, "GOAL_APPLY_APPROVAL_REQUIRED"):
|
|
EXECUTOR.verify_approval(job)
|
|
|
|
mismatched = {"proposals": [{"id": "AP-1", "status": "approved", "action": "goal.workspace.execute", "payload": {"goal_id": "goal-1", "patch_sha256": "different"}}]}
|
|
with patch.object(EXECUTOR, "run", return_value=Result(stdout=json.dumps(mismatched))):
|
|
with self.assertRaisesRegex(PermissionError, "GOAL_APPLY_PATCH_HASH_MISMATCH"):
|
|
EXECUTOR.verify_approval(job)
|
|
|
|
def test_frontend_patch_runs_build_and_tests(self):
|
|
commands = EXECUTOR.verification_commands(["apps/okr/frontend/src/pages/KeyResultDetail.tsx"])
|
|
self.assertIn(["npm", "run", "build", "-w", "@ainative-okr/frontend"], commands)
|
|
self.assertIn(["npm", "test", "-w", "@ainative-okr/frontend"], commands)
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|