420 lines
27 KiB
Python
420 lines
27 KiB
Python
#!/usr/bin/env python3
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
|
|
ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", ".."))
|
|
|
|
|
|
def load_module(name, relative):
|
|
spec = importlib.util.spec_from_file_location(name, os.path.join(ROOT, relative))
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
ORCHESTRATOR = load_module("goal_orchestrator_patch", "packages/casan-harness/scripts/bash/goal-orchestrator.py")
|
|
EXECUTOR = load_module("goal_patch_executor", "packages/casan-harness/scripts/bash/goal-patch-executor.py")
|
|
|
|
|
|
class Result:
|
|
def __init__(self, code=0, stdout="", stderr=""):
|
|
self.returncode = code
|
|
self.stdout = stdout
|
|
self.stderr = stderr
|
|
|
|
|
|
class GoalPatchWorkflowTests(unittest.TestCase):
|
|
def setUp(self):
|
|
# Keep unit tests independent from production route-health settings.
|
|
# Tests that exercise preflight mock model_preflight explicitly.
|
|
self.environment = patch.dict(
|
|
os.environ, {"CASAN_GOAL_MODEL_PREFLIGHT": "0"}, clear=False,
|
|
)
|
|
self.environment.start()
|
|
self.addCleanup(self.environment.stop)
|
|
|
|
def test_vietnamese_completion_goal_is_write_intent(self):
|
|
self.assertTrue(ORCHESTRATOR.requests_side_effect("Hoàn thành component KeyResultDetail với form update progress đầy đủ"))
|
|
|
|
def test_phase_labeled_backend_completion_is_write_intent(self):
|
|
objective = """PHASE 1 — Hoàn thiện backend Objective và Key Result cho ứng dụng OKR.
|
|
|
|
Chỉ làm việc trong phạm vi apps/okr/backend.
|
|
Tạo patch nhưng không tự apply. Chờ Independent Reviewer phê duyệt.
|
|
"""
|
|
self.assertTrue(ORCHESTRATOR.requests_side_effect(objective))
|
|
|
|
def test_read_only_backend_audit_is_not_write_intent(self):
|
|
objective = "Rà soát backend và liệt kê các file có rủi ro. Không sửa code, không tạo patch."
|
|
self.assertFalse(ORCHESTRATOR.requests_side_effect(objective))
|
|
|
|
def test_scoped_negative_constraint_does_not_cancel_write_intent(self):
|
|
objective = """Hoàn thiện tính nguyên tử của luồng cập nhật tiến độ Key Result trong ứng dụng OKR.
|
|
|
|
Không thay đổi API contract. Chỉ trả về unified git diff và chờ Independent Reviewer phê duyệt.
|
|
"""
|
|
self.assertTrue(ORCHESTRATOR.requests_side_effect(objective))
|
|
|
|
def test_valid_worker_patch_still_reaches_human_approval_when_h3_is_unavailable(self):
|
|
diff = "diff --git a/apps/okr/backend/a.ts b/apps/okr/backend/a.ts\n--- a/apps/okr/backend/a.ts\n+++ b/apps/okr/backend/a.ts\n@@ -1 +1 @@\n-a\n+b\n"
|
|
manifest = {"file_count": 1, "characters": 10, "truncated": False, "bundle_sha256": "context-sha"}
|
|
artifact = {"path": ".specify/state/goals/default/job.patch", "sha256": "patch-sha", "files": ["apps/okr/backend/a.ts"], "status": "awaiting_approval"}
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
path = os.path.join(directory, "job.json")
|
|
with open(path, "w", encoding="utf-8") as handle:
|
|
json.dump({
|
|
"id": "goal-1", "goal": "Hoàn thiện backend nhưng không thay đổi API contract.",
|
|
"project": "AINative_OKR_CASAN4", "actor": "owner", "local_provider": "ollama",
|
|
"cloud_provider": "omniroute", "workspace": {"context_roots": ["apps/okr/backend"]},
|
|
"stages": [],
|
|
}, handle)
|
|
with (
|
|
patch.dict(os.environ, {"CASAN_GOAL_LOCAL_MODEL": "ollama:worker", "CASAN_GOAL_CLOUD_MODEL": "openai-compatible:reviewer"}, clear=False),
|
|
patch.object(ORCHESTRATOR, "scan", side_effect=lambda text, mode: (True, text)),
|
|
patch.object(ORCHESTRATOR, "build_context", return_value=("snapshot", manifest, "manifest.json")),
|
|
patch.object(ORCHESTRATOR, "call_model", return_value=(True, diff, {}, "ok")),
|
|
patch.object(ORCHESTRATOR, "validate_write_output", return_value=diff),
|
|
patch.object(ORCHESTRATOR, "run_reviewer_chain", return_value=(False, "", {}, "model_output_empty", {"provider": "ollama", "model": "ollama:worker"})),
|
|
patch.object(ORCHESTRATOR, "validate_and_store_patch", return_value=artifact),
|
|
patch.object(ORCHESTRATOR, "submit_side_effect", return_value={"id": "AP-1", "status": "pending", "action": "goal.workspace.execute"}),
|
|
patch.object(ORCHESTRATOR, "emit"), patch.object(ORCHESTRATOR, "metric"),
|
|
patch.object(ORCHESTRATOR, "audit", return_value="audit-sha"),
|
|
):
|
|
exit_code = ORCHESTRATOR.run(path)
|
|
with open(path, encoding="utf-8") as handle:
|
|
job = json.load(handle)
|
|
self.assertEqual(exit_code, 0)
|
|
self.assertEqual(job["status"], "requires_approval")
|
|
self.assertEqual(job["approval"]["id"], "AP-1")
|
|
self.assertIn("Automated H3 review was unavailable", job["result"])
|
|
self.assertNotIn("independently reviewed", job["result"])
|
|
|
|
def test_source_code_delete_method_is_not_promoted_to_security_block(self):
|
|
source = "export class Service { async delete(id: number) { return this.repo.delete({ where: { id } }); } }"
|
|
with tempfile.TemporaryDirectory() as directory, \
|
|
patch.dict(os.environ, {"CASAN_STATE_ROOT": directory, "CASAN_SECURITY_STRICT": "0"}, clear=False):
|
|
allowed, safe_source = ORCHESTRATOR.scan(source, "input")
|
|
self.assertTrue(allowed)
|
|
self.assertIn("repo.delete", safe_source)
|
|
|
|
def test_extract_patch_requires_unified_diff(self):
|
|
with self.assertRaisesRegex(ValueError, "goal_patch_missing"):
|
|
ORCHESTRATOR.extract_patch("implementation plan only")
|
|
value = ORCHESTRATOR.extract_patch("```diff\ndiff --git a/apps/okr/frontend/a.ts b/apps/okr/frontend/a.ts\n--- a/apps/okr/frontend/a.ts\n+++ b/apps/okr/frontend/a.ts\n@@ -1 +1 @@\n-a\n+b\n```")
|
|
self.assertTrue(value.startswith("diff --git"))
|
|
|
|
def test_write_contract_rejects_a_truncated_diff_before_h3(self):
|
|
truncated = "diff --git a/apps/okr/frontend/a.ts b/apps/okr/frontend/a.ts\n--- a/apps/okr/frontend/a.ts\n+++ b/apps/okr/frontend/a.ts\n@@ -1 +1 @@\n-a\n"
|
|
with self.assertRaisesRegex(ValueError, "goal_patch_check_failed"):
|
|
ORCHESTRATOR.validate_write_output(truncated)
|
|
|
|
def test_hunk_counts_and_blank_context_are_repaired_deterministically(self):
|
|
corrupt = (
|
|
"diff --git a/apps/okr/frontend/a.ts b/apps/okr/frontend/a.ts\n"
|
|
"--- a/apps/okr/frontend/a.ts\n+++ b/apps/okr/frontend/a.ts\n"
|
|
"@@ -10,8 +10,9 @@\n context\n-old\n+new\n\n tail\n"
|
|
)
|
|
normalized = ORCHESTRATOR.normalize_unified_diff(corrupt)
|
|
self.assertIn("@@ -10,4 +10,4 @@", normalized)
|
|
self.assertIn("\n \n tail\n", normalized)
|
|
|
|
def test_write_validation_uses_normalized_patch_after_corrupt_error(self):
|
|
corrupt = "diff --git a/a b/a\n--- a/a\n+++ b/a\n@@ -1,9 +1,9 @@\n-old\n+new\n"
|
|
with patch.object(ORCHESTRATOR, "validate_patch_check", side_effect=[ValueError("goal_patch_check_failed:error: corrupt patch at line 7"), None]) as check:
|
|
normalized = ORCHESTRATOR.validate_write_output(corrupt)
|
|
self.assertIn("@@ -1,1 +1,1 @@", normalized)
|
|
self.assertEqual(check.call_count, 2)
|
|
|
|
def test_patch_fragment_error_triggers_deterministic_hunk_recount(self):
|
|
wrong_counts = (
|
|
"diff --git a/a b/a\n--- a/a\n+++ b/a\n"
|
|
"@@ -1,5 +1,5 @@\n-old\n+new\n"
|
|
"@@ -10,7 +10,7 @@\n-tail-old\n+tail-new\n"
|
|
)
|
|
with patch.object(
|
|
ORCHESTRATOR,
|
|
"validate_patch_check",
|
|
side_effect=[ValueError("goal_patch_check_failed:error: patch fragment without header at line 7: @@ -10,7 +10,7 @@"), None],
|
|
) as check:
|
|
normalized = ORCHESTRATOR.validate_write_output(wrong_counts)
|
|
self.assertIn("@@ -1,1 +1,1 @@", normalized)
|
|
self.assertIn("@@ -10,1 +10,1 @@", normalized)
|
|
self.assertEqual(check.call_count, 2)
|
|
|
|
def test_patch_fragment_error_adds_recount_instruction_to_model_repair(self):
|
|
with (
|
|
patch.dict(os.environ, {"CASAN_GOAL_PATCH_REPAIR_ATTEMPTS": "1", "CASAN_GOAL_PATCH_REPAIR_MODELS": "openai:gpt"}, clear=False),
|
|
patch.object(ORCHESTRATOR, "model_preflight", return_value=(True, "ok")),
|
|
patch.object(ORCHESTRATOR, "call_model", return_value=(False, "", {}, "stopped")) as call,
|
|
):
|
|
ORCHESTRATOR.repair_write_output("openai:gpt", "original", "invalid", "goal_patch_check_failed:patch fragment without header")
|
|
self.assertIn("Recompute every `@@ -old,count +new,count @@` header", call.call_args.args[1])
|
|
|
|
def test_truncated_replacement_is_not_reinterpreted_as_deletion(self):
|
|
truncated = "diff --git a/a b/a\n--- a/a\n+++ b/a\n@@ -1,1 +1,1 @@\n-old\n"
|
|
self.assertEqual(ORCHESTRATOR.normalize_unified_diff(truncated), truncated)
|
|
|
|
def test_write_context_keeps_the_most_relevant_target_file_complete(self):
|
|
raw = "export function KeyResultDetail() {\n" + (" return null;\n" * 180) + "}\n"
|
|
candidates = [
|
|
(20, "apps/okr/frontend/src/pages/KeyResultDetail.tsx", raw),
|
|
(5, "docs/technical_architecture.md", "architecture" * 400),
|
|
]
|
|
project = {"domain": "OKR", "domain_root": "apps/okr", "roots": []}
|
|
with tempfile.TemporaryDirectory() as directory, \
|
|
patch.object(ORCHESTRATOR, "registered_project", return_value=project), \
|
|
patch.object(ORCHESTRATOR, "context_candidates", return_value=candidates), \
|
|
patch.object(ORCHESTRATOR, "scan", side_effect=lambda text, mode: (True, text)):
|
|
bundle, manifest, _ = ORCHESTRATOR.build_context(os.path.join(directory, "goal.json"), "okr", "KeyResultDetail", True)
|
|
self.assertIn(raw.strip(), bundle)
|
|
self.assertEqual(manifest["files"][0]["characters"], len(raw.strip()))
|
|
self.assertFalse(manifest["files"][0]["truncated"])
|
|
|
|
def test_explicit_scope_paths_rank_source_ahead_of_general_architecture(self):
|
|
project = ORCHESTRATOR.registered_project("AINative_OKR_CASAN4")
|
|
objective = """PHASE 1 — Hoàn thiện backend Objective và Key Result.
|
|
Nguồn sự thật:
|
|
- docs/technical_architecture.md
|
|
- apps/okr/domain/input/okr-requirement.md
|
|
Chỉ làm việc trong:
|
|
- apps/okr/backend/src/objectives/**
|
|
- apps/okr/backend/src/key-results/**
|
|
- apps/okr/backend/prisma/schema.prisma
|
|
Tạo patch nhưng không tự apply.
|
|
"""
|
|
ranked = ORCHESTRATOR.context_candidates(project, objective)
|
|
top_paths = [row[1] for row in ranked[:10]]
|
|
self.assertTrue(any(path.startswith("apps/okr/backend/src/objectives/") for path in top_paths))
|
|
self.assertTrue(any(path.startswith("apps/okr/backend/src/key-results/") for path in top_paths))
|
|
self.assertIn("apps/okr/backend/prisma/schema.prisma", top_paths)
|
|
self.assertNotIn("docs/technical_architecture.md", top_paths)
|
|
self.assertNotIn("apps/okr/domain/input/okr-requirement.md", top_paths)
|
|
|
|
def test_explicit_only_scope_excludes_unrequested_context_files(self):
|
|
goal = """Hoàn thiện backend.
|
|
Chỉ được đọc và thay đổi:
|
|
apps/okr/backend/src/key-results/key-results.service.ts
|
|
apps/okr/backend/test/services.test.ts
|
|
Không sửa bất kỳ file nào khác.
|
|
"""
|
|
candidates = [
|
|
(2000, "apps/okr/backend/src/key-results/key-results.service.ts", "service"),
|
|
(2000, "apps/okr/backend/test/services.test.ts", "tests"),
|
|
(50, "apps/okr/backend/src/objectives/objectives.service.ts", "unrequested"),
|
|
]
|
|
project = {"domain": "OKR", "domain_root": "apps/okr", "roots": []}
|
|
with tempfile.TemporaryDirectory() as directory, \
|
|
patch.object(ORCHESTRATOR, "registered_project", return_value=project), \
|
|
patch.object(ORCHESTRATOR, "context_candidates", return_value=candidates), \
|
|
patch.object(ORCHESTRATOR, "scan", side_effect=lambda text, mode: (True, text)):
|
|
bundle, manifest, _ = ORCHESTRATOR.build_context(os.path.join(directory, "goal.json"), "okr", goal, True)
|
|
self.assertEqual([row["path"] for row in manifest["files"]], [
|
|
"apps/okr/backend/src/key-results/key-results.service.ts",
|
|
"apps/okr/backend/test/services.test.ts",
|
|
])
|
|
self.assertNotIn("unrequested", bundle)
|
|
|
|
def test_invalid_write_output_gets_one_bounded_repair_attempt(self):
|
|
repaired = "```diff\ndiff --git a/apps/okr/frontend/a.ts b/apps/okr/frontend/a.ts\n--- a/apps/okr/frontend/a.ts\n+++ b/apps/okr/frontend/a.ts\n@@ -1 +1 @@\n-a\n+b\n```"
|
|
with patch.dict(os.environ, {"CASAN_GOAL_PATCH_REPAIR_ATTEMPTS": "1"}, clear=False), \
|
|
patch.object(ORCHESTRATOR, "call_model", return_value=(True, repaired, {"output_tokens": 9}, "ok")) as call, \
|
|
patch.object(ORCHESTRATOR, "validate_write_output", return_value=repaired):
|
|
ok, value, usage, reason = ORCHESTRATOR.repair_write_output("ollama:test", "original prompt", "implementation plan")
|
|
self.assertTrue(ok)
|
|
self.assertEqual(reason, "ok")
|
|
self.assertEqual(usage["output_tokens"], 9)
|
|
self.assertIn("diff --git", value)
|
|
self.assertEqual(call.call_count, 1)
|
|
|
|
def test_write_output_repair_can_be_disabled(self):
|
|
with patch.dict(os.environ, {"CASAN_GOAL_PATCH_REPAIR_ATTEMPTS": "0"}, clear=False), \
|
|
patch.object(ORCHESTRATOR, "call_model") as call:
|
|
ok, _, _, reason = ORCHESTRATOR.repair_write_output("ollama:test", "original prompt", "implementation plan")
|
|
self.assertFalse(ok)
|
|
self.assertEqual(reason, "goal_patch_missing")
|
|
call.assert_not_called()
|
|
|
|
def test_repair_prefers_direct_cloud_over_gateway_and_local(self):
|
|
repaired = "diff --git a/apps/okr/frontend/a.ts b/apps/okr/frontend/a.ts\n--- a/apps/okr/frontend/a.ts\n+++ b/apps/okr/frontend/a.ts\n@@ -1 +1 @@\n-a\n+b\n"
|
|
with (
|
|
patch.dict(os.environ, {"CASAN_GOAL_PATCH_REPAIR_ATTEMPTS": "1", "CASAN_GOAL_PATCH_REPAIR_MODELS": "openai:gpt-4o-mini,anthropic:claude,openai-compatible:auto/coding,ollama:ornith"}, clear=False),
|
|
patch.object(ORCHESTRATOR, "call_model", return_value=(True, repaired, {}, "ok")) as call,
|
|
patch.object(ORCHESTRATOR, "validate_write_output", return_value=repaired),
|
|
):
|
|
ok, _, usage, reason = ORCHESTRATOR.repair_write_output("ollama:ornith", "original", "plan")
|
|
self.assertTrue(ok)
|
|
self.assertEqual(reason, "ok")
|
|
self.assertEqual(usage["repair_model"], "openai:gpt-4o-mini")
|
|
self.assertEqual(call.call_args.args[0], "openai:gpt-4o-mini")
|
|
self.assertEqual(call.call_args.kwargs["max_output_tokens"], ORCHESTRATOR.patch_output_tokens())
|
|
|
|
def test_repair_skips_model_that_fails_generation_preflight(self):
|
|
repaired = "diff --git a/apps/okr/frontend/a.ts b/apps/okr/frontend/a.ts\n--- a/apps/okr/frontend/a.ts\n+++ b/apps/okr/frontend/a.ts\n@@ -1 +1 @@\n-a\n+b\n"
|
|
with (
|
|
patch.dict(os.environ, {"CASAN_GOAL_PATCH_REPAIR_ATTEMPTS": "2", "CASAN_GOAL_PATCH_REPAIR_MODELS": "openai-compatible:auto/coding,ollama:ornith:9b"}, clear=False),
|
|
patch.object(ORCHESTRATOR, "model_preflight", side_effect=[(False, "gateway_503"), (True, "ok")]),
|
|
patch.object(ORCHESTRATOR, "call_model", return_value=(True, repaired, {}, "ok")) as call,
|
|
patch.object(ORCHESTRATOR, "validate_write_output", return_value=repaired),
|
|
):
|
|
ok, _, usage, reason = ORCHESTRATOR.repair_write_output("ollama:ornith:9b", "original", "invalid")
|
|
self.assertTrue(ok)
|
|
self.assertEqual(reason, "ok")
|
|
self.assertEqual(call.call_count, 1)
|
|
self.assertEqual(call.call_args.args[0], "ollama:ornith:9b")
|
|
self.assertIn("model_preflight_failed", usage["repair_attempts"][0]["reason"])
|
|
|
|
def test_logged_in_account_worker_is_first_patch_repair_candidate(self):
|
|
repaired = "diff --git a/apps/okr/frontend/a.ts b/apps/okr/frontend/a.ts\n--- a/apps/okr/frontend/a.ts\n+++ b/apps/okr/frontend/a.ts\n@@ -1 +1 @@\n-a\n+b\n"
|
|
with (
|
|
patch.dict(os.environ, {"CASAN_GOAL_PATCH_REPAIR_ATTEMPTS": "2", "CASAN_GOAL_PATCH_REPAIR_MODELS": "openai-compatible:auto/coding,ollama:ornith:9b"}, clear=False),
|
|
patch.object(ORCHESTRATOR, "model_preflight", return_value=(True, "ok")),
|
|
patch.object(ORCHESTRATOR, "call_account_model", return_value=(True, repaired, {}, "ok")) as account_call,
|
|
patch.object(ORCHESTRATOR, "call_model") as routed_call,
|
|
patch.object(ORCHESTRATOR, "validate_write_output", return_value=repaired),
|
|
):
|
|
ok, _, usage, reason = ORCHESTRATOR.repair_write_output("account:codex", "original", "invalid")
|
|
self.assertTrue(ok)
|
|
self.assertEqual(reason, "ok")
|
|
self.assertEqual(account_call.call_args.args[0], "codex")
|
|
routed_call.assert_not_called()
|
|
self.assertEqual(usage["repair_attempts"][0]["provider"], "codex-account")
|
|
|
|
def test_account_codex_gets_corrective_pass_before_other_routes(self):
|
|
repaired = "diff --git a/apps/okr/frontend/a.ts b/apps/okr/frontend/a.ts\n--- a/apps/okr/frontend/a.ts\n+++ b/apps/okr/frontend/a.ts\n@@ -1 +1 @@\n-a\n+b\n"
|
|
with (
|
|
patch.dict(os.environ, {"CASAN_GOAL_PATCH_REPAIR_ATTEMPTS": "3", "CASAN_GOAL_PATCH_REPAIR_MODELS": "openai-compatible:auto/coding,ollama:ornith:9b"}, clear=False),
|
|
patch.object(ORCHESTRATOR, "model_preflight", return_value=(True, "ok")),
|
|
patch.object(ORCHESTRATOR, "call_account_model", side_effect=[(True, "corrupt", {}, "ok"), (True, repaired, {}, "ok")]) as account_call,
|
|
patch.object(ORCHESTRATOR, "call_model") as routed_call,
|
|
patch.object(ORCHESTRATOR, "validate_write_output", side_effect=[ValueError("goal_patch_check_failed:patch fragment without header"), repaired]),
|
|
):
|
|
ok, _, usage, reason = ORCHESTRATOR.repair_write_output("account:codex", "original", "invalid")
|
|
self.assertTrue(ok)
|
|
self.assertEqual(reason, "ok")
|
|
self.assertEqual(account_call.call_count, 2)
|
|
routed_call.assert_not_called()
|
|
self.assertEqual([row["model"] for row in usage["repair_attempts"]], ["account:codex", "account:codex"])
|
|
|
|
def test_repair_tries_next_stronger_candidate_when_first_patch_is_corrupt(self):
|
|
repaired = "diff --git a/apps/okr/frontend/a.ts b/apps/okr/frontend/a.ts\n--- a/apps/okr/frontend/a.ts\n+++ b/apps/okr/frontend/a.ts\n@@ -1 +1 @@\n-a\n+b\n"
|
|
with (
|
|
patch.dict(os.environ, {"CASAN_GOAL_PATCH_REPAIR_ATTEMPTS": "2", "CASAN_GOAL_PATCH_REPAIR_MODELS": "openai:gpt-4.1,openai-compatible:aug/claude-sonnet"}, clear=False),
|
|
patch.object(ORCHESTRATOR, "call_model", side_effect=[(True, "corrupt", {}, "ok"), (True, repaired, {}, "ok")]) as call,
|
|
patch.object(ORCHESTRATOR, "validate_write_output", side_effect=[ValueError("goal_patch_check_failed:corrupt"), repaired]),
|
|
):
|
|
ok, output, usage, reason = ORCHESTRATOR.repair_write_output("ollama:ornith", "original", "invalid")
|
|
self.assertTrue(ok)
|
|
self.assertEqual(output, repaired)
|
|
self.assertEqual(reason, "ok")
|
|
self.assertEqual(usage["repair_model"], "openai-compatible:aug/claude-sonnet")
|
|
self.assertEqual(usage["repair_attempts"], [
|
|
{"attempt": 1, "provider": "openai", "model": "openai:gpt-4.1", "status": "failed", "reason": "goal_patch_check_failed:corrupt"},
|
|
{"attempt": 2, "provider": "omniroute", "model": "openai-compatible:aug/claude-sonnet", "status": "pass", "reason": "ok"},
|
|
])
|
|
self.assertEqual([item.args[0] for item in call.call_args_list], ["openai:gpt-4.1", "openai-compatible:aug/claude-sonnet"])
|
|
|
|
def test_repair_records_each_provider_failure_without_hiding_the_primary(self):
|
|
with (
|
|
patch.dict(os.environ, {"CASAN_GOAL_PATCH_REPAIR_ATTEMPTS": "2", "CASAN_GOAL_PATCH_REPAIR_MODELS": "openai:gpt-5.3-codex,openai-compatible:gateway"}, clear=False),
|
|
patch.object(ORCHESTRATOR, "call_model", side_effect=[
|
|
(False, "", {}, "model_exit_2:MODEL_ROUTER_ERROR provider_response_empty openai-responses content_type=unknown"),
|
|
(False, "", {}, "model_exit_2:MODEL_ROUTER_ERROR provider_response_invalid openai-compatible JSONDecodeError bytes=0 content_type=unknown"),
|
|
]),
|
|
):
|
|
ok, _, usage, reason = ORCHESTRATOR.repair_write_output("ollama:ornith", "original", "plan")
|
|
self.assertFalse(ok)
|
|
self.assertIn("openai-compatible", reason)
|
|
self.assertEqual([entry["provider"] for entry in usage["repair_attempts"]], ["openai", "omniroute"])
|
|
self.assertIn("openai-responses", usage["repair_attempts"][0]["reason"])
|
|
|
|
def test_codex_gets_its_own_corrective_pass_before_gateway_fallback(self):
|
|
repaired = "diff --git a/apps/okr/frontend/a.ts b/apps/okr/frontend/a.ts\n--- a/apps/okr/frontend/a.ts\n+++ b/apps/okr/frontend/a.ts\n@@ -1 +1 @@\n-a\n+b\n"
|
|
with (
|
|
patch.dict(os.environ, {"CASAN_GOAL_PATCH_REPAIR_ATTEMPTS": "3", "CASAN_GOAL_PATCH_REPAIR_MODELS": "openai:gpt-5.3-codex,openai-compatible:gateway,ollama:ornith"}, clear=False),
|
|
patch.object(ORCHESTRATOR, "call_model", side_effect=[(True, "corrupt", {}, "ok"), (True, repaired, {}, "ok")]) as call,
|
|
patch.object(ORCHESTRATOR, "validate_write_output", side_effect=[ValueError("goal_patch_check_failed:corrupt"), repaired]),
|
|
):
|
|
ok, _, usage, reason = ORCHESTRATOR.repair_write_output("ollama:ornith", "original", "invalid")
|
|
self.assertTrue(ok)
|
|
self.assertEqual(reason, "ok")
|
|
self.assertEqual([item.args[0] for item in call.call_args_list], ["openai:gpt-5.3-codex", "openai:gpt-5.3-codex"])
|
|
self.assertEqual(usage["repair_attempts"][-1]["status"], "pass")
|
|
|
|
def test_codex_keeps_all_repair_attempts_when_its_patch_headers_remain_corrupt(self):
|
|
with (
|
|
patch.dict(os.environ, {"CASAN_GOAL_PATCH_REPAIR_ATTEMPTS": "3", "CASAN_GOAL_PATCH_REPAIR_MODELS": "openai:gpt-5.3-codex,ollama:ornith"}, clear=False),
|
|
patch.object(ORCHESTRATOR, "call_model", return_value=(True, "corrupt", {}, "ok")) as call,
|
|
patch.object(ORCHESTRATOR, "validate_write_output", side_effect=ValueError("goal_patch_check_failed:error: corrupt patch at line 125")),
|
|
):
|
|
ok, _, usage, _ = ORCHESTRATOR.repair_write_output("ollama:ornith", "original", "invalid")
|
|
self.assertFalse(ok)
|
|
self.assertEqual([item.args[0] for item in call.call_args_list], ["openai:gpt-5.3-codex"] * 3)
|
|
self.assertEqual([attempt["provider"] for attempt in usage["repair_attempts"]], ["openai"] * 3)
|
|
|
|
def test_patch_outside_workspace_is_denied(self):
|
|
job = {"workspace": {"context_roots": ["apps/okr/frontend"]}}
|
|
content = "diff --git a/package.json b/package.json\n--- a/package.json\n+++ b/package.json\n@@ -1 +1 @@\n-a\n+b\n"
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
path = os.path.join(directory, "job.json")
|
|
with self.assertRaisesRegex(ValueError, "goal_patch_outside_workspace"):
|
|
ORCHESTRATOR.validate_and_store_patch(path, job, content)
|
|
|
|
def test_patch_outside_explicit_file_scope_is_denied(self):
|
|
job = {
|
|
"goal": "Chỉ được đọc và thay đổi:\napps/okr/backend/allowed.ts\nKhông sửa bất kỳ file nào khác.",
|
|
"workspace": {"context_roots": ["apps/okr/backend"]},
|
|
}
|
|
content = "diff --git a/apps/okr/backend/other.ts b/apps/okr/backend/other.ts\n--- a/apps/okr/backend/other.ts\n+++ b/apps/okr/backend/other.ts\n@@ -1 +1 @@\n-a\n+b\n"
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
with self.assertRaisesRegex(ValueError, "goal_patch_outside_workspace"):
|
|
ORCHESTRATOR.validate_and_store_patch(os.path.join(directory, "job.json"), job, content)
|
|
|
|
def test_patch_rename_source_outside_workspace_is_denied(self):
|
|
job = {"workspace": {"context_roots": ["apps/okr/frontend"]}}
|
|
content = "diff --git a/package.json b/apps/okr/frontend/package.json\nsimilarity index 100%\nrename from package.json\nrename to apps/okr/frontend/package.json\n"
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
with self.assertRaisesRegex(ValueError, "goal_patch_outside_workspace"):
|
|
ORCHESTRATOR.validate_and_store_patch(os.path.join(directory, "job.json"), job, content)
|
|
|
|
def test_executor_requires_approved_matching_proposal(self):
|
|
job = {"id": "goal-1", "approval": {"id": "AP-1"}, "patch_artifact": {"sha256": "abc"}}
|
|
pending = {"proposals": [{"id": "AP-1", "status": "pending", "action": "goal.workspace.execute", "payload": {"goal_id": "goal-1", "patch_sha256": "abc"}}]}
|
|
with patch.object(EXECUTOR, "run", return_value=Result(stdout=json.dumps(pending))):
|
|
with self.assertRaisesRegex(PermissionError, "GOAL_APPLY_APPROVAL_REQUIRED"):
|
|
EXECUTOR.verify_approval(job)
|
|
|
|
mismatched = {"proposals": [{"id": "AP-1", "status": "approved", "action": "goal.workspace.execute", "payload": {"goal_id": "goal-1", "patch_sha256": "different"}}]}
|
|
with patch.object(EXECUTOR, "run", return_value=Result(stdout=json.dumps(mismatched))):
|
|
with self.assertRaisesRegex(PermissionError, "GOAL_APPLY_PATCH_HASH_MISMATCH"):
|
|
EXECUTOR.verify_approval(job)
|
|
|
|
def test_frontend_patch_runs_build_and_tests(self):
|
|
commands = EXECUTOR.verification_commands(["apps/okr/frontend/src/pages/KeyResultDetail.tsx"])
|
|
self.assertFalse(any(command[:2] == ["git", "diff"] for command in commands))
|
|
self.assertIn(["npm", "run", "build", "-w", "@ainative-okr/frontend"], commands)
|
|
self.assertIn(["npm", "test", "-w", "@ainative-okr/frontend"], commands)
|
|
|
|
def test_executor_can_retry_only_a_verified_rollback_failure(self):
|
|
artifact = {"path": "job.patch"}
|
|
self.assertTrue(EXECUTOR.ready_for_apply({"status": "requires_approval", "patch_artifact": artifact}))
|
|
self.assertTrue(EXECUTOR.ready_for_apply({
|
|
"status": "failed",
|
|
"error": "GOAL_APPLY_VERIFICATION_FAILED_ROLLED_BACK",
|
|
"patch_artifact": artifact,
|
|
}))
|
|
self.assertFalse(EXECUTOR.ready_for_apply({"status": "failed", "error": "OTHER", "patch_artifact": artifact}))
|
|
|
|
def test_service_desk_patch_uses_service_desk_manifest_commands(self):
|
|
commands = EXECUTOR.verification_commands(["apps/service-desk/src/ticket.js"])
|
|
self.assertIn(["node", "--check", "apps/service-desk/src/ticket.js"], commands)
|
|
self.assertIn(["node", "--test", "apps/service-desk/test/ticket.test.mjs"], commands)
|
|
self.assertNotIn(["npm", "test", "-w", "@ainative-okr/backend"], commands)
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|