fix: strengthen goal patch repair routing
This commit is contained in:
@@ -32,7 +32,7 @@ services:
|
|||||||
# gateway; a gateway key must never be assumed valid for api.openai.com.
|
# gateway; a gateway key must never be assumed valid for api.openai.com.
|
||||||
OPENAI_API_KEY: ${OPENAI_API_KEY:-}
|
OPENAI_API_KEY: ${OPENAI_API_KEY:-}
|
||||||
ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-}
|
ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-}
|
||||||
CASAN_GOAL_OPENAI_MODEL: ${CASAN_GOAL_OPENAI_MODEL:-gpt-4o-mini}
|
CASAN_GOAL_OPENAI_MODEL: ${CASAN_GOAL_OPENAI_MODEL:-gpt-4.1}
|
||||||
CASAN_GOAL_ANTHROPIC_MODEL: ${CASAN_GOAL_ANTHROPIC_MODEL:-claude-3-5-sonnet-latest}
|
CASAN_GOAL_ANTHROPIC_MODEL: ${CASAN_GOAL_ANTHROPIC_MODEL:-claude-3-5-sonnet-latest}
|
||||||
CASAN_PREFLIGHT: ${CASAN_PREFLIGHT:-0}
|
CASAN_PREFLIGHT: ${CASAN_PREFLIGHT:-0}
|
||||||
CASAN_AUTH_BRIDGE_URL: http://host.docker.internal:20130
|
CASAN_AUTH_BRIDGE_URL: http://host.docker.internal:20130
|
||||||
|
|||||||
@@ -10,5 +10,5 @@ CASAN_PREFLIGHT=1
|
|||||||
# in governed goal review and H2 patch-format recovery when configured.
|
# in governed goal review and H2 patch-format recovery when configured.
|
||||||
OPENAI_API_KEY=
|
OPENAI_API_KEY=
|
||||||
ANTHROPIC_API_KEY=
|
ANTHROPIC_API_KEY=
|
||||||
CASAN_GOAL_OPENAI_MODEL=gpt-4o-mini
|
CASAN_GOAL_OPENAI_MODEL=gpt-4.1
|
||||||
CASAN_GOAL_ANTHROPIC_MODEL=claude-3-5-sonnet-latest
|
CASAN_GOAL_ANTHROPIC_MODEL=claude-3-5-sonnet-latest
|
||||||
|
|||||||
@@ -158,7 +158,7 @@ export class GoalsService {
|
|||||||
return { model: String(runtime.CASAN_CHAT_SELECTED_MODEL || model), runtime };
|
return { model: String(runtime.CASAN_CHAT_SELECTED_MODEL || model), runtime };
|
||||||
});
|
});
|
||||||
const envCloudCandidates = [
|
const envCloudCandidates = [
|
||||||
process.env.OPENAI_API_KEY ? { model: `openai:${process.env.CASAN_GOAL_OPENAI_MODEL || 'gpt-4o-mini'}`, runtime: {} } : null,
|
process.env.OPENAI_API_KEY ? { model: `openai:${process.env.CASAN_GOAL_OPENAI_MODEL || 'gpt-4.1'}`, runtime: {} } : null,
|
||||||
process.env.ANTHROPIC_API_KEY ? { model: `anthropic:${process.env.CASAN_GOAL_ANTHROPIC_MODEL || 'claude-3-5-sonnet-latest'}`, runtime: {} } : null,
|
process.env.ANTHROPIC_API_KEY ? { model: `anthropic:${process.env.CASAN_GOAL_ANTHROPIC_MODEL || 'claude-3-5-sonnet-latest'}`, runtime: {} } : null,
|
||||||
].filter((candidate): candidate is { model: string; runtime: Record<string, string> } => candidate !== null);
|
].filter((candidate): candidate is { model: string; runtime: Record<string, string> } => candidate !== null);
|
||||||
const cloudCandidates = [...envCloudCandidates, ...savedCloudCandidates].filter((candidate, index, rows) => rows.findIndex((row) => row.model === candidate.model) === index);
|
const cloudCandidates = [...envCloudCandidates, ...savedCloudCandidates].filter((candidate, index, rows) => rows.findIndex((row) => row.model === candidate.model) === index);
|
||||||
|
|||||||
@@ -299,10 +299,10 @@ def validate_write_output(text: str) -> str:
|
|||||||
def patch_repair_attempts() -> int:
|
def patch_repair_attempts() -> int:
|
||||||
"""Return the bounded number of chances to repair an invalid model diff."""
|
"""Return the bounded number of chances to repair an invalid model diff."""
|
||||||
try:
|
try:
|
||||||
configured = int(os.environ.get("CASAN_GOAL_PATCH_REPAIR_ATTEMPTS", "1"))
|
configured = int(os.environ.get("CASAN_GOAL_PATCH_REPAIR_ATTEMPTS", "2"))
|
||||||
except ValueError:
|
except ValueError:
|
||||||
configured = 1
|
configured = 2
|
||||||
return min(max(configured, 0), 1)
|
return min(max(configured, 0), 3)
|
||||||
|
|
||||||
|
|
||||||
def patch_output_tokens() -> int:
|
def patch_output_tokens() -> int:
|
||||||
@@ -354,14 +354,23 @@ def repair_write_output(model: str, original_prompt: str, invalid_output: str, c
|
|||||||
"Every hunk must be complete and the patch must pass `git apply --check`. Do not emit `index` lines, placeholder hashes, commentary, plans, summaries, or side effects. Preserve the original objective and workspace restrictions.\n\n"
|
"Every hunk must be complete and the patch must pass `git apply --check`. Do not emit `index` lines, placeholder hashes, commentary, plans, summaries, or side effects. Preserve the original objective and workspace restrictions.\n\n"
|
||||||
f"ORIGINAL CONTRACT:\n{original_prompt}\n\nPREVIOUS INVALID RESPONSE:\n{invalid_output[:8000]}"
|
f"ORIGINAL CONTRACT:\n{original_prompt}\n\nPREVIOUS INVALID RESPONSE:\n{invalid_output[:8000]}"
|
||||||
)
|
)
|
||||||
candidates = patch_repair_models(model)
|
candidates = patch_repair_models(model)[:patch_repair_attempts()]
|
||||||
candidate = candidates[0] if candidates else model
|
last_metadata, last_reason = {}, "goal_patch_missing"
|
||||||
ok, output, metadata, reason = call_model(candidate, repair_prompt, candidate.startswith(("openai:", "anthropic:", "openai-compatible:")), max_output_tokens=patch_output_tokens())
|
for candidate in candidates:
|
||||||
metadata = dict(metadata)
|
ok, output, metadata, reason = call_model(candidate, repair_prompt, candidate.startswith(("openai:", "anthropic:", "openai-compatible:")), max_output_tokens=patch_output_tokens())
|
||||||
metadata["repair_model"] = candidate
|
metadata = dict(metadata)
|
||||||
if not ok:
|
metadata["repair_model"] = candidate
|
||||||
return False, "", metadata, f"goal_patch_repair_failed:{reason}"
|
last_metadata = metadata
|
||||||
return True, output, metadata, "ok"
|
if not ok:
|
||||||
|
last_reason = f"goal_patch_repair_failed:{reason}"
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
validate_write_output(output)
|
||||||
|
except ValueError as error:
|
||||||
|
last_reason = str(error)
|
||||||
|
continue
|
||||||
|
return True, output, metadata, "ok"
|
||||||
|
return False, "", last_metadata, f"goal_patch_repair_invalid:{last_reason}"
|
||||||
|
|
||||||
|
|
||||||
def merge_usage(primary: dict, additional: dict) -> dict:
|
def merge_usage(primary: dict, additional: dict) -> dict:
|
||||||
@@ -755,10 +764,11 @@ def run(job_path: str) -> int:
|
|||||||
try:
|
try:
|
||||||
validate_write_output(safe_local)
|
validate_write_output(safe_local)
|
||||||
except ValueError as first_error:
|
except ValueError as first_error:
|
||||||
repair_model = patch_repair_models(local_model)[0] if patch_repair_models(local_model) else local_model
|
repair_candidates = patch_repair_models(local_model)[:patch_repair_attempts()]
|
||||||
|
repair_model = repair_candidates[0] if repair_candidates else local_model
|
||||||
repair_provider = provider_for_model(repair_model)
|
repair_provider = provider_for_model(repair_model)
|
||||||
stage(job_path, "local-worker", "running", "Repairing invalid patch output contract", repair_provider, repair_model)
|
stage(job_path, "local-worker", "running", "Repairing invalid patch output contract", repair_provider, repair_model)
|
||||||
emit(goal_id, "H2-tool", "running", "Worker is repairing patch output contract", {"reason": str(first_error), "max_attempts": patch_repair_attempts(), "repair_provider": repair_provider, "repair_model": repair_model})
|
emit(goal_id, "H2-tool", "running", "Worker is repairing patch output contract", {"reason": str(first_error), "max_attempts": patch_repair_attempts(), "repair_provider": repair_provider, "repair_model": repair_model, "repair_candidates": repair_candidates})
|
||||||
repaired, repaired_output, repaired_meta, repair_reason = repair_write_output(local_model, local_prompt, safe_local, str(first_error))
|
repaired, repaired_output, repaired_meta, repair_reason = repair_write_output(local_model, local_prompt, safe_local, str(first_error))
|
||||||
local_meta = merge_usage(local_meta, repaired_meta)
|
local_meta = merge_usage(local_meta, repaired_meta)
|
||||||
if not repaired:
|
if not repaired:
|
||||||
|
|||||||
@@ -46,7 +46,8 @@ class GoalPatchWorkflowTests(unittest.TestCase):
|
|||||||
def test_invalid_write_output_gets_one_bounded_repair_attempt(self):
|
def test_invalid_write_output_gets_one_bounded_repair_attempt(self):
|
||||||
repaired = "```diff\ndiff --git a/apps/okr/frontend/a.ts b/apps/okr/frontend/a.ts\n--- a/apps/okr/frontend/a.ts\n+++ b/apps/okr/frontend/a.ts\n@@ -1 +1 @@\n-a\n+b\n```"
|
repaired = "```diff\ndiff --git a/apps/okr/frontend/a.ts b/apps/okr/frontend/a.ts\n--- a/apps/okr/frontend/a.ts\n+++ b/apps/okr/frontend/a.ts\n@@ -1 +1 @@\n-a\n+b\n```"
|
||||||
with patch.dict(os.environ, {"CASAN_GOAL_PATCH_REPAIR_ATTEMPTS": "1"}, clear=False), \
|
with patch.dict(os.environ, {"CASAN_GOAL_PATCH_REPAIR_ATTEMPTS": "1"}, clear=False), \
|
||||||
patch.object(ORCHESTRATOR, "call_model", return_value=(True, repaired, {"output_tokens": 9}, "ok")) as call:
|
patch.object(ORCHESTRATOR, "call_model", return_value=(True, repaired, {"output_tokens": 9}, "ok")) as call, \
|
||||||
|
patch.object(ORCHESTRATOR, "validate_write_output", return_value=repaired):
|
||||||
ok, value, usage, reason = ORCHESTRATOR.repair_write_output("ollama:test", "original prompt", "implementation plan")
|
ok, value, usage, reason = ORCHESTRATOR.repair_write_output("ollama:test", "original prompt", "implementation plan")
|
||||||
self.assertTrue(ok)
|
self.assertTrue(ok)
|
||||||
self.assertEqual(reason, "ok")
|
self.assertEqual(reason, "ok")
|
||||||
@@ -67,6 +68,7 @@ class GoalPatchWorkflowTests(unittest.TestCase):
|
|||||||
with (
|
with (
|
||||||
patch.dict(os.environ, {"CASAN_GOAL_PATCH_REPAIR_ATTEMPTS": "1", "CASAN_GOAL_PATCH_REPAIR_MODELS": "openai:gpt-4o-mini,anthropic:claude,openai-compatible:auto/coding,ollama:ornith"}, clear=False),
|
patch.dict(os.environ, {"CASAN_GOAL_PATCH_REPAIR_ATTEMPTS": "1", "CASAN_GOAL_PATCH_REPAIR_MODELS": "openai:gpt-4o-mini,anthropic:claude,openai-compatible:auto/coding,ollama:ornith"}, clear=False),
|
||||||
patch.object(ORCHESTRATOR, "call_model", return_value=(True, repaired, {}, "ok")) as call,
|
patch.object(ORCHESTRATOR, "call_model", return_value=(True, repaired, {}, "ok")) as call,
|
||||||
|
patch.object(ORCHESTRATOR, "validate_write_output", return_value=repaired),
|
||||||
):
|
):
|
||||||
ok, _, usage, reason = ORCHESTRATOR.repair_write_output("ollama:ornith", "original", "plan")
|
ok, _, usage, reason = ORCHESTRATOR.repair_write_output("ollama:ornith", "original", "plan")
|
||||||
self.assertTrue(ok)
|
self.assertTrue(ok)
|
||||||
@@ -75,6 +77,20 @@ class GoalPatchWorkflowTests(unittest.TestCase):
|
|||||||
self.assertEqual(call.call_args.args[0], "openai:gpt-4o-mini")
|
self.assertEqual(call.call_args.args[0], "openai:gpt-4o-mini")
|
||||||
self.assertEqual(call.call_args.kwargs["max_output_tokens"], ORCHESTRATOR.patch_output_tokens())
|
self.assertEqual(call.call_args.kwargs["max_output_tokens"], ORCHESTRATOR.patch_output_tokens())
|
||||||
|
|
||||||
|
def test_repair_tries_next_stronger_candidate_when_first_patch_is_corrupt(self):
|
||||||
|
repaired = "diff --git a/apps/okr/frontend/a.ts b/apps/okr/frontend/a.ts\n--- a/apps/okr/frontend/a.ts\n+++ b/apps/okr/frontend/a.ts\n@@ -1 +1 @@\n-a\n+b\n"
|
||||||
|
with (
|
||||||
|
patch.dict(os.environ, {"CASAN_GOAL_PATCH_REPAIR_ATTEMPTS": "2", "CASAN_GOAL_PATCH_REPAIR_MODELS": "openai:gpt-4.1,openai-compatible:aug/claude-sonnet"}, clear=False),
|
||||||
|
patch.object(ORCHESTRATOR, "call_model", side_effect=[(True, "corrupt", {}, "ok"), (True, repaired, {}, "ok")]) as call,
|
||||||
|
patch.object(ORCHESTRATOR, "validate_write_output", side_effect=[ValueError("goal_patch_check_failed:corrupt"), repaired]),
|
||||||
|
):
|
||||||
|
ok, output, usage, reason = ORCHESTRATOR.repair_write_output("ollama:ornith", "original", "invalid")
|
||||||
|
self.assertTrue(ok)
|
||||||
|
self.assertEqual(output, repaired)
|
||||||
|
self.assertEqual(reason, "ok")
|
||||||
|
self.assertEqual(usage["repair_model"], "openai-compatible:aug/claude-sonnet")
|
||||||
|
self.assertEqual([item.args[0] for item in call.call_args_list], ["openai:gpt-4.1", "openai-compatible:aug/claude-sonnet"])
|
||||||
|
|
||||||
def test_patch_outside_workspace_is_denied(self):
|
def test_patch_outside_workspace_is_denied(self):
|
||||||
job = {"workspace": {"context_roots": ["apps/okr/frontend"]}}
|
job = {"workspace": {"context_roots": ["apps/okr/frontend"]}}
|
||||||
content = "diff --git a/package.json b/package.json\n--- a/package.json\n+++ b/package.json\n@@ -1 +1 @@\n-a\n+b\n"
|
content = "diff --git a/package.json b/package.json\n--- a/package.json\n+++ b/package.json\n@@ -1 +1 @@\n-a\n+b\n"
|
||||||
|
|||||||
Reference in New Issue
Block a user