feat: create new project added
This commit is contained in:
thanhnv
@@ -171,3 +171,12 @@
|
||||
{"timestamp":"2026-07-11T06:51:41Z","trace_id":"e22f32cf-615f-416c-8148-7d0c1d438dc1","harness":"H4-security","mode":"input","status":"blocked","action":"block","risk_level":"high","input_hash":"b837d3eb2da4298e443aad246d4a8b13092af5332cd49c58a44c8ac9144db2b7","output_hash":"c56dc3b5c36c8da3652796b65c47b52f36d7a4ac0fe60dcbf5632471575369f7"}
|
||||
{"timestamp":"2026-07-11T06:52:39Z","trace_id":"22155f0f-2095-4c9e-8624-4e476167d6ca","harness":"H4-security","mode":"input","status":"blocked","action":"block","risk_level":"high","input_hash":"b837d3eb2da4298e443aad246d4a8b13092af5332cd49c58a44c8ac9144db2b7","output_hash":"c56dc3b5c36c8da3652796b65c47b52f36d7a4ac0fe60dcbf5632471575369f7"}
|
||||
{"timestamp":"2026-07-11T06:53:49Z","trace_id":"d70ead8a-1f79-473b-aefe-62746cda2ac4","harness":"H4-security","mode":"input","status":"blocked","action":"block","risk_level":"high","input_hash":"578eaad95e6441e9e3221d3117d11ef179a9c041b768a1de1ac9cc8b8f1320d7","output_hash":"578eaad95e6441e9e3221d3117d11ef179a9c041b768a1de1ac9cc8b8f1320d7"}
|
||||
{"timestamp":"2026-07-11T06:56:50Z","trace_id":"trace-1783753010-281","harness":"H4-security","mode":"input","status":"pass","action":"allow","risk_level":"low","input_hash":"d445befd39096a8026a34bd3284860a37e41797b21c9aad95fe18ca084c49f80","output_hash":"d445befd39096a8026a34bd3284860a37e41797b21c9aad95fe18ca084c49f80"}
|
||||
{"timestamp":"2026-07-11T06:56:51Z","trace_id":"trace-1783753011-757","harness":"H4-security","mode":"input","status":"pass","action":"allow","risk_level":"low","input_hash":"1cdfd3f9eea8387b0b05dcd72f3f243282c3f3beebc49bebe74b1b2cfbc56bd5","output_hash":"d235b5973c2eda0dd9493e85a90452d33a3035e2e4f7042cb3559902fa35acf1"}
|
||||
{"timestamp":"2026-07-11T06:57:52Z","trace_id":"trace-1783753072-1525","harness":"H4-security","mode":"output","status":"pass","action":"allow","risk_level":"low","input_hash":"686d70223e0b33754a14bb18db12fe9331d04810578815bd456a2587e27329c6","output_hash":"686d70223e0b33754a14bb18db12fe9331d04810578815bd456a2587e27329c6"}
|
||||
{"timestamp":"2026-07-11T06:59:07Z","trace_id":"trace-1783753147-1896","harness":"H4-security","mode":"input","status":"pass","action":"allow","risk_level":"low","input_hash":"a8040c2049a45cc0a8f1b9b5bba42fdfb613ec0f4a8cc746ea330f5b5a560318","output_hash":"a8040c2049a45cc0a8f1b9b5bba42fdfb613ec0f4a8cc746ea330f5b5a560318"}
|
||||
{"timestamp":"2026-07-11T06:59:07Z","trace_id":"trace-1783753147-2483","harness":"H4-security","mode":"input","status":"pass","action":"allow","risk_level":"low","input_hash":"e1128c8ee75ea2135e9b3f8169c682b85b039a9e04a5be27a92323786b0dab41","output_hash":"34e3e8a172d525f18f5b432e7509011fdce5f721802bfd033b5e1c63038f24b0"}
|
||||
{"timestamp":"2026-07-11T07:00:08Z","trace_id":"trace-1783753208-3246","harness":"H4-security","mode":"output","status":"pass","action":"allow","risk_level":"low","input_hash":"6ec5725464f966f4d58c9e4ad45e982312fa2d3c97a1e4a1586a31101fa25ceb","output_hash":"6ec5725464f966f4d58c9e4ad45e982312fa2d3c97a1e4a1586a31101fa25ceb"}
|
||||
{"timestamp":"2026-07-11T08:21:56Z","trace_id":"trace-1783758116-4061","harness":"H4-security","mode":"input","status":"pass","action":"allow","risk_level":"low","input_hash":"7b77f4a52022a531c0a8598358a588846eda6a28da14ab394208572e076be2db","output_hash":"7b77f4a52022a531c0a8598358a588846eda6a28da14ab394208572e076be2db"}
|
||||
{"timestamp":"2026-07-11T08:21:56Z","trace_id":"trace-1783758116-4542","harness":"H4-security","mode":"input","status":"pass","action":"allow","risk_level":"low","input_hash":"f593d60a891794f31f42af1392f179a2ba7cf98fb280a07aecc0d089efeece25","output_hash":"f593d60a891794f31f42af1392f179a2ba7cf98fb280a07aecc0d089efeece25"}
|
||||
{"timestamp":"2026-07-11T08:23:30Z","trace_id":"trace-1783758210-5443","harness":"H4-security","mode":"output","status":"pass","action":"allow","risk_level":"low","input_hash":"9eb9836790c7004a65222047c0bfe8cc10489fc161356a661a1d74842c7c31e2","output_hash":"9eb9836790c7004a65222047c0bfe8cc10489fc161356a661a1d74842c7c31e2"}
|
||||
|
||||
@@ -35,3 +35,6 @@
|
||||
{"timestamp": "2026-07-11T06:32:57Z", "trace_id": "07fbb2de-5044-4cad-90de-ac87387d74e8", "harness": "H6-agentops", "agent": "goal.orchestrator", "step": "local-worker-cloud-reviewer", "status": "degraded", "exit_code": 0, "latency_ms": 647, "input_tokens": 0, "output_tokens": 0, "total_tokens": 0, "cost_estimate": 0.0, "cost_source": "provider_usage_logs", "input_hash": "2b844ca284348014088be9eea9455a4bb00ce40960799ca974c239bd9a562cca", "output_hash": "a96d7829c4e1453035563d1e36c0e1cc5e8aabcf18ecb79e0934a54c53df579f"}
|
||||
{"timestamp": "2026-07-11T06:34:56Z", "trace_id": "092ded09-3bbf-4469-9438-f4fe5b9e3d61", "harness": "H6-agentops", "agent": "goal.orchestrator", "step": "local-worker-cloud-reviewer", "status": "degraded", "exit_code": 0, "latency_ms": 9940, "input_tokens": 0, "output_tokens": 0, "total_tokens": 0, "cost_estimate": 0.0, "cost_source": "provider_usage_logs", "input_hash": "2b844ca284348014088be9eea9455a4bb00ce40960799ca974c239bd9a562cca", "output_hash": "a96d7829c4e1453035563d1e36c0e1cc5e8aabcf18ecb79e0934a54c53df579f"}
|
||||
{"timestamp": "2026-07-11T06:38:58Z", "trace_id": "8cb7ab8b-c84e-425c-a761-673888ebce72", "harness": "H6-agentops", "agent": "goal.orchestrator", "step": "local-worker-cloud-reviewer", "status": "success", "exit_code": 0, "latency_ms": 124131, "input_tokens": 2435, "output_tokens": 1400, "total_tokens": 3835, "cost_estimate": 0.0, "cost_source": "provider_usage_logs", "input_hash": "d445befd39096a8026a34bd3284860a37e41797b21c9aad95fe18ca084c49f80", "output_hash": "62dd13b6753d71fd098d5929443afe324b0b8e965f43e97bf88d8eddcf797ecd"}
|
||||
{"timestamp": "2026-07-11T06:57:57Z", "trace_id": "6d02ac98-35ca-4629-90d0-2086b0575cdd", "harness": "H6-agentops", "agent": "goal.orchestrator", "step": "local-worker-cloud-reviewer", "status": "degraded", "exit_code": 0, "latency_ms": 66358, "input_tokens": 2476, "output_tokens": 774, "total_tokens": 3250, "cost_estimate": 0.0, "cost_source": "provider_usage_logs", "input_hash": "d445befd39096a8026a34bd3284860a37e41797b21c9aad95fe18ca084c49f80", "output_hash": "686d70223e0b33754a14bb18db12fe9331d04810578815bd456a2587e27329c6"}
|
||||
{"timestamp": "2026-07-11T07:00:15Z", "trace_id": "d5a68140-987a-46e3-985d-3d0b098e6156", "harness": "H6-agentops", "agent": "goal.orchestrator", "step": "local-worker-cloud-reviewer", "status": "degraded", "exit_code": 0, "latency_ms": 67755, "input_tokens": 2241, "output_tokens": 911, "total_tokens": 3152, "cost_estimate": 0.0, "cost_source": "provider_usage_logs", "input_hash": "a8040c2049a45cc0a8f1b9b5bba42fdfb613ec0f4a8cc746ea330f5b5a560318", "output_hash": "6ec5725464f966f4d58c9e4ad45e982312fa2d3c97a1e4a1586a31101fa25ceb"}
|
||||
{"timestamp": "2026-07-11T08:23:36Z", "trace_id": "d3054a9b-967c-4c9b-9c93-cff40abd1116", "harness": "H6-agentops", "agent": "goal.orchestrator", "step": "local-worker-cloud-reviewer", "status": "degraded", "exit_code": 0, "latency_ms": 100369, "input_tokens": 2448, "output_tokens": 1400, "total_tokens": 3848, "cost_estimate": 0.0, "cost_source": "provider_usage_logs", "input_hash": "7b77f4a52022a531c0a8598358a588846eda6a28da14ab394208572e076be2db", "output_hash": "9eb9836790c7004a65222047c0bfe8cc10489fc161356a661a1d74842c7c31e2"}
|
||||
|
||||
@@ -50,3 +50,6 @@
|
||||
{"timestamp": "2026-07-11T06:15:26Z", "harness": "L5-provider-telemetry", "provider": "ollama", "model": "ornith:9b", "run_id": "adhoc", "step": "generate", "role": "generate", "input_tokens": 126, "output_tokens": 213, "total_tokens": 339, "cost_usd": 0.0, "cost_source": "ollama_local_real_tokens", "latency_ms": 12167, "status": "success"}
|
||||
{"timestamp": "2026-07-11T06:17:12Z", "harness": "L5-provider-telemetry", "provider": "ollama", "model": "ornith:9b", "run_id": "adhoc", "step": "generate", "role": "generate", "input_tokens": 499, "output_tokens": 454, "total_tokens": 953, "cost_usd": 0.0, "cost_source": "ollama_local_real_tokens", "latency_ms": 27972, "status": "success"}
|
||||
{"timestamp": "2026-07-11T06:38:28Z", "harness": "L5-provider-telemetry", "provider": "ollama", "model": "ornith:9b", "run_id": "adhoc", "step": "generate", "role": "generate", "input_tokens": 2435, "output_tokens": 1400, "total_tokens": 3835, "cost_usd": 0.0, "cost_source": "ollama_local_real_tokens", "latency_ms": 93084, "status": "success"}
|
||||
{"timestamp": "2026-07-11T06:57:52Z", "harness": "L5-provider-telemetry", "provider": "ollama", "model": "ornith:9b", "run_id": "adhoc", "step": "generate", "role": "generate", "input_tokens": 2476, "output_tokens": 774, "total_tokens": 3250, "cost_usd": 0.0, "cost_source": "ollama_local_real_tokens", "latency_ms": 61199, "status": "success"}
|
||||
{"timestamp": "2026-07-11T07:00:08Z", "harness": "L5-provider-telemetry", "provider": "ollama", "model": "ornith:9b", "run_id": "adhoc", "step": "generate", "role": "generate", "input_tokens": 2241, "output_tokens": 911, "total_tokens": 3152, "cost_usd": 0.0, "cost_source": "ollama_local_real_tokens", "latency_ms": 60429, "status": "success"}
|
||||
{"timestamp": "2026-07-11T08:23:30Z", "harness": "L5-provider-telemetry", "provider": "ollama", "model": "ornith:9b", "run_id": "adhoc", "step": "generate", "role": "generate", "input_tokens": 2448, "output_tokens": 1400, "total_tokens": 3848, "cost_usd": 0.0, "cost_source": "ollama_local_real_tokens", "latency_ms": 94149, "status": "success"}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { Body, Controller, Get, Headers, Inject, Param, Post, Query } from '@nestjs/common';
|
||||
import { ok } from '../common/api-response.js';
|
||||
import { actorFromHeaders } from '../common/auth-context.js';
|
||||
import { GoalsService, type GoalStartInput } from './goals.service.js';
|
||||
import { GoalsService, type GoalProjectCreateInput, type GoalStartInput } from './goals.service.js';
|
||||
|
||||
@Controller('api/v1/goals')
|
||||
export class GoalsController {
|
||||
@@ -17,6 +17,11 @@ export class GoalsController {
|
||||
return ok(this.service.projects(actorFromHeaders(headers)));
|
||||
}
|
||||
|
||||
@Post('projects')
|
||||
createProject(@Body() body: GoalProjectCreateInput, @Headers() headers: Record<string, string | string[] | undefined>) {
|
||||
return ok(this.service.createProject(body, actorFromHeaders(headers)));
|
||||
}
|
||||
|
||||
@Get()
|
||||
list(@Headers() headers: Record<string, string | string[] | undefined>, @Query('limit') limit?: string) {
|
||||
return ok(this.service.list(actorFromHeaders(headers), Number(limit) || 20));
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { BadRequestException, ForbiddenException, HttpException, HttpStatus, Injectable, InternalServerErrorException, NotFoundException } from '@nestjs/common';
|
||||
import { chmodSync, existsSync, mkdirSync, readFileSync, readdirSync, realpathSync, statSync, writeFileSync } from 'node:fs';
|
||||
import { chmodSync, closeSync, existsSync, mkdirSync, openSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync } from 'node:fs';
|
||||
import { execFileSync, spawn } from 'node:child_process';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { join } from 'node:path';
|
||||
@@ -18,6 +18,11 @@ export interface GoalProject {
|
||||
context_roots: string[];
|
||||
}
|
||||
|
||||
export interface GoalProjectCreateInput {
|
||||
projectId: string;
|
||||
domain: string;
|
||||
}
|
||||
|
||||
export interface GoalStage {
|
||||
id: string;
|
||||
status: string;
|
||||
@@ -53,6 +58,19 @@ export interface GoalJob {
|
||||
workspace?: GoalProject;
|
||||
context_manifest?: { files: number; characters: number; truncated: boolean; path?: string };
|
||||
approval?: { id: string; status: string; action: string };
|
||||
reviewer_attempts?: GoalReviewerAttempt[];
|
||||
}
|
||||
|
||||
export interface GoalReviewerAttempt {
|
||||
attempt: number;
|
||||
provider: string;
|
||||
model: string;
|
||||
status: 'pass' | 'failed';
|
||||
reason: string;
|
||||
retryable: boolean;
|
||||
started_at: string;
|
||||
finished_at: string;
|
||||
latency_ms: number;
|
||||
}
|
||||
|
||||
interface ModelConnection {
|
||||
@@ -108,17 +126,32 @@ export class GoalsService {
|
||||
|
||||
const connections = this.connections(actor);
|
||||
const local = connections.find((connection) => connection.connected && connection.kind === 'local');
|
||||
const cloud = connections.find((connection) => connection.connected && connection.kind === 'cloud')
|
||||
?? connections.find((connection) => connection.connected && connection.kind === 'gateway');
|
||||
const cloudConnections = connections.filter((connection) => connection.connected && connection.kind === 'cloud' && (connection.defaultModel || connection.models[0]));
|
||||
const cloud = cloudConnections[0];
|
||||
const gateway = connections.find((connection) => connection.connected && connection.kind === 'gateway');
|
||||
const account = await this.accountReviewer();
|
||||
const localModel = local?.defaultModel || local?.models[0] || 'ornith:9b';
|
||||
const cloudModel = cloud?.defaultModel || cloud?.models[0] || '';
|
||||
const cloudModel = cloud?.defaultModel || cloud?.models[0] || gateway?.defaultModel || gateway?.models[0] || '';
|
||||
const localRuntime = local ? this.runtime(local.id, localModel, actor) : {
|
||||
CASAN_CHAT_SELECTED_MODEL: `ollama:${localModel}`,
|
||||
CASAN_OLLAMA_HOST: process.env.CASAN_OLLAMA_HOST || 'host.docker.internal:11434',
|
||||
OLLAMA_HOST: process.env.OLLAMA_HOST || 'host.docker.internal:11434',
|
||||
};
|
||||
const cloudRuntime = cloud ? this.runtime(cloud.id, cloudModel, actor) : {};
|
||||
const selectedReviewer = cloud ?? gateway;
|
||||
const cloudRuntime = selectedReviewer ? this.runtime(selectedReviewer.id, cloudModel, actor) : {};
|
||||
const gatewayRuntime = gateway
|
||||
? this.runtime(gateway.id, gateway.defaultModel || gateway.models[0] || '', actor)
|
||||
: {};
|
||||
const gatewayCredentials = Object.fromEntries(
|
||||
Object.entries(gatewayRuntime).filter(([key]) => key.startsWith('CASAN_OPENAI_COMPATIBLE_')),
|
||||
);
|
||||
const cloudCandidates = cloudConnections.map((connection) => {
|
||||
const model = connection.defaultModel || connection.models[0];
|
||||
const runtime = this.runtime(connection.id, model, actor);
|
||||
return { model: String(runtime.CASAN_CHAT_SELECTED_MODEL || model), runtime };
|
||||
});
|
||||
const cloudCredentials = Object.assign({}, ...cloudCandidates.map(({ runtime }) => runtime));
|
||||
const gatewayModels = gateway?.models.slice(0, 5).map((model) => `openai-compatible:${model}`) ?? [];
|
||||
const id = randomUUID();
|
||||
const timestamp = new Date().toISOString();
|
||||
const job: GoalJob = {
|
||||
@@ -134,11 +167,11 @@ export class GoalsService {
|
||||
updated_at: timestamp,
|
||||
local_provider: local?.id || 'local-policy',
|
||||
local_model: String(localRuntime.CASAN_CHAT_SELECTED_MODEL || `ollama:${localModel}`),
|
||||
cloud_provider: account ? `${account}-account` : (cloud?.id || 'unavailable'),
|
||||
cloud_provider: account ? `${account}-account` : (selectedReviewer?.id || 'unavailable'),
|
||||
cloud_model: account ? `${account}-account-default` : String(cloudRuntime.CASAN_CHAT_SELECTED_MODEL || ''),
|
||||
stages: [
|
||||
{ id: 'local-worker', status: 'queued', detail: 'Waiting for local worker', provider: local?.id || 'local-policy', model: localModel },
|
||||
{ id: 'cloud-reviewer', status: 'queued', detail: account || cloud ? 'Waiting for cloud reviewer' : 'No cloud connection; local fallback will be explicit', provider: account ? `${account}-account` : (cloud?.id || 'unavailable'), model: account ? `${account}-account-default` : cloudModel },
|
||||
{ id: 'cloud-reviewer', status: 'queued', detail: account || selectedReviewer ? 'Waiting for independent reviewer' : 'Cloud unavailable; local reviewer will be used', provider: account ? `${account}-account` : (selectedReviewer?.id || 'local-policy'), model: account ? `${account}-account-default` : cloudModel || localModel },
|
||||
],
|
||||
};
|
||||
const jobFile = this.jobPath(actor.tenant, id);
|
||||
@@ -151,7 +184,9 @@ export class GoalsService {
|
||||
env: {
|
||||
...process.env,
|
||||
...localRuntime,
|
||||
...cloudCredentials,
|
||||
...cloudRuntime,
|
||||
...gatewayCredentials,
|
||||
CASAN_TENANT_ID: actor.tenant || 'default',
|
||||
CASAN_GOAL_LOCAL_MODEL: job.local_model,
|
||||
CASAN_GOAL_CLOUD_MODEL: job.cloud_model,
|
||||
@@ -159,6 +194,11 @@ export class GoalsService {
|
||||
CASAN_GOAL_CLOUD_PROVIDER: job.cloud_provider,
|
||||
CASAN_GOAL_ACCOUNT_PROVIDER: account || '',
|
||||
CASAN_GOAL_CLOUD_FALLBACK_MODEL: String(cloudRuntime.CASAN_CHAT_SELECTED_MODEL || ''),
|
||||
CASAN_GOAL_CLOUD_MODELS: cloudCandidates.map(({ model }) => model).join(','),
|
||||
CASAN_GOAL_OMNIROUTE_MODELS: gatewayModels.join(','),
|
||||
CASAN_GOAL_LOCAL_REVIEWER_MODEL: String(localRuntime.CASAN_CHAT_SELECTED_MODEL || `ollama:${localModel}`),
|
||||
CASAN_GOAL_REVIEWER_MAX_ATTEMPTS: process.env.CASAN_GOAL_REVIEWER_MAX_ATTEMPTS || '8',
|
||||
CASAN_GOAL_REVIEWER_DEADLINE_SEC: process.env.CASAN_GOAL_REVIEWER_DEADLINE_SEC || '600',
|
||||
},
|
||||
stdio: 'ignore',
|
||||
});
|
||||
@@ -182,6 +222,58 @@ export class GoalsService {
|
||||
return { count: projects.length, projects };
|
||||
}
|
||||
|
||||
createProject(input: GoalProjectCreateInput, actor: SettingsActor): GoalProject {
|
||||
if (actor.role !== 'org-admin') throw new ForbiddenException('GOAL_PROJECT_CREATE_DENIED');
|
||||
const projectId = String(input.projectId ?? '').trim();
|
||||
const domain = String(input.domain ?? '').trim();
|
||||
if (!/^[A-Za-z][A-Za-z0-9._-]{2,63}$/.test(projectId)) {
|
||||
throw new BadRequestException('GOAL_PROJECT_ID_INVALID');
|
||||
}
|
||||
if (domain.length < 3 || domain.length > 100) {
|
||||
throw new BadRequestException('GOAL_PROJECT_DOMAIN_INVALID');
|
||||
}
|
||||
const lockPath = `${PROJECT_REGISTRY}.lock`;
|
||||
let lockFd: number | undefined;
|
||||
for (let attempt = 0; attempt < 100 && lockFd === undefined; attempt += 1) {
|
||||
try { lockFd = openSync(lockPath, 'wx', 0o600); } catch {
|
||||
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 25);
|
||||
}
|
||||
}
|
||||
if (lockFd === undefined) throw new HttpException('GOAL_PROJECT_REGISTRY_BUSY', HttpStatus.CONFLICT);
|
||||
let absoluteRoot = '';
|
||||
let createdRoot = false;
|
||||
try {
|
||||
const registry = parseJson<{ projects?: Array<Record<string, unknown>> }>(readFileSync(PROJECT_REGISTRY, 'utf8'));
|
||||
if (!registry || !Array.isArray(registry.projects)) throw new InternalServerErrorException('GOAL_PROJECT_REGISTRY_INVALID');
|
||||
if (registry.projects.some((entry) => String(entry.project_id) === projectId)) throw new BadRequestException('GOAL_PROJECT_ALREADY_EXISTS');
|
||||
|
||||
const canonicalRoot = realpathSync(APP_ROOT);
|
||||
const appsRoot = realpathSync(join(APP_ROOT, 'apps'));
|
||||
if (!appsRoot.startsWith(`${canonicalRoot}/`)) throw new ForbiddenException('GOAL_PROJECT_PARENT_DENIED');
|
||||
const projectsRoot = join(appsRoot, 'projects');
|
||||
if (!existsSync(projectsRoot)) mkdirSync(projectsRoot, { mode: 0o750 });
|
||||
const canonicalProjects = realpathSync(projectsRoot);
|
||||
if (!canonicalProjects.startsWith(`${canonicalRoot}/`)) throw new ForbiddenException('GOAL_PROJECT_PARENT_DENIED');
|
||||
absoluteRoot = join(canonicalProjects, projectId);
|
||||
mkdirSync(absoluteRoot, { mode: 0o750 });
|
||||
createdRoot = true;
|
||||
const relativeRoot = join('apps', 'projects', projectId);
|
||||
const entry = { project_id: projectId, domain, domain_root: relativeRoot, context_roots: [relativeRoot], harness_package: 'fpt-casan-sdd-harness', harness_version: '1.0.0', status: 'active' };
|
||||
const updated = { ...registry, projects: [...registry.projects, entry] };
|
||||
const temporary = `${PROJECT_REGISTRY}.${process.pid}.${randomUUID()}.tmp`;
|
||||
writeFileSync(temporary, `${JSON.stringify(updated, null, 2)}\n`, { encoding: 'utf8', mode: 0o600, flag: 'wx' });
|
||||
renameSync(temporary, PROJECT_REGISTRY);
|
||||
return { project_id: projectId, domain, domain_root: relativeRoot, context_roots: [relativeRoot] };
|
||||
} catch (error) {
|
||||
if (createdRoot && absoluteRoot) rmSync(absoluteRoot, { recursive: true, force: true });
|
||||
if (error instanceof HttpException) throw error;
|
||||
throw new InternalServerErrorException('GOAL_PROJECT_CREATE_FAILED');
|
||||
} finally {
|
||||
closeSync(lockFd);
|
||||
unlinkSync(lockPath);
|
||||
}
|
||||
}
|
||||
|
||||
get(id: string, actor: SettingsActor): GoalJob {
|
||||
if (!/^[a-f0-9-]{36}$/.test(id)) throw new NotFoundException('GOAL_NOT_FOUND');
|
||||
const path = this.jobPath(actor.tenant, id);
|
||||
|
||||
@@ -4,7 +4,7 @@ import { execFileSync } from 'node:child_process';
|
||||
import { mkdtempSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { BadRequestException } from '@nestjs/common';
|
||||
import { BadRequestException, ForbiddenException } from '@nestjs/common';
|
||||
import { GoalsService } from '../src/goals/goals.service.js';
|
||||
|
||||
const root = join(import.meta.dirname, '..', '..', '..', '..');
|
||||
@@ -24,6 +24,13 @@ test('goal start rejects project ids outside the server registry before model ro
|
||||
);
|
||||
});
|
||||
|
||||
test('goal project creation is restricted to organization administrators', () => {
|
||||
assert.throws(
|
||||
() => new GoalsService().createProject({ projectId: 'denied-project', domain: 'Denied Project' }, { ...admin, role: 'viewer' }),
|
||||
ForbiddenException,
|
||||
);
|
||||
});
|
||||
|
||||
test('H1 creates a bounded manifest and routes workspace side effects to approval without writing source', () => {
|
||||
const stateRoot = mkdtempSync(join(tmpdir(), 'casan-goal-context-'));
|
||||
const tenantRoot = join(stateRoot, 'tenants');
|
||||
|
||||
@@ -293,6 +293,19 @@ export interface GoalJob {
|
||||
workspace?: GoalProject;
|
||||
context_manifest?: { files: number; characters: number; truncated: boolean; path?: string };
|
||||
approval?: { id: string; status: string; action: string };
|
||||
reviewer_attempts?: GoalReviewerAttempt[];
|
||||
}
|
||||
|
||||
export interface GoalReviewerAttempt {
|
||||
attempt: number;
|
||||
provider: string;
|
||||
model: string;
|
||||
status: 'pass' | 'failed';
|
||||
reason: string;
|
||||
retryable: boolean;
|
||||
started_at: string;
|
||||
finished_at: string;
|
||||
latency_ms: number;
|
||||
}
|
||||
|
||||
export interface GoalProject {
|
||||
@@ -481,6 +494,8 @@ export const api = {
|
||||
post<{ success: boolean; reason: string; provider: ProviderAuthStatus }>(`provider-auth/${provider}/login`, {}, actorHeaders(actor)),
|
||||
goalProjects: (actor: SettingsActor) =>
|
||||
getWithHeaders<{ count: number; projects: GoalProject[] }>('goals/projects', actorHeaders(actor)),
|
||||
createGoalProject: (actor: SettingsActor, body: { projectId: string; domain: string }) =>
|
||||
post<GoalProject>('goals/projects', body, actorHeaders(actor)),
|
||||
startGoal: (actor: SettingsActor, goal: string, projectId: string) =>
|
||||
post<GoalJob>('goals', { goal, projectId }, actorHeaders(actor)),
|
||||
goal: (actor: SettingsActor, id: string) =>
|
||||
|
||||
@@ -46,6 +46,9 @@ function WorkerCard({ title, subtitle, status, detail, provider, model }: {
|
||||
export function Goals() {
|
||||
const [goal, setGoal] = useState('');
|
||||
const [projectId, setProjectId] = useState('');
|
||||
const [creatingProject, setCreatingProject] = useState(false);
|
||||
const [newProjectId, setNewProjectId] = useState('');
|
||||
const [newProjectDomain, setNewProjectDomain] = useState('');
|
||||
const [actor] = useState(DEFAULT_ACTOR);
|
||||
const [searchParams, setSearchParams] = useSearchParams();
|
||||
const selectedId = searchParams.get('id') ?? '';
|
||||
@@ -73,6 +76,16 @@ export function Goals() {
|
||||
void queryClient.invalidateQueries({ queryKey: ['goals'] });
|
||||
},
|
||||
});
|
||||
const createProject = useMutation({
|
||||
mutationFn: () => api.createGoalProject(actor, { projectId: newProjectId.trim(), domain: newProjectDomain.trim() }),
|
||||
onSuccess: (project) => {
|
||||
setProjectId(project.project_id);
|
||||
setNewProjectId('');
|
||||
setNewProjectDomain('');
|
||||
setCreatingProject(false);
|
||||
void queryClient.invalidateQueries({ queryKey: ['goal-projects'] });
|
||||
},
|
||||
});
|
||||
|
||||
const selected = selectedQuery.data;
|
||||
const localStage = selected?.stages.find((stage) => stage.id === 'local-worker');
|
||||
@@ -89,15 +102,32 @@ export function Goals() {
|
||||
</section>
|
||||
|
||||
<Card title="Give CASAN an objective">
|
||||
<label className="mb-4 block">
|
||||
<span className="text-sm font-semibold text-slate-800">Project workspace</span>
|
||||
<div className="mb-4">
|
||||
<div className="flex items-center justify-between gap-3">
|
||||
<span className="text-sm font-semibold text-slate-800">Project workspace</span>
|
||||
<button type="button" onClick={() => setCreatingProject((value) => !value)} className="rounded-lg border border-indigo-200 bg-indigo-50 px-3 py-1.5 text-xs font-semibold text-indigo-700 transition hover:border-indigo-300 hover:bg-indigo-100 focus:outline-none focus:ring-2 focus:ring-indigo-400">
|
||||
{creatingProject ? 'Cancel' : '+ New project'}
|
||||
</button>
|
||||
</div>
|
||||
<span className="mt-1 block text-xs leading-5 text-slate-500">Only server-registered, allowlisted roots are available. Models receive a bounded redacted snapshot—not filesystem access.</span>
|
||||
<select value={effectiveProject} onChange={(event) => setProjectId(event.target.value)} disabled={projectsQuery.isLoading || projects.length === 0} className="mt-2 w-full rounded-xl border border-slate-300 bg-white px-4 py-3 text-sm outline-none transition focus:border-indigo-400 focus:ring-4 focus:ring-indigo-100 disabled:bg-slate-100">
|
||||
{projects.map((project) => <option key={project.project_id} value={project.project_id}>{project.domain} · {project.project_id}</option>)}
|
||||
</select>
|
||||
{projectsQuery.isError && <span role="alert" className="mt-2 block text-xs font-medium text-rose-700">Could not load the allowlisted project registry.</span>}
|
||||
{effectiveProject && <span className="mt-2 block font-mono text-[11px] text-slate-400">Context roots: {projects.find((project) => project.project_id === effectiveProject)?.context_roots.join(', ')}</span>}
|
||||
</label>
|
||||
{creatingProject && (
|
||||
<div className="mt-4 rounded-2xl border border-indigo-200 bg-indigo-50/60 p-4">
|
||||
<div className="text-sm font-semibold text-indigo-950">Register a new governed workspace</div>
|
||||
<p className="mt-1 text-xs leading-5 text-indigo-800/75">CASAN creates an empty directory under <span className="font-mono">apps/projects/<project-id></span>. Absolute paths and external roots are never accepted.</p>
|
||||
<div className="mt-4 grid gap-3 sm:grid-cols-2">
|
||||
<label><span className="text-xs font-semibold text-slate-700">Project ID</span><input value={newProjectId} onChange={(event) => setNewProjectId(event.target.value)} placeholder="customer-portal" maxLength={64} className="mt-1 w-full rounded-xl border border-slate-300 bg-white px-3 py-2.5 text-sm outline-none transition focus:border-indigo-400 focus:ring-4 focus:ring-indigo-100" /></label>
|
||||
<label><span className="text-xs font-semibold text-slate-700">Display name</span><input value={newProjectDomain} onChange={(event) => setNewProjectDomain(event.target.value)} placeholder="Customer Portal" maxLength={100} className="mt-1 w-full rounded-xl border border-slate-300 bg-white px-3 py-2.5 text-sm outline-none transition focus:border-indigo-400 focus:ring-4 focus:ring-indigo-100" /></label>
|
||||
</div>
|
||||
<div className="mt-4 flex justify-end"><button type="button" disabled={!/^[A-Za-z][A-Za-z0-9._-]{2,63}$/.test(newProjectId.trim()) || newProjectDomain.trim().length < 3 || createProject.isPending} onClick={() => createProject.mutate()} className="rounded-xl bg-indigo-600 px-4 py-2 text-sm font-semibold text-white transition hover:bg-indigo-700 disabled:cursor-not-allowed disabled:bg-slate-300">{createProject.isPending ? 'Creating…' : 'Create and select'}</button></div>
|
||||
{createProject.isError && <div role="alert" className="mt-3 rounded-xl border border-rose-200 bg-rose-50 p-3 text-xs font-medium text-rose-700">{errorMessage(createProject.error)}</div>}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
<textarea
|
||||
value={goal}
|
||||
onChange={(event) => setGoal(event.target.value)}
|
||||
@@ -143,6 +173,7 @@ export function Goals() {
|
||||
{selected.error && <div className="mt-4 rounded-xl border border-rose-200 bg-rose-50 p-3 text-sm text-rose-700">{selected.error}</div>}
|
||||
{selected.approval && <div className="mt-4 rounded-xl border border-amber-200 bg-amber-50 p-4 text-sm text-amber-900"><div className="font-semibold">Workspace side effect withheld</div><p className="mt-1 leading-6">Proposal <span className="font-mono">{selected.approval.id}</span> is {selected.approval.status}. No source file or runtime action was changed.</p><Link to="/approvals" className="mt-3 inline-flex rounded-lg bg-amber-700 px-3 py-2 text-xs font-semibold text-white hover:bg-amber-800 focus:outline-none focus:ring-2 focus:ring-amber-500">Open Approvals</Link></div>}
|
||||
{selected.local_draft && <details className="mt-5 rounded-xl border border-slate-200 p-4"><summary className="cursor-pointer text-sm font-semibold text-slate-700">Inspect local worker draft</summary><div className="mt-4"><MarkdownText text={selected.local_draft} /></div></details>}
|
||||
{selected.reviewer_attempts && selected.reviewer_attempts.length > 0 && <details className="mt-5 rounded-xl border border-slate-200 p-4"><summary className="cursor-pointer text-sm font-semibold text-slate-700">Fallback attempt ledger ({selected.reviewer_attempts.length})</summary><div className="mt-4 space-y-2">{selected.reviewer_attempts.map((attempt) => <div key={attempt.attempt} className="flex flex-wrap items-center justify-between gap-2 rounded-lg bg-slate-50 px-3 py-2 text-xs"><span className="font-semibold text-slate-700">{attempt.attempt}. reviewer</span><span className="font-mono text-slate-500">{attempt.provider} · {attempt.model}</span><StatusBadge value={attempt.status} /></div>)}</div></details>}
|
||||
</Card>
|
||||
|
||||
<TraceExplorer traceId={selected.trace_id} />
|
||||
|
||||
@@ -303,7 +303,7 @@ def scan(text: str, mode: str):
|
||||
return result.returncode == 0, safe
|
||||
|
||||
|
||||
def call_model(model: str, prompt: str, cloud: bool):
|
||||
def call_model(model: str, prompt: str, cloud: bool, timeout_seconds=None):
|
||||
if not model:
|
||||
return False, "", {}, "model_unconfigured"
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
@@ -318,6 +318,10 @@ def call_model(model: str, prompt: str, cloud: bool):
|
||||
# remains the hard ceiling; this only raises the router's 60s default.
|
||||
environment.setdefault("CASAN_MODEL_TIMEOUT_SEC", os.environ.get("CASAN_GOAL_LOCAL_TIMEOUT_SEC", "240") if not cloud else "120")
|
||||
environment.setdefault("CASAN_MODEL_GENERATE_MAX_TOKENS", os.environ.get("CASAN_GOAL_MAX_OUTPUT_TOKENS", "1400"))
|
||||
timeout = max(1, int(timeout_seconds or os.environ.get("CASAN_GOAL_MODEL_TIMEOUT", "300")))
|
||||
environment["CASAN_MODEL_TIMEOUT_SEC"] = str(min(
|
||||
int(environment.get("CASAN_MODEL_TIMEOUT_SEC", timeout)), timeout
|
||||
))
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["bash", MODEL_ROUTER, prompt_path, output_path, "--role", "generate", "--model", model],
|
||||
@@ -325,7 +329,7 @@ def call_model(model: str, prompt: str, cloud: bool):
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=environment,
|
||||
timeout=int(os.environ.get("CASAN_GOAL_MODEL_TIMEOUT", "300")),
|
||||
timeout=timeout,
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
return False, "", {}, "model_timeout"
|
||||
@@ -349,7 +353,7 @@ def call_model(model: str, prompt: str, cloud: bool):
|
||||
return True, text, metadata, "ok"
|
||||
|
||||
|
||||
def call_account_model(provider: str, prompt: str):
|
||||
def call_account_model(provider: str, prompt: str, timeout_seconds=None):
|
||||
base_url = os.environ.get("CASAN_AUTH_BRIDGE_URL", "").rstrip("/")
|
||||
token = os.environ.get("CASAN_AUTH_BRIDGE_TOKEN", "")
|
||||
if provider not in {"codex", "claude"} or not base_url or not token:
|
||||
@@ -361,7 +365,7 @@ def call_account_model(provider: str, prompt: str):
|
||||
method="POST",
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=310) as response:
|
||||
with urllib.request.urlopen(request, timeout=max(1, int(timeout_seconds or 310))) as response:
|
||||
payload = json.loads(response.read().decode("utf-8"))
|
||||
except (urllib.error.URLError, TimeoutError, ValueError):
|
||||
return False, "", {}, "account_bridge_failed"
|
||||
@@ -378,6 +382,94 @@ def call_account_model(provider: str, prompt: str):
|
||||
return True, text, metadata, "ok"
|
||||
|
||||
|
||||
def reviewer_failure_retryable(reason: str) -> bool:
|
||||
"""Only transient/provider failures may advance to another reviewer."""
|
||||
normalized = str(reason or "").lower()
|
||||
non_retryable = (
|
||||
"endpoint_not_allowed", "model_unconfigured", "model_not_discovered",
|
||||
"invalid_request", "content_policy", "security_blocked",
|
||||
)
|
||||
return not any(marker in normalized for marker in non_retryable)
|
||||
|
||||
|
||||
def reviewer_candidates(account_provider: str, cloud_model: str, local_model: str):
|
||||
"""Build a stable, de-duplicated account/cloud -> OmniRoute -> local chain."""
|
||||
candidates = []
|
||||
if account_provider:
|
||||
candidates.append({
|
||||
"kind": "account", "provider": f"{account_provider}-account",
|
||||
"model": f"{account_provider}-account-default", "value": account_provider,
|
||||
})
|
||||
fallback_model = os.environ.get("CASAN_GOAL_CLOUD_FALLBACK_MODEL", "").strip()
|
||||
if fallback_model:
|
||||
fallback_provider = "omniroute" if fallback_model.startswith("openai-compatible:") else "cloud-fallback"
|
||||
candidates.append({"kind": "model", "provider": fallback_provider, "model": fallback_model, "value": fallback_model})
|
||||
elif cloud_model:
|
||||
candidates.append({"kind": "model", "provider": os.environ.get("CASAN_GOAL_CLOUD_PROVIDER", "cloud"), "model": cloud_model, "value": cloud_model})
|
||||
|
||||
for model in [item.strip() for item in os.environ.get("CASAN_GOAL_CLOUD_MODELS", "").split(",") if item.strip()]:
|
||||
provider = "anthropic" if model.startswith("anthropic:") else "openai" if model.startswith("openai:") else "cloud"
|
||||
candidates.append({"kind": "model", "provider": provider, "model": model, "value": model})
|
||||
|
||||
raw_omniroute = os.environ.get("CASAN_GOAL_OMNIROUTE_MODELS", "")
|
||||
omniroute_models = [item.strip() for item in raw_omniroute.split(",") if item.strip()]
|
||||
for model in omniroute_models:
|
||||
value = model if ":" in model else f"openai-compatible:{model}"
|
||||
candidates.append({"kind": "model", "provider": "omniroute", "model": value, "value": value})
|
||||
|
||||
local_reviewer = os.environ.get("CASAN_GOAL_LOCAL_REVIEWER_MODEL", "").strip() or local_model
|
||||
if local_reviewer:
|
||||
candidates.append({"kind": "model", "provider": os.environ.get("CASAN_GOAL_LOCAL_PROVIDER", "local-policy"), "model": local_reviewer, "value": local_reviewer})
|
||||
|
||||
unique, seen = [], set()
|
||||
for candidate in candidates:
|
||||
key = (candidate["kind"], candidate["value"])
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
unique.append(candidate)
|
||||
return unique
|
||||
|
||||
|
||||
def run_reviewer_chain(job_path: str, prompt: str, account_provider: str, cloud_model: str, local_model: str):
|
||||
max_attempts = max(1, min(int(os.environ.get("CASAN_GOAL_REVIEWER_MAX_ATTEMPTS", "5")), 10))
|
||||
deadline_seconds = max(1, min(int(os.environ.get("CASAN_GOAL_REVIEWER_DEADLINE_SEC", "360")), 900))
|
||||
deadline = time.monotonic() + deadline_seconds
|
||||
ledger = []
|
||||
update_job(job_path, reviewer_attempts=ledger)
|
||||
last_reason = "reviewer_candidates_unavailable"
|
||||
candidates = reviewer_candidates(account_provider, cloud_model, local_model)
|
||||
if len(candidates) > max_attempts and candidates[-1].get("provider") == os.environ.get("CASAN_GOAL_LOCAL_PROVIDER", "local-policy"):
|
||||
candidates = candidates[:max_attempts - 1] + [candidates[-1]] if max_attempts > 1 else [candidates[-1]]
|
||||
else:
|
||||
candidates = candidates[:max_attempts]
|
||||
for candidate in candidates:
|
||||
remaining = int(deadline - time.monotonic())
|
||||
if remaining <= 0:
|
||||
last_reason = "reviewer_deadline_exceeded"
|
||||
break
|
||||
attempt_number = len(ledger) + 1
|
||||
stage(job_path, "cloud-reviewer", "running", f"Reviewer attempt {attempt_number}/{max_attempts}", candidate["provider"], candidate["model"])
|
||||
started_at, started_clock = now(), time.monotonic()
|
||||
if candidate["kind"] == "account":
|
||||
ok, result, metadata, reason = call_account_model(candidate["value"], prompt, remaining)
|
||||
else:
|
||||
ok, result, metadata, reason = call_model(candidate["value"], prompt, candidate["provider"] != os.environ.get("CASAN_GOAL_LOCAL_PROVIDER", "local-policy"), remaining)
|
||||
retryable = False if ok else reviewer_failure_retryable(reason)
|
||||
ledger.append({
|
||||
"attempt": attempt_number, "provider": candidate["provider"], "model": candidate["model"],
|
||||
"status": "pass" if ok else "failed", "reason": reason, "retryable": retryable,
|
||||
"started_at": started_at, "finished_at": now(),
|
||||
"latency_ms": int((time.monotonic() - started_clock) * 1000),
|
||||
})
|
||||
update_job(job_path, reviewer_attempts=ledger)
|
||||
if ok:
|
||||
return True, result, metadata, "ok", candidate
|
||||
last_reason = reason
|
||||
if not retryable and any(marker in str(reason).lower() for marker in ("security_blocked", "content_policy")):
|
||||
break
|
||||
return False, "", {}, last_reason, (ledger[-1] if ledger else {})
|
||||
|
||||
|
||||
def audit(job: dict, status: str) -> str:
|
||||
path = os.path.join(STATE_ROOT, "logs", "audit", "goal-orchestrator.jsonl")
|
||||
head_path = os.path.join(STATE_ROOT, "logs", "audit", "goal-orchestrator-head.txt")
|
||||
@@ -510,7 +602,7 @@ def run(job_path: str) -> int:
|
||||
update_job(job_path, local_draft=safe_local, local_usage=local_meta)
|
||||
emit(goal_id, "H2-tool", "pass", "Local solution prepared", {"provider": job.get("local_provider", ""), "model": local_model, **local_meta})
|
||||
|
||||
stage(job_path, "cloud-reviewer", "running", "Cloud model is challenging and improving the local solution", job.get("cloud_provider", ""), cloud_model)
|
||||
stage(job_path, "cloud-reviewer", "running", "Independent reviewer is challenging and improving the local solution", job.get("cloud_provider", ""), cloud_model)
|
||||
emit(goal_id, "H3-eval", "running", "Cloud reviewer is evaluating the local solution", {"provider": job.get("cloud_provider", ""), "model": cloud_model})
|
||||
review_prompt = (
|
||||
"You are the cloud CASAN reviewer. Critically review the local worker's proposal "
|
||||
@@ -522,26 +614,23 @@ def run(job_path: str) -> int:
|
||||
f"OBJECTIVE:\n{safe_goal}\n\nWORKSPACE SNAPSHOT ({project_id}):\n{context_bundle}\n\n"
|
||||
f"LOCAL WORKER PROPOSAL:\n{safe_local[:5000]}"
|
||||
)
|
||||
if account_provider:
|
||||
cloud_ok, cloud_result, cloud_meta, cloud_reason = call_account_model(account_provider, review_prompt)
|
||||
if not cloud_ok and os.environ.get("CASAN_GOAL_CLOUD_FALLBACK_MODEL"):
|
||||
cloud_ok, cloud_result, cloud_meta, cloud_reason = call_model(
|
||||
os.environ["CASAN_GOAL_CLOUD_FALLBACK_MODEL"], review_prompt, True
|
||||
)
|
||||
else:
|
||||
cloud_ok, cloud_result, cloud_meta, cloud_reason = call_model(cloud_model, review_prompt, True)
|
||||
cloud_ok, cloud_result, cloud_meta, cloud_reason, reviewer = run_reviewer_chain(
|
||||
job_path, review_prompt, account_provider, cloud_model, local_model
|
||||
)
|
||||
reviewer_provider = str(reviewer.get("provider") or job.get("cloud_provider", ""))
|
||||
reviewer_model = str(reviewer.get("model") or cloud_model)
|
||||
if cloud_ok:
|
||||
allowed, safe_result = scan(cloud_result, "output")
|
||||
if not allowed:
|
||||
emit(goal_id, "H4-security", "blocked", "Cloud reviewer output rejected")
|
||||
raise ValueError("cloud_output_security_blocked")
|
||||
stage(job_path, "cloud-reviewer", "pass", "Cloud review incorporated", job.get("cloud_provider", ""), cloud_model)
|
||||
emit(goal_id, "H3-eval", "pass", "Cloud review incorporated", {"provider": job.get("cloud_provider", ""), "model": cloud_model, **cloud_meta})
|
||||
stage(job_path, "cloud-reviewer", "pass", "Independent review incorporated", reviewer_provider, reviewer_model)
|
||||
emit(goal_id, "H3-eval", "pass", "Independent review incorporated", {"provider": reviewer_provider, "model": reviewer_model, **cloud_meta})
|
||||
final_status = "completed"
|
||||
metric_status = "success"
|
||||
else:
|
||||
safe_result = safe_local
|
||||
stage(job_path, "cloud-reviewer", "warning", cloud_reason, job.get("cloud_provider", ""), cloud_model)
|
||||
stage(job_path, "cloud-reviewer", "warning", cloud_reason, reviewer_provider, reviewer_model)
|
||||
emit(goal_id, "H3-eval", "warning", "Cloud reviewer unavailable; local solution retained", {"reason": cloud_reason})
|
||||
final_status = "degraded"
|
||||
metric_status = "degraded"
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
#!/usr/bin/env python3
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
|
||||
ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", ".."))
|
||||
SCRIPT = os.path.join(ROOT, "packages", "casan-harness", "scripts", "bash", "goal-orchestrator.py")
|
||||
SPEC = importlib.util.spec_from_file_location("goal_orchestrator", SCRIPT)
|
||||
MODULE = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(MODULE)
|
||||
|
||||
|
||||
def job_file(directory):
|
||||
path = os.path.join(directory, "job.json")
|
||||
with open(path, "w", encoding="utf-8") as handle:
|
||||
json.dump({
|
||||
"id": "11111111-1111-1111-1111-111111111111",
|
||||
"updated_at": MODULE.now(),
|
||||
"stages": [{"id": "cloud-reviewer", "status": "queued"}],
|
||||
}, handle)
|
||||
return path
|
||||
|
||||
|
||||
class ReviewerFallbackTests(unittest.TestCase):
|
||||
def test_account_then_omniroute_then_local_and_persists_ledger(self):
|
||||
calls = []
|
||||
|
||||
def account(provider, prompt, timeout):
|
||||
calls.append(("account", provider))
|
||||
return False, "", {}, "account_bridge_failed"
|
||||
|
||||
def model(model, prompt, cloud, timeout):
|
||||
calls.append(("model", model, cloud))
|
||||
if model.startswith("openai-compatible:"):
|
||||
return False, "", {}, "model_timeout"
|
||||
return True, "reviewed result", {"output_tokens": 7}, "ok"
|
||||
|
||||
environment = {
|
||||
"CASAN_GOAL_OMNIROUTE_MODELS": "route-a,openai-compatible:route-b",
|
||||
"CASAN_GOAL_LOCAL_PROVIDER": "ollama",
|
||||
"CASAN_GOAL_REVIEWER_MAX_ATTEMPTS": "4",
|
||||
"CASAN_GOAL_REVIEWER_DEADLINE_SEC": "30",
|
||||
}
|
||||
with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, environment, clear=False), \
|
||||
patch.object(MODULE, "call_account_model", account), patch.object(MODULE, "call_model", model):
|
||||
path = job_file(directory)
|
||||
ok, result, metadata, reason, reviewer = MODULE.run_reviewer_chain(
|
||||
path, "review prompt", "claude", "openai:gpt-primary", "ollama:worker-model"
|
||||
)
|
||||
with open(path, encoding="utf-8") as handle:
|
||||
job = json.load(handle)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual(result, "reviewed result")
|
||||
self.assertEqual(metadata["output_tokens"], 7)
|
||||
self.assertEqual(reason, "ok")
|
||||
self.assertEqual(reviewer["provider"], "ollama")
|
||||
self.assertEqual(calls, [
|
||||
("account", "claude"),
|
||||
("model", "openai-compatible:route-a", True),
|
||||
("model", "openai-compatible:route-b", True),
|
||||
("model", "ollama:worker-model", False),
|
||||
])
|
||||
self.assertEqual([row["status"] for row in job["reviewer_attempts"]], ["failed", "failed", "failed", "pass"])
|
||||
self.assertEqual(job["stages"][0]["provider"], "ollama")
|
||||
self.assertEqual(job["stages"][0]["model"], "ollama:worker-model")
|
||||
|
||||
def test_attempt_limit_is_bounded(self):
|
||||
calls = []
|
||||
|
||||
def model(model, prompt, cloud, timeout):
|
||||
calls.append(model)
|
||||
return False, "", {}, "model_timeout"
|
||||
|
||||
environment = {
|
||||
"CASAN_GOAL_OMNIROUTE_MODELS": "route-a,route-b,route-c",
|
||||
"CASAN_GOAL_REVIEWER_MAX_ATTEMPTS": "2",
|
||||
"CASAN_GOAL_REVIEWER_DEADLINE_SEC": "30",
|
||||
}
|
||||
with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, environment, clear=False), patch.object(MODULE, "call_model", model):
|
||||
path = job_file(directory)
|
||||
ok, _, _, reason, _ = MODULE.run_reviewer_chain(path, "prompt", "", "openai:gpt", "ollama:local")
|
||||
with open(path, encoding="utf-8") as handle:
|
||||
job = json.load(handle)
|
||||
|
||||
self.assertFalse(ok)
|
||||
self.assertEqual(reason, "model_timeout")
|
||||
self.assertEqual(len(calls), 2)
|
||||
self.assertEqual(len(job["reviewer_attempts"]), 2)
|
||||
|
||||
def test_non_retryable_candidate_failure_advances_to_fallbacks(self):
|
||||
calls = []
|
||||
|
||||
def model(model, prompt, cloud, timeout):
|
||||
calls.append(model)
|
||||
return False, "", {}, "endpoint_not_allowed"
|
||||
|
||||
environment = {
|
||||
"CASAN_GOAL_OMNIROUTE_MODELS": "route-a",
|
||||
"CASAN_GOAL_REVIEWER_MAX_ATTEMPTS": "5",
|
||||
"CASAN_GOAL_REVIEWER_DEADLINE_SEC": "30",
|
||||
}
|
||||
with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, environment, clear=False), patch.object(MODULE, "call_model", model):
|
||||
path = job_file(directory)
|
||||
ok, _, _, reason, _ = MODULE.run_reviewer_chain(path, "prompt", "", "openai:gpt", "ollama:local")
|
||||
with open(path, encoding="utf-8") as handle:
|
||||
job = json.load(handle)
|
||||
|
||||
self.assertFalse(ok)
|
||||
self.assertEqual(reason, "endpoint_not_allowed")
|
||||
self.assertEqual(calls, ["openai:gpt", "openai-compatible:route-a", "ollama:local"])
|
||||
self.assertFalse(job["reviewer_attempts"][0]["retryable"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -6,3 +6,6 @@
|
||||
{"timestamp": "2026-07-11T06:15:59Z", "provider": "codex", "status": "success", "prompt_hash": "4fc3efd3865a359e2c028d7d3e1a8bcf8bf802c245dd69ed4acf5d5760c6b4d8", "prompt_characters": 606, "latency_ms": 33208}
|
||||
{"timestamp": "2026-07-11T06:17:44Z", "provider": "codex", "status": "success", "prompt_hash": "94192d305fd1113da7f3b81253709e56167ed2f1ed8bb04ceba591df7f97ecfb", "prompt_characters": 2767, "latency_ms": 31710}
|
||||
{"timestamp": "2026-07-11T06:38:58Z", "provider": "codex", "status": "success", "prompt_hash": "7e594a01da1eec5914ff9843541d6a0c4c093ce133add49956ec07073b567671", "prompt_characters": 10535, "latency_ms": 30139}
|
||||
{"timestamp": "2026-07-11T06:57:57Z", "provider": "codex", "status": "failed", "prompt_hash": "b86029ef27b16a12c78a3ce9ca123dee72553f01fcc4b0db22d56909dcc5b96e", "prompt_characters": 9837, "latency_ms": 4215}
|
||||
{"timestamp": "2026-07-11T07:00:15Z", "provider": "codex", "status": "failed", "prompt_hash": "c4942417c0d930a4668053ce2845622d04ed2a65ca985533965b5132402ae507", "prompt_characters": 10758, "latency_ms": 6432}
|
||||
{"timestamp": "2026-07-11T08:23:36Z", "provider": "codex", "status": "failed", "prompt_hash": "5ee797f2318bf12c9d76792507e7c7e9aab26dc79a0e254a7d5e5d6b06c17b76", "prompt_characters": 12413, "latency_ms": 5311}
|
||||
|
||||
Reference in New Issue
Block a user