feat: complete llm sourcegen plan
This commit is contained in:
thanhnv
+269
-16
@@ -1,5 +1,6 @@
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync, unlinkSync } from 'node:fs';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync, unlinkSync, appendFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
@@ -69,7 +70,8 @@ function judgeArtifact(filePath, criteria) {
|
||||
}
|
||||
|
||||
function renderSourcePrompt(stepId, title, templateContent, extraInstructions = '') {
|
||||
return `You are generating a CASAN SDLC artifact.
|
||||
const promptTemplatePath = join(__appRoot, 'packages', 'casan-harness', 'prompts', 'sourcegen', `${stepId}.md`);
|
||||
const defaultTemplate = `You are generating a CASAN SDLC artifact.
|
||||
|
||||
Rules:
|
||||
- Output markdown only.
|
||||
@@ -93,6 +95,75 @@ ${architecture.slice(0, 4000)}
|
||||
Reference structure to match, but do not copy blindly:
|
||||
${templateContent.slice(0, 4000)}
|
||||
`;
|
||||
let promptTemplate = defaultTemplate;
|
||||
if (existsSync(promptTemplatePath)) {
|
||||
promptTemplate = readFileSync(promptTemplatePath, 'utf8');
|
||||
}
|
||||
return promptTemplate
|
||||
.replaceAll('{{featureId}}', featureId)
|
||||
.replaceAll('{{moduleId}}', moduleId)
|
||||
.replaceAll('{{stepId}}', stepId)
|
||||
.replaceAll('{{title}}', title)
|
||||
.replaceAll('{{extraInstructions}}', extraInstructions)
|
||||
.replaceAll('{{requirement}}', requirement.slice(0, 6000))
|
||||
.replaceAll('{{architecture}}', architecture.slice(0, 4000))
|
||||
.replaceAll('{{templateContent}}', templateContent.slice(0, 4000));
|
||||
}
|
||||
|
||||
function sha256(text) {
|
||||
return createHash('sha256').update(text).digest('hex');
|
||||
}
|
||||
|
||||
function redactForAudit(text) {
|
||||
return String(text)
|
||||
.replace(/(sk-[A-Za-z0-9_-]{12,})/g, '***REDACTED_KEY***')
|
||||
.replace(/(Bearer\s+)[A-Za-z0-9._-]+/gi, '$1***REDACTED_TOKEN***')
|
||||
.slice(0, 2000);
|
||||
}
|
||||
|
||||
function sourcegenAuditPath() {
|
||||
return process.env.CASAN_SOURCEGEN_AUDIT_LOG || '.specify/logs/audit/sourcegen.jsonl';
|
||||
}
|
||||
|
||||
function sourcegenTelemetryPath() {
|
||||
return process.env.CASAN_SOURCEGEN_TELEMETRY_LOG || '.specify/logs/level5/sourcegen-provider-usage.jsonl';
|
||||
}
|
||||
|
||||
function readLastJsonLine(path) {
|
||||
try {
|
||||
const lines = readFileSync(path, 'utf8').trim().split('\n').filter(Boolean);
|
||||
return lines.length ? JSON.parse(lines[lines.length - 1]) : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function appendSourcegenAudit(record) {
|
||||
const path = sourcegenAuditPath();
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
const prev = readLastJsonLine(path);
|
||||
const base = {
|
||||
timestamp: new Date().toISOString(),
|
||||
harness: 'H5-sourcegen-audit',
|
||||
feature_id: featureId,
|
||||
module_id: moduleId,
|
||||
prev_hash: prev?.record_hash || 'GENESIS',
|
||||
...record,
|
||||
};
|
||||
const recordHash = sha256(JSON.stringify(base));
|
||||
appendFileSync(path, `${JSON.stringify({ ...base, record_hash: recordHash })}\n`, 'utf8');
|
||||
}
|
||||
|
||||
function appendSourcegenTelemetry(record) {
|
||||
const path = sourcegenTelemetryPath();
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
appendFileSync(path, `${JSON.stringify({
|
||||
timestamp: new Date().toISOString(),
|
||||
harness: 'H6-sourcegen-telemetry',
|
||||
feature_id: featureId,
|
||||
module_id: moduleId,
|
||||
...record,
|
||||
})}\n`, 'utf8');
|
||||
}
|
||||
|
||||
function generateArtifact({ stepId, title, templateContent, required = [], extraInstructions = '' }) {
|
||||
@@ -105,24 +176,68 @@ function generateArtifact({ stepId, title, templateContent, required = [], extra
|
||||
const tmpOut = join(tmpdir(), `casan-generate-out-${uid}.json`);
|
||||
const tmpDraft = join(tmpdir(), `casan-generate-draft-${uid}.md`);
|
||||
try {
|
||||
writeFileSync(tmpPrompt, renderSourcePrompt(stepId, title, templateContent, extraInstructions), 'utf8');
|
||||
const prompt = renderSourcePrompt(stepId, title, templateContent, extraInstructions);
|
||||
const promptHash = sha256(prompt);
|
||||
writeFileSync(tmpPrompt, prompt, 'utf8');
|
||||
logDebug(`model call role=generate step=${stepId}`);
|
||||
const r = spawnSync(
|
||||
'bash',
|
||||
[join(SCRIPTS_DIR, 'model-router.sh'), tmpPrompt, tmpOut, '--role', 'generate'],
|
||||
{ timeout: 120000, encoding: 'utf8', env: { ...process.env, CASAN_STEP_NAME: stepId } },
|
||||
{
|
||||
timeout: Number(process.env.CASAN_SOURCEGEN_MODEL_TIMEOUT_MS || 300000),
|
||||
encoding: 'utf8',
|
||||
env: {
|
||||
...process.env,
|
||||
CASAN_STEP_NAME: stepId,
|
||||
CASAN_MODEL_TIMEOUT_SEC: process.env.CASAN_MODEL_TIMEOUT_SEC || process.env.CASAN_SOURCEGEN_MODEL_TIMEOUT_SEC || '240',
|
||||
},
|
||||
},
|
||||
);
|
||||
if (r.status !== 0) {
|
||||
appendSourcegenAudit({
|
||||
step_id: stepId,
|
||||
event: 'generate_fallback',
|
||||
reason: `model_generate_rc=${r.status}`,
|
||||
prompt_sha256: promptHash,
|
||||
prompt_text: redactForAudit(prompt),
|
||||
source: 'template-fallback',
|
||||
});
|
||||
return { content: templateContent, source: 'template-fallback', note: `model_generate_rc=${r.status}` };
|
||||
}
|
||||
|
||||
const d = JSON.parse(readFileSync(tmpOut, 'utf8'));
|
||||
const generated = String(d.text || '').trim();
|
||||
if (generated.length < 80) {
|
||||
appendSourcegenAudit({
|
||||
step_id: stepId,
|
||||
event: 'generate_fallback',
|
||||
reason: 'model_output_too_short',
|
||||
prompt_sha256: promptHash,
|
||||
prompt_text: redactForAudit(prompt),
|
||||
model_id: d.model_id,
|
||||
route: d.route,
|
||||
input_tokens: d.input_tokens,
|
||||
output_tokens: d.output_tokens,
|
||||
total_tokens: d.total_tokens,
|
||||
source: 'template-fallback',
|
||||
});
|
||||
return { content: templateContent, source: 'template-fallback', note: 'model_output_too_short' };
|
||||
}
|
||||
const missing = required.filter((token) => !generated.includes(token));
|
||||
if (missing.length > 0) {
|
||||
appendSourcegenAudit({
|
||||
step_id: stepId,
|
||||
event: 'generate_fallback',
|
||||
reason: `missing_required=${missing.join(',')}`,
|
||||
prompt_sha256: promptHash,
|
||||
prompt_text: redactForAudit(prompt),
|
||||
model_id: d.model_id,
|
||||
route: d.route,
|
||||
input_tokens: d.input_tokens,
|
||||
output_tokens: d.output_tokens,
|
||||
total_tokens: d.total_tokens,
|
||||
source: 'template-fallback',
|
||||
});
|
||||
return { content: templateContent, source: 'template-fallback', note: `missing_required=${missing.join(',')}` };
|
||||
}
|
||||
|
||||
@@ -133,8 +248,44 @@ function generateArtifact({ stepId, title, templateContent, required = [], extra
|
||||
{ timeout: 30000, encoding: 'utf8' },
|
||||
);
|
||||
if (scan.status !== 0) {
|
||||
appendSourcegenAudit({
|
||||
step_id: stepId,
|
||||
event: 'generate_fallback',
|
||||
reason: `artifact_scan_rc=${scan.status}`,
|
||||
prompt_sha256: promptHash,
|
||||
prompt_text: redactForAudit(prompt),
|
||||
model_id: d.model_id,
|
||||
route: d.route,
|
||||
input_tokens: d.input_tokens,
|
||||
output_tokens: d.output_tokens,
|
||||
total_tokens: d.total_tokens,
|
||||
source: 'template-fallback',
|
||||
});
|
||||
return { content: templateContent, source: 'template-fallback', note: `artifact_scan_rc=${scan.status}` };
|
||||
}
|
||||
appendSourcegenAudit({
|
||||
step_id: stepId,
|
||||
event: 'generate_accept',
|
||||
prompt_sha256: promptHash,
|
||||
prompt_text: redactForAudit(prompt),
|
||||
output_sha256: sha256(generated),
|
||||
model_id: d.model_id,
|
||||
route: d.route,
|
||||
input_tokens: d.input_tokens,
|
||||
output_tokens: d.output_tokens,
|
||||
total_tokens: d.total_tokens,
|
||||
source: 'model',
|
||||
});
|
||||
appendSourcegenTelemetry({
|
||||
provider: String(d.route || '').split(':')[0] || 'unknown',
|
||||
model: d.model_id || 'unknown',
|
||||
step: stepId,
|
||||
role: 'generate',
|
||||
input_tokens: Number(d.input_tokens || 0),
|
||||
output_tokens: Number(d.output_tokens || 0),
|
||||
total_tokens: Number(d.total_tokens || 0),
|
||||
status: 'success',
|
||||
});
|
||||
return { content: `${generated}\n`, source: 'model', note: `tokens=${d.total_tokens ?? '?'}` };
|
||||
} catch (err) {
|
||||
return { content: templateContent, source: 'template-fallback', note: `generate_error=${err?.name || 'Error'}` };
|
||||
@@ -222,6 +373,9 @@ const contractPath = `docs/output/specs/${featureId}/contracts/openapi.md`;
|
||||
const ddPath = `docs/output/ipa-docs/dd/dd-mod01-okr-management.md`;
|
||||
const testcasePath = `docs/output/ipa-docs/testcase/testcase-mod01-okr-management.md`;
|
||||
const tasksPath = `docs/output/specs/${featureId}/tasks.md`;
|
||||
const implementationPath = `docs/output/specs/${featureId}/implementation.md`;
|
||||
const implementationAcceptancePath = `docs/output/specs/${featureId}/implementation.accepted.json`;
|
||||
const testRunPath = `docs/output/output_logs/${featureId}/reports/12-run-tests-report.md`;
|
||||
const codeReviewPath = `docs/output/output_logs/${featureId}/reports/11-review-code-report.md`;
|
||||
let finalTitle = `# ${step}`;
|
||||
let finalBody = '';
|
||||
@@ -269,8 +423,16 @@ switch (step) {
|
||||
break;
|
||||
}
|
||||
case '03-spec': {
|
||||
write(specPath, `# Feature Specification: OKR Web App\n\n## Feature ID\n${featureId}\n\n## Requirements\n- FR-01: Login authenticates username/email plus password and returns standard envelope.\n- FR-02: Objective creation validates title, owner, and quarter.\n- FR-03: Key Result creation validates objective, title, values, deadline, and progress.\n- FR-04: Progress update accepts 0-100 and stores a ProgressUpdate record.\n- FR-05: Dashboard list filters by role: ADMIN and MANAGER see all; EMPLOYEE sees own objectives.\n\n## Acceptance Criteria\n- Employee cannot read or update another employee objective or key result.\n- Manager can read all seeded objectives.\n- Invalid quarter format returns validation error.\n- Golden objective response fails on drift.\n\n## Source Trace\nRequirement characters read: ${requirement.length}. Architecture characters read: ${architecture.length}.\n`);
|
||||
report(`docs/output/output_logs/${featureId}/reports/03-spec-report.md`, '# STEP 3: Specify Report', `Generated ${specPath}.`, 'APPROVED', [specPath]);
|
||||
const templateContent = `# Feature Specification: OKR Web App\n\n## Feature ID\n${featureId}\n\n## Requirements\n- FR-01: Login authenticates username/email plus password and returns standard envelope.\n- FR-02: Objective creation validates title, owner, and quarter.\n- FR-03: Key Result creation validates objective, title, values, deadline, and progress.\n- FR-04: Progress update accepts 0-100 and stores a ProgressUpdate record.\n- FR-05: Dashboard list filters by role: ADMIN and MANAGER see all; EMPLOYEE sees own objectives.\n\n## Acceptance Criteria\n- Employee cannot read or update another employee objective or key result.\n- Manager can read all seeded objectives.\n- Invalid quarter format returns validation error.\n- Golden objective response fails on drift.\n\n## Role-Based Filtering\n- ADMIN can read all objectives.\n- MANAGER can read all seeded objectives.\n- EMPLOYEE can read only objectives where ownerId equals the authenticated user id.\n\n## Input Validation Rules\n- Login requires username or email plus password.\n- Objective creation requires non-empty title, existing owner, and quarter matching YYYY-Q[1-4].\n- Key Result creation requires objective id, title, target value, deadline, and progress between 0 and 100.\n- Progress update rejects values below 0 or above 100.\n\n## Source Trace\nRequirement characters read: ${requirement.length}. Architecture characters read: ${architecture.length}.\n`;
|
||||
const generated = generateArtifact({
|
||||
stepId: '03-spec',
|
||||
title: 'Feature Specification',
|
||||
templateContent,
|
||||
required: ['FR-01', 'FR-02', 'FR-03', 'FR-04', 'FR-05', 'Acceptance Criteria'],
|
||||
extraInstructions: 'Create a feature specification with FR coverage, role filtering, validation rules, acceptance criteria, and source trace.',
|
||||
});
|
||||
write(specPath, generated.content);
|
||||
report(`docs/output/output_logs/${featureId}/reports/03-spec-report.md`, '# STEP 3: Specify Report', `Generated ${specPath}. source=${generated.source} note=${generated.note}.`, 'APPROVED', [specPath]);
|
||||
break;
|
||||
}
|
||||
case '04-reviewspec': {
|
||||
@@ -302,14 +464,24 @@ switch (step) {
|
||||
try { writeFileSync(CHECKPOINT_SIDECAR, txId, 'utf8'); } catch {}
|
||||
}
|
||||
}
|
||||
write(planPath, `# Implementation Plan: OKR Web App\n\n## Stack\nNestJS, Prisma Client, SQLite, React, Vite, Tailwind, Zod, TanStack Query.\n\n## Backend Modules\n- auth: JWT login and cookie issuance.\n- users: admin/manager user list.\n- objectives: role-filtered list, detail, create.\n- key-results: detail, create, progress update.\n\n## Tests\n- Backend service tests.\n- Backend HTTP e2e tests.\n${incomplete ? '- TODO: define golden regression and rollback strategy.\n' : '- Golden regression test compares seeded manager objectives with backend/test/golden/objectives.manager.json.\n- Rollback strategy restores changed artifacts from backups through rollback-manager.sh.\n'}\n## Build\nRun npm test and npm run build for backend and frontend.\n`);
|
||||
const templateContent = `# Implementation Plan: OKR Web App\n\n## Stack\nNestJS, Prisma Client, SQLite, React, Vite, Tailwind, Zod, TanStack Query.\n\n## Backend Modules\n- auth: JWT login and cookie issuance.\n- users: admin/manager user list.\n- objectives: role-filtered list, detail, create.\n- key-results: detail, create, progress update.\n\n## Tests\n- Backend service tests.\n- Backend HTTP e2e tests.\n${incomplete ? '- TODO: define golden regression and rollback strategy.\n' : '- Golden regression test compares seeded manager objectives with backend/test/golden/objectives.manager.json.\n- Rollback strategy restores changed artifacts from backups through rollback-manager.sh.\n'}\n## Build\nRun npm test and npm run build for backend and frontend.\n`;
|
||||
const generated = generateArtifact({
|
||||
stepId: '05-plan',
|
||||
title: 'Implementation Plan',
|
||||
templateContent,
|
||||
required: ['Backend Modules', 'Tests', ...(incomplete ? [] : ['Golden regression test', 'Rollback strategy'])],
|
||||
extraInstructions: incomplete
|
||||
? 'Create the first implementation plan draft. Leave companion artifacts for the retry loop so review-plan can reject missing artifacts.'
|
||||
: 'Create the final implementation plan with Backend Modules, Tests, Golden regression test, Rollback strategy, and build commands.',
|
||||
});
|
||||
write(planPath, generated.content);
|
||||
if (!incomplete) {
|
||||
write(dataModelPath, '# Data Model\n\nUser 1:N Objective. Objective 1:N KeyResult. KeyResult 1:N ProgressUpdate. Role/status are SQLite strings constrained in service/types.\n');
|
||||
write(researchPath, '# Research\n\nSQLite selected to satisfy no Docker/Postgres e2e. Prisma Client remains application ORM. Node built-in sqlite applies migration SQL because Prisma schema-engine push fails in this Node 24 local environment.\n');
|
||||
write(quickstartPath, '# Quickstart\n\n1. npm install\n2. npm run db:setup -w backend\n3. npm run seed -w backend\n4. npm run dev -w backend\n5. npm run dev -w frontend\n');
|
||||
write(contractPath, '# API Contract\n\nPOST /auth/login\nGET /objectives\nGET /objectives/:id\nPOST /objectives\nGET /key-results/:id\nPOST /key-results\nPATCH /key-results/:id/progress\n');
|
||||
}
|
||||
report(`docs/output/output_logs/${featureId}/reports/05-plan-report-attempt-${attempt}.md`, `# STEP 5: Plan Report Attempt ${attempt}`, `Generated ${planPath}.`, 'APPROVED', [planPath]);
|
||||
report(`docs/output/output_logs/${featureId}/reports/05-plan-report-attempt-${attempt}.md`, `# STEP 5: Plan Report Attempt ${attempt}`, `Generated ${planPath}. source=${generated.source} note=${generated.note}.`, 'APPROVED', [planPath]);
|
||||
break;
|
||||
}
|
||||
case '06-reviewplan': {
|
||||
@@ -355,21 +527,66 @@ switch (step) {
|
||||
break;
|
||||
}
|
||||
case '07-dd': {
|
||||
write(ddPath, `# DD-MOD-01 OKR Management\n\n## Backend Design\nControllers are thin and call AuthService, UsersService, ObjectivesService, KeyResultsService. PrismaService is the only database access layer.\n\n## Authorization\nJwtAuthGuard verifies Bearer/cookie token. Employees are constrained to ownerId == user.sub. Managers/Admins read all objectives.\n\n## Progress Calculation\nKeyResultsService updates progress in a transaction, writes ProgressUpdate, then recalculates objective status.\n`);
|
||||
report(`docs/output/output_logs/${featureId}/reports/07-dd-report.md`, '# STEP 7: Detail Design Report', `Generated ${ddPath}.`, 'APPROVED', [ddPath]);
|
||||
const templateContent = `# DD-MOD-01 OKR Management\n\n## Backend Design\nControllers are thin and call AuthService, UsersService, ObjectivesService, KeyResultsService. PrismaService is the only database access layer.\n\n## Authorization\nJwtAuthGuard verifies Bearer/cookie token. Employees are constrained to ownerId == user.sub. Managers/Admins read all objectives.\n\n## Progress Calculation\nKeyResultsService updates progress in a transaction, writes ProgressUpdate, then recalculates objective status.\n`;
|
||||
const generated = generateArtifact({
|
||||
stepId: '07-dd',
|
||||
title: 'Detail Design',
|
||||
templateContent,
|
||||
required: ['Backend Design', 'Authorization', 'Progress Calculation'],
|
||||
extraInstructions: 'Create a detailed design for backend modules, authorization, database access, and progress calculation.',
|
||||
});
|
||||
write(ddPath, generated.content);
|
||||
report(`docs/output/output_logs/${featureId}/reports/07-dd-report.md`, '# STEP 7: Detail Design Report', `Generated ${ddPath}. source=${generated.source} note=${generated.note}.`, 'APPROVED', [ddPath]);
|
||||
break;
|
||||
}
|
||||
case '08-testkit': {
|
||||
write(testcasePath, `# Test Cases MOD-01\n\n- TC-01 login rejects wrong password.\n- TC-02 employee list returns only own objectives.\n- TC-03 manager list returns all seeded objectives.\n- TC-04 invalid objective payload returns 400.\n- TC-05 progress patch updates a key result and stores progress.\n- TC-06 golden manager objective response fails on drift.\n`);
|
||||
report(`docs/output/output_logs/${featureId}/reports/08-testkit-report.md`, '# STEP 8: Testkit Report', `Generated ${testcasePath}.`, 'APPROVED', [testcasePath]);
|
||||
const templateContent = `# Test Cases MOD-01\n\n- TC-01 login rejects wrong password.\n- TC-02 employee list returns only own objectives.\n- TC-03 manager list returns all seeded objectives.\n- TC-04 invalid objective payload returns 400.\n- TC-05 progress patch updates a key result and stores progress.\n- TC-06 golden manager objective response fails on drift.\n`;
|
||||
const generated = generateArtifact({
|
||||
stepId: '08-testkit',
|
||||
title: 'Test Case Design',
|
||||
templateContent,
|
||||
required: ['TC-01', 'TC-02', 'TC-03', 'TC-04', 'TC-05', 'TC-06'],
|
||||
extraInstructions: 'Create test cases with TC IDs covering login, role-filtering, validation, progress update, and golden drift.',
|
||||
});
|
||||
write(testcasePath, generated.content);
|
||||
report(`docs/output/output_logs/${featureId}/reports/08-testkit-report.md`, '# STEP 8: Testkit Report', `Generated ${testcasePath}. source=${generated.source} note=${generated.note}.`, 'APPROVED', [testcasePath]);
|
||||
break;
|
||||
}
|
||||
case '09-tasks': {
|
||||
write(tasksPath, `# Tasks\n\n- [X] Backend auth module with JWT and bcrypt.\n- [X] Backend users/objectives/key-results modules.\n- [X] Prisma schema, SQLite migration SQL, idempotent seed.\n- [X] Frontend login, dashboard, objective detail, create objective, key result detail.\n- [X] Backend service and e2e tests.\n- [X] Golden regression fixture and deliberate failure evidence.\n- [X] Build/test logs captured.\n`);
|
||||
report(`docs/output/output_logs/${featureId}/reports/09-tasks-report.md`, '# STEP 9: Tasks Report', `Generated ${tasksPath}.`, 'APPROVED', [tasksPath]);
|
||||
const templateContent = `# Tasks\n\n- [X] Backend auth module with JWT and bcrypt.\n- [X] Backend users/objectives/key-results modules.\n- [X] Prisma schema, SQLite migration SQL, idempotent seed.\n- [X] Frontend login, dashboard, objective detail, create objective, key result detail.\n- [X] Backend service and e2e tests.\n- [X] Golden regression fixture and deliberate failure evidence.\n- [X] Build/test logs captured.\n`;
|
||||
const generated = generateArtifact({
|
||||
stepId: '09-tasks',
|
||||
title: 'Implementation Tasks',
|
||||
templateContent,
|
||||
required: ['Backend', 'Frontend', 'tests', 'Golden'],
|
||||
extraInstructions: 'Create a task list that keeps test work and implementation work traceable to the plan and acceptance criteria.',
|
||||
});
|
||||
write(tasksPath, generated.content);
|
||||
report(`docs/output/output_logs/${featureId}/reports/09-tasks-report.md`, '# STEP 9: Tasks Report', `Generated ${tasksPath}. source=${generated.source} note=${generated.note}.`, 'APPROVED', [tasksPath]);
|
||||
break;
|
||||
}
|
||||
case '10-reviewcode': {
|
||||
case '10-implement': {
|
||||
const templateContent = `# Implementation Draft: OKR Web App\n\n## Scope\nThe OKR implementation is represented by the existing backend and frontend source files. This step records the implementation candidate that must be accepted only after STEP12 run-tests passes.\n\n## Backend Source\n- backend/src/auth/auth.service.ts\n- backend/src/objectives/objectives.service.ts\n- backend/src/key-results/key-results.service.ts\n- backend/test/services.test.ts\n- backend/test/e2e.test.ts\n\n## Frontend Source\n- frontend/src/lib/api.ts\n- frontend/src/pages/DashboardPage.tsx\n- frontend/src/pages/ObjectiveDetailPage.tsx\n\n## Acceptance Gate\nThis draft is not accepted until STEP12 writes ${implementationAcceptancePath} after the real test command exits 0.\n`;
|
||||
const generated = generateArtifact({
|
||||
stepId: '10-implement',
|
||||
title: 'Implementation Draft',
|
||||
templateContent,
|
||||
required: ['Backend Source', 'Frontend Source', 'Acceptance Gate', 'STEP12'],
|
||||
extraInstructions: 'Create a code implementation draft/manifest. Reference concrete source files and state that acceptance requires STEP12 test PASS.',
|
||||
});
|
||||
write(implementationPath, generated.content);
|
||||
try { unlinkSync(implementationAcceptancePath); } catch {}
|
||||
report(
|
||||
`docs/output/output_logs/${featureId}/reports/10-implement-report.md`,
|
||||
'# STEP 10: Implementation Draft Report',
|
||||
`Generated ${implementationPath}. source=${generated.source} note=${generated.note}. acceptance=pending STEP12.`,
|
||||
'APPROVED',
|
||||
[implementationPath],
|
||||
);
|
||||
break;
|
||||
}
|
||||
case '10-reviewcode':
|
||||
case '11-reviewcode': {
|
||||
const missingFiles = backendFiles.filter((path) => {
|
||||
try {
|
||||
readFileSync(path, 'utf8');
|
||||
@@ -382,7 +599,7 @@ switch (step) {
|
||||
const issues = [];
|
||||
if (!serviceText.includes('PrismaService')) issues.push('PrismaService not used');
|
||||
if (!serviceText.includes('ForbiddenException')) issues.push('authorization exception not found');
|
||||
if (!serviceText.includes('golden objective list response does not drift')) issues.push('golden test not found');
|
||||
if (!/golden:.*objective list.*does not drift/i.test(serviceText)) issues.push('golden test not found');
|
||||
issues.push(...missingFiles.map((path) => `missing file: ${path}`));
|
||||
let verdict = issues.length === 0 ? 'APPROVED' : 'REJECTED';
|
||||
// WP-B: model judge gate
|
||||
@@ -402,6 +619,42 @@ switch (step) {
|
||||
);
|
||||
break;
|
||||
}
|
||||
case '12-runtests': {
|
||||
let implementation = '';
|
||||
try { implementation = readFileSync(implementationPath, 'utf8'); } catch {}
|
||||
const testCmd = process.env.CASAN_SOURCEGEN_TEST_CMD || 'npm test';
|
||||
const r = spawnSync('bash', ['-lc', testCmd], {
|
||||
cwd: __appRoot,
|
||||
encoding: 'utf8',
|
||||
timeout: Number(process.env.CASAN_SOURCEGEN_TEST_TIMEOUT_MS || 180000),
|
||||
env: { ...process.env },
|
||||
});
|
||||
const output = `${r.stdout || ''}\n${r.stderr || ''}`.trim().slice(0, 6000);
|
||||
const passed = r.status === 0;
|
||||
const acceptance = {
|
||||
timestamp: new Date().toISOString(),
|
||||
feature_id: featureId,
|
||||
implementation_artifact: implementationPath,
|
||||
implementation_sha256: sha256(implementation),
|
||||
test_command: testCmd,
|
||||
test_exit_code: r.status,
|
||||
accepted: passed,
|
||||
};
|
||||
if (passed) {
|
||||
write(implementationAcceptancePath, `${JSON.stringify(acceptance, null, 2)}\n`);
|
||||
} else {
|
||||
try { unlinkSync(implementationAcceptancePath); } catch {}
|
||||
}
|
||||
report(
|
||||
testRunPath,
|
||||
'# STEP 12: Run Tests Report',
|
||||
`Command: ${testCmd}\nExit code: ${r.status}\nAccepted implementation: ${passed ? 'yes' : 'no'}\n\n## Output\n${output || '(no output)'}`,
|
||||
passed ? 'PASS' : 'FAIL',
|
||||
passed ? [implementationPath, implementationAcceptancePath] : [implementationPath],
|
||||
passed ? [] : [`test command failed rc=${r.status}`],
|
||||
);
|
||||
break;
|
||||
}
|
||||
default:
|
||||
throw new Error(`Unknown step: ${step}`);
|
||||
}
|
||||
|
||||
@@ -40,7 +40,10 @@ const DIAGRAM_STEP = {
|
||||
'07-dd': 'STEP8',
|
||||
'08-testkit': 'STEP8b',
|
||||
'09-tasks': 'STEP9',
|
||||
'10-implement': 'STEP10',
|
||||
'10-reviewcode': 'STEP11',
|
||||
'11-reviewcode': 'STEP11',
|
||||
'12-runtests': 'STEP12',
|
||||
};
|
||||
|
||||
const FULL_DIAGRAM = [
|
||||
@@ -168,7 +171,7 @@ function logLoop(fromStep, verdict, toStep, note) {
|
||||
log('warn', 'boss', line);
|
||||
}
|
||||
|
||||
function runHarness({ id, agent, step, attempt = '1' }) {
|
||||
function runHarness({ id, agent, step, attempt = '1', extraEnv = {} }) {
|
||||
const diagram = DIAGRAM_STEP[step] ?? step;
|
||||
const input = `${casanDir}/${id}-input.txt`;
|
||||
const output = `${casanDir}/${id}-output.md`;
|
||||
@@ -192,6 +195,7 @@ function runHarness({ id, agent, step, attempt = '1' }) {
|
||||
CASAN_AGENT_NAME: agent,
|
||||
CASAN_STEP_NAME: id,
|
||||
CASAN_PHASE_REPORT: phaseReport,
|
||||
...extraEnv,
|
||||
},
|
||||
},
|
||||
);
|
||||
@@ -321,19 +325,37 @@ const sequence = [
|
||||
{ id: '01-srs', agent: 'okr.srs', step: '01-srs' },
|
||||
{ id: '02-bd', agent: 'okr.bd', step: '02-bd' },
|
||||
{ id: '03-spec', agent: 'speckit.specify', step: '03-spec' },
|
||||
{ id: '04-reviewspec', agent: 'okr.reviewspec', step: '04-reviewspec' },
|
||||
{ id: '05-plan-attempt-1', agent: 'speckit.plan', step: '05-plan', attempt: '1' },
|
||||
];
|
||||
|
||||
for (const item of sequence) {
|
||||
const { verdict } = runHarness(item);
|
||||
if (item.step === '04-reviewspec' && verdict === 'REJECTED') {
|
||||
// Diagram loop STEP5 -> STEP3. The current sequence expects APPROVED here;
|
||||
// if a rejection ever happens we surface the loop instead of hiding it.
|
||||
logLoop('STEP5', verdict, 'STEP3', 'review-spec rejected; sequence continues but needs attention');
|
||||
runHarness(item);
|
||||
}
|
||||
|
||||
let reviewSpec = runHarness({ id: '04-reviewspec', agent: 'okr.reviewspec', step: '04-reviewspec' });
|
||||
if (reviewSpec.verdict === 'REJECTED') {
|
||||
logLoop('STEP5', reviewSpec.verdict, 'STEP3', 'review-spec rejected; retrying spec with template fallback');
|
||||
runHarness({
|
||||
id: '04b-spec-attempt-2',
|
||||
agent: 'speckit.specify',
|
||||
step: '03-spec',
|
||||
attempt: '2',
|
||||
extraEnv: { CASAN_GEN_MODE: 'template' },
|
||||
});
|
||||
reviewSpec = runHarness({
|
||||
id: '04c-reviewspec-attempt-2',
|
||||
agent: 'okr.reviewspec',
|
||||
step: '04-reviewspec',
|
||||
attempt: '2',
|
||||
extraEnv: { CASAN_MODEL_PRIMARY: process.env.CASAN_REVIEW_ESCALATE_MODEL || 'openai:gpt-4o-mini' },
|
||||
});
|
||||
if (reviewSpec.verdict === 'REJECTED') {
|
||||
logLoop('STEP5', reviewSpec.verdict, 'STOP', 'review-spec rejected after fallback retry');
|
||||
throw new Error('review-spec rejected after fallback retry');
|
||||
}
|
||||
}
|
||||
|
||||
runHarness({ id: '05-plan-attempt-1', agent: 'speckit.plan', step: '05-plan', attempt: '1' });
|
||||
|
||||
const reviewPlan1 = runHarness({ id: '06-reviewplan-attempt-1', agent: 'okr.reviewplan', step: '06-reviewplan', attempt: '1' });
|
||||
if (reviewPlan1.verdict === 'REJECTED') {
|
||||
logLoop('STEP7', reviewPlan1.verdict, 'STEP6', 'BACK-TO-PLAN: retrying plan with missing criteria fixed');
|
||||
@@ -345,7 +367,7 @@ runHarness({ id: '07-plan-attempt-2', agent: 'speckit.plan', step: '05-plan', at
|
||||
const fallbackOut = `${casanDir}/model-fallback-output.txt`;
|
||||
log('debug', 'boss', `model-fallback invoked (real primary failure) → ${fallbackOut}`);
|
||||
execFileSync(
|
||||
`/model-fallback.sh`,
|
||||
`${HARNESS_BASH}/model-fallback.sh`,
|
||||
[
|
||||
fallbackOut,
|
||||
// Real primary failure: reading a nonexistent path exits non-zero (not a
|
||||
@@ -366,46 +388,66 @@ appendBoss(`Model fallback invoked; output ${fallbackOut}`);
|
||||
const driftCandidate = `${casanDir}/drift-plan-candidate.txt`;
|
||||
copyFileSync(fallbackOut, driftCandidate);
|
||||
log('debug', 'boss', 'drift-detect: fallback output vs golden baseline');
|
||||
execFileSync(`/drift-detect.sh`, [
|
||||
execFileSync(`${HARNESS_BASH}/drift-detect.sh`, [
|
||||
GOLDEN_PLAN,
|
||||
driftCandidate,
|
||||
'.specify/logs/level5/okr-plan-drift-report.json',
|
||||
], { cwd: root, stdio: 'inherit' });
|
||||
appendBoss('Drift detection invoked: fallback output vs golden baseline.');
|
||||
|
||||
const reviewPlan2 = runHarness({ id: '08-reviewplan-attempt-2', agent: 'okr.reviewplan', step: '06-reviewplan', attempt: '2' });
|
||||
const reviewPlan2 = runHarness({
|
||||
id: '08-reviewplan-attempt-2',
|
||||
agent: 'okr.reviewplan',
|
||||
step: '06-reviewplan',
|
||||
attempt: '2',
|
||||
extraEnv: { CASAN_MODEL_PRIMARY: process.env.CASAN_REVIEW_ESCALATE_MODEL || 'openai:gpt-4o-mini' },
|
||||
});
|
||||
if (reviewPlan2.verdict === 'REJECTED') {
|
||||
logLoop('STEP7', reviewPlan2.verdict, 'STEP6', 'BACK-TO-PLAN attempt 2 still rejected');
|
||||
throw new Error('review-plan rejected after retry');
|
||||
}
|
||||
runHarness({ id: '09-dd', agent: 'okr.dd', step: '07-dd' });
|
||||
runHarness({ id: '10-testkit', agent: 'okr.testkit', step: '08-testkit' });
|
||||
runHarness({ id: '11-tasks', agent: 'speckit.tasks', step: '09-tasks' });
|
||||
const reviewCode = runHarness({ id: '12-reviewcode', agent: 'okr.reviewcode', step: '10-reviewcode' });
|
||||
runHarness({ id: '12-implement', agent: 'speckit.implement', step: '10-implement' });
|
||||
const reviewCode = runHarness({ id: '13-reviewcode', agent: 'okr.reviewcode', step: '11-reviewcode' });
|
||||
if (reviewCode.verdict === 'REJECTED') {
|
||||
// Diagram loop STEP11 -> STEP10 (implement is not an agent step yet).
|
||||
logLoop('STEP11', reviewCode.verdict, 'STEP10', 'review-code rejected; implement step must be re-run');
|
||||
runHarness({ id: '13b-implement-attempt-2', agent: 'speckit.implement', step: '10-implement', attempt: '2' });
|
||||
const reviewCode2 = runHarness({
|
||||
id: '13c-reviewcode-attempt-2',
|
||||
agent: 'okr.reviewcode',
|
||||
step: '11-reviewcode',
|
||||
attempt: '2',
|
||||
extraEnv: { CASAN_MODEL_PRIMARY: process.env.CASAN_REVIEW_ESCALATE_MODEL || 'openai:gpt-4o-mini' },
|
||||
});
|
||||
if (reviewCode2.verdict === 'REJECTED') {
|
||||
logLoop('STEP11', reviewCode2.verdict, 'STOP', 'review-code rejected after implement retry');
|
||||
throw new Error('review-code rejected after implement retry');
|
||||
}
|
||||
}
|
||||
const runTests = runHarness({ id: '14-runtests', agent: 'okr.testkit run-tests', step: '12-runtests' });
|
||||
if (runTests.verdict === 'FAIL') {
|
||||
logLoop('STEP12', runTests.verdict, 'STEP6', 'tests FAIL: re-plan/re-implement before acceptance');
|
||||
}
|
||||
|
||||
const rollbackDir = 'docs/output/casan/app-evidence';
|
||||
mkdirSync(rollbackDir, { recursive: true });
|
||||
const rollbackTarget = `${rollbackDir}/rollback-target.txt`;
|
||||
const rollbackBackup = `${rollbackDir}/rollback-target.backup.txt`;
|
||||
writeFileSync(rollbackTarget, 'original pipeline rollback content\n', 'utf8');
|
||||
copyFileSync(rollbackTarget, rollbackBackup);
|
||||
writeFileSync(`${rollbackDir}/rollback-before.txt`, readFileSync(rollbackTarget, 'utf8'), 'utf8');
|
||||
writeFileSync(rollbackTarget, 'changed content that must be undone\n', 'utf8');
|
||||
writeFileSync(`${rollbackDir}/rollback-changed.txt`, readFileSync(rollbackTarget, 'utf8'), 'utf8');
|
||||
const record = execFileSync(`/rollback-manager.sh`, [
|
||||
'record',
|
||||
'restore rollback target evidence file',
|
||||
`cp ${rollbackBackup} ${rollbackTarget}`,
|
||||
const record = execFileSync(`${HARNESS_BASH}/rollback-manager.sh`, [
|
||||
'checkpoint',
|
||||
rollbackTarget,
|
||||
], { cwd: root, encoding: 'utf8' });
|
||||
writeFileSync(`${rollbackDir}/rollback-record.stdout`, record, 'utf8');
|
||||
const tx = record.match(/transaction_id=([^\s]+)/)?.[1];
|
||||
if (!tx) {
|
||||
throw new Error('rollback transaction id not found');
|
||||
}
|
||||
const execute = execFileSync(`/rollback-manager.sh`, ['execute', tx], {
|
||||
writeFileSync(rollbackTarget, 'changed content that must be undone\n', 'utf8');
|
||||
writeFileSync(`${rollbackDir}/rollback-changed.txt`, readFileSync(rollbackTarget, 'utf8'), 'utf8');
|
||||
const execute = execFileSync(`${HARNESS_BASH}/rollback-manager.sh`, ['execute', tx], {
|
||||
cwd: root,
|
||||
encoding: 'utf8',
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user