import { execFileSync } from 'node:child_process'; import { existsSync, appendFileSync, copyFileSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from 'node:fs'; import { dirname, join } from 'node:path'; import { enabled, log, redact, LOG_LEVEL } from './casan-log.mjs'; // --dry-run: walk the FULL 13-STEP diagram with stub agents. Every stub still // goes through casan-harness.sh (H4-in -> H5 -> H6 -> exec -> H4-out), so log // levels and pipeline-run.jsonl can be demonstrated deterministically offline. const dryRun = process.argv.includes('--dry-run'); const featureId = '001-okr-web-app'; const root = process.cwd(); // Plan-01: harness relocated to packages/casan-harness/, domain to apps/okr/domain/; // fall back to the pre-move paths so this runner works from either layout. const HARNESS_BASH = existsSync(join(root, 'packages/casan-harness/scripts/bash')) ? 'packages/casan-harness/scripts/bash' : '.specify/scripts/bash'; const HARNESS = `${HARNESS_BASH}/casan-harness.sh`; const GOLDEN_PLAN = existsSync(join(root, 'apps/okr/domain/golden-runs/okr-plan.golden.txt')) ? 'apps/okr/domain/golden-runs/okr-plan.golden.txt' : '.specify/level5/golden-runs/okr-plan.golden.txt'; const logDir = `docs/output/output_logs/${featureId}`; const casanDir = `${logDir}/casan`; const reportsDir = `${logDir}/reports`; const contextPath = dryRun ? `${logDir}/pipeline-context.dryrun.yaml` : `${logDir}/pipeline-context.yaml`; const bossLog = dryRun ? `${logDir}/00-boss.dryrun.log.md` : `${logDir}/00-boss.log.md`; const runLogPath = '.specify/logs/pipeline-run.jsonl'; const runId = `${dryRun ? 'dry' : 'run'}-${new Date().toISOString().replace(/[:.]/g, '-')}-${process.pid}`; // Map casan-step keys -> STEP ids of the FULL diagram // (optimize-docs/CASAN_PIPELINE_WORKFLOW.md section 1). STEP4/10/12/13 have no // real agent step yet; they run as stubs in --dry-run only. const DIAGRAM_STEP = { '01-srs': 'STEP1', '02-bd': 'STEP2', '03-spec': 'STEP3', '04-reviewspec': 'STEP5', '05-plan': 'STEP6', '06-reviewplan': 'STEP7', '07-dd': 'STEP8', '08-testkit': 'STEP8b', '09-tasks': 'STEP9', '10-implement': 'STEP10', '10-reviewcode': 'STEP11', '11-reviewcode': 'STEP11', '12-runtests': 'STEP12', }; const FULL_DIAGRAM = [ ['STEP1', 'okr.srs'], ['STEP2', 'okr.bd'], ['STEP3', 'speckit.specify'], ['STEP4', 'speckit.clarify'], ['STEP5', 'okr.reviewspec'], ['STEP6', 'speckit.plan'], ['STEP7', 'okr.reviewplan'], ['STEP8', 'okr.dd'], ['STEP8b', 'okr.testkit'], ['STEP9', 'speckit.tasks'], ['STEP10', 'speckit.implement'], ['STEP11', 'okr.reviewcode'], ['STEP12', 'okr.testkit run-tests'], ['STEP13', 'deploy'], ]; const summaryRows = []; const loopsFired = []; mkdirSync(casanDir, { recursive: true }); mkdirSync(reportsDir, { recursive: true }); writeFileSync( contextPath, `feature-id: ${featureId}\nmodule-id: MOD-01\nmodule-keyword: okr-management\ntech-stack: NestJS + Prisma + SQLite + React + Vite + Tailwind\nsteps:\n`, 'utf8', ); writeFileSync(bossLog, `# Boss Log ${featureId}\n\n`, 'utf8'); log('info', 'boss', `pipeline start run_id=${runId} mode=${dryRun ? 'dry-run' : 'real'} log_level=${LOG_LEVEL}`); function appendBoss(line) { const ts = new Date().toISOString(); writeFileSync(bossLog, `${readFileSync(bossLog, 'utf8')}- ${ts} ${line}\n`, 'utf8'); } function appendRunLog(record) { mkdirSync(dirname(runLogPath), { recursive: true }); appendFileSync(runLogPath, `${JSON.stringify(record)}\n`, 'utf8'); } function latestAgentTrace(stepName) { const traceDir = '.specify/logs/trace'; const files = readdirSync(traceDir) .filter((file) => file.startsWith('agentops-') && file.endsWith('.json')) .map((file) => join(traceDir, file)) .sort((a, b) => statSync(b).mtimeMs - statSync(a).mtimeMs); for (const file of files) { const record = JSON.parse(readFileSync(file, 'utf8')); if (record.step === stepName) { return { traceId: record.trace_id, path: file, status: record.status, record }; } } throw new Error(`No trace found for ${stepName}`); } function verdictFromOutput(path) { const text = readFileSync(path, 'utf8'); const match = text.match(/verdict:\s*([A-Z_]+)/); return match?.[1] ?? 'UNKNOWN'; } function appendContext({ id, agent, output, trace }) { const verdict = verdictFromOutput(output); const existing = readFileSync(contextPath, 'utf8'); writeFileSync( contextPath, `${existing} - id: ${id}\n agent: ${agent}\n artifact: ${output}\n verdict: ${verdict}\n trace_id: ${trace.traceId}\n trace_file: ${trace.path}\n status: ${trace.status}\n`, 'utf8', ); } function readPhaseReport(path) { try { return JSON.parse(readFileSync(path, 'utf8')); } catch { return null; } } function phasesBrief(phases) { if (!phases?.phases?.length) return 'unavailable'; return `${phases.phases.map((p) => `${p.phase}:rc=${p.rc}`).join(' → ')} (cache=${phases.cache})`; } // Shared post-execution bookkeeping for real and dry-run steps: human logs at // info/debug, one machine-readable line per step in pipeline-run.jsonl. function recordStep({ diagram, id, agent, attempt, verdict, ms, trace, phases, input, output, error }) { const tokens = trace?.record?.total_tokens ?? null; log('info', 'boss', `${diagram} · ${agent} · verdict=${verdict} · ${ms}ms · attempt=${attempt}`); log( 'debug', 'boss', `${diagram} detail id=${id} trace_id=${trace?.traceId ?? 'n/a'} status=${trace?.status ?? 'n/a'} tokens=${tokens ?? '?'} input=${input} output=${output} harness=${phasesBrief(phases)}`, ); appendRunLog({ ts: new Date().toISOString(), run_id: runId, mode: dryRun ? 'dry-run' : 'real', step: diagram, id, agent, attempt: Number(attempt), verdict, error: error ?? null, ms, tokens, trace_id: trace?.traceId ?? null, harness: phases?.phases ?? null, cache: phases?.cache ?? null, input, output, }); summaryRows.push({ diagram, id, agent, attempt, verdict, ms, tokens }); } // The diagram's self-correcting loops. Logged whenever a review verdict makes // the Boss re-run an earlier step (STEP5->STEP3, STEP7->STEP6, STEP11->STEP10, // STEP12->STEP6). function logLoop(fromStep, verdict, toStep, note) { const line = `LOOP ${fromStep} verdict=${verdict} → ${toStep} (${note})`; loopsFired.push(line); log('warn', 'boss', line); } function runHarness({ id, agent, step, attempt = '1', extraEnv = {} }) { const diagram = DIAGRAM_STEP[step] ?? step; const input = `${casanDir}/${id}-input.txt`; const output = `${casanDir}/${id}-output.md`; const phaseReport = `${casanDir}/${id}-phases.json`; const payload = `feature ${featureId}\nstep ${id}\nagent ${agent}\nattempt ${attempt}\nsource docs/input/okr-requirement.md\n`; writeFileSync(input, payload, 'utf8'); appendBoss(`START ${id} ${agent} attempt ${attempt}`); log('debug', 'boss', `${diagram} start agent=${agent} attempt=${attempt} action=agent_step_${id}`); if (enabled('trace')) log('trace', 'boss', `${diagram} input payload (redacted): ${redact(payload)}`); const startedAt = Date.now(); try { execFileSync( HARNESS, [input, output, `agent_step_${id}`, '--', 'node', 'scripts/casan-step.mjs', step, attempt], { cwd: root, stdio: 'inherit', env: { ...process.env, CASAN_AGENT: agent, CASAN_AGENT_NAME: agent, CASAN_STEP_NAME: id, CASAN_PHASE_REPORT: phaseReport, ...extraEnv, }, }, ); } catch (error) { const ms = Date.now() - startedAt; const phases = readPhaseReport(phaseReport); log('error', 'boss', `${diagram} · ${agent} · FAILED rc=${error.status ?? '?'} · ${ms}ms · harness=${phasesBrief(phases)}`); recordStep({ diagram, id, agent, attempt, verdict: 'ERROR', ms, trace: null, phases, input, output, error: `harness rc=${error.status ?? 'unknown'}` }); throw error; } const ms = Date.now() - startedAt; const trace = latestAgentTrace(id); appendContext({ id, agent, output, trace }); const verdict = verdictFromOutput(output); appendBoss(`END ${id} verdict ${verdict} trace ${trace.traceId}`); const phases = readPhaseReport(phaseReport); recordStep({ diagram, id, agent, attempt, verdict, ms, trace, phases, input, output }); if (enabled('trace')) log('trace', 'boss', `${diagram} output excerpt (redacted): ${redact(readFileSync(output, 'utf8'))}`); return { output, verdict, trace }; } function printSummary() { if (!enabled('info')) return; const lines = []; lines.push(''); lines.push(`═══ CASAN pipeline summary · run_id=${runId} · mode=${dryRun ? 'dry-run' : 'real'} ═══`); lines.push('STEP | agent | verdict | attempts | last ms | tokens'); lines.push('--------|------------------------|-----------|----------|---------|-------'); for (const [diagram, defaultAgent] of FULL_DIAGRAM) { const rows = summaryRows.filter((r) => r.diagram === diagram); if (rows.length === 0) { lines.push(`${diagram.padEnd(7)} | ${defaultAgent.padEnd(22)} | — | 0 | — | — (not in this run${dryRun ? '' : '; stub available via --dry-run'})`); continue; } const last = rows[rows.length - 1]; lines.push( `${diagram.padEnd(7)} | ${last.agent.padEnd(22)} | ${String(last.verdict).padEnd(9)} | ${String(rows.length).padEnd(8)} | ${String(last.ms).padEnd(7)} | ${last.tokens ?? '—'}`, ); } lines.push(''); lines.push(loopsFired.length ? `Loops fired:\n${loopsFired.map((l) => ` - ${l}`).join('\n')}` : 'Loops fired: none'); lines.push(`Machine-readable log: ${runLogPath} (jq 'select(.run_id=="${runId}")' — 1 dòng/step)`); console.log(lines.join('\n')); } // ───────────────────────────── dry-run mode ───────────────────────────── // Stub every agent call but keep the production wrapper in the loop. The stub // command writes a deterministic artifact containing the wanted verdict, so // the Boss's verdict parsing, loop handling, and logging all run for real. function runDryStep({ diagram, agent, attempt = '1', verdict = 'APPROVED', extraEnv = {} }) { const id = `dry-${diagram}-attempt-${attempt}`; const input = `${casanDir}/${id}-input.txt`; const output = `${casanDir}/${id}-output.md`; const phaseReport = `${casanDir}/${id}-phases.json`; const payload = `feature ${featureId}\nstep ${diagram}\nagent ${agent}\nattempt ${attempt}\nmode dry-run stub\n`; writeFileSync(input, payload, 'utf8'); appendBoss(`START ${id} ${agent} attempt ${attempt} (dry-run stub)`); log('debug', 'boss', `${diagram} start agent=${agent} attempt=${attempt} action=agent_step_${id} (stub agent, harness thật)`); if (enabled('trace')) log('trace', 'boss', `${diagram} input payload (redacted): ${redact(payload)}`); const stubCmd = `printf '# %s dry-run stub artifact\\n\\nverdict: %s\\n' '${diagram}' '${verdict}' > "$CASAN_OUTPUT"`; const startedAt = Date.now(); execFileSync( HARNESS, [input, output, `agent_step_${id}`, '--', 'bash', '-c', stubCmd], { cwd: root, stdio: 'inherit', env: { ...process.env, CASAN_AGENT: agent, CASAN_AGENT_NAME: agent, CASAN_STEP_NAME: id, CASAN_PHASE_REPORT: phaseReport, ...extraEnv, }, }, ); const ms = Date.now() - startedAt; const trace = latestAgentTrace(id); appendContext({ id, agent, output, trace }); const got = verdictFromOutput(output); appendBoss(`END ${id} verdict ${got} trace ${trace.traceId}`); recordStep({ diagram, id, agent, attempt, verdict: got, ms, trace, phases: readPhaseReport(phaseReport), input, output }); return got; } if (dryRun) { const agentOf = Object.fromEntries(FULL_DIAGRAM); for (const diagram of ['STEP1', 'STEP2', 'STEP3', 'STEP4', 'STEP5']) { runDryStep({ diagram, agent: agentOf[diagram] }); } // STEP6 -> STEP7 with the BACK-TO-PLAN loop firing once (attempt 1 REJECTED). runDryStep({ diagram: 'STEP6', agent: agentOf.STEP6 }); let v = runDryStep({ diagram: 'STEP7', agent: agentOf.STEP7, verdict: 'REJECTED' }); if (v === 'REJECTED') { logLoop('STEP7', v, 'STEP6', 'BACK-TO-PLAN: re-plan attempt 2'); runDryStep({ diagram: 'STEP6', agent: agentOf.STEP6, attempt: '2' }); v = runDryStep({ diagram: 'STEP7', agent: agentOf.STEP7, attempt: '2' }); } for (const diagram of ['STEP8', 'STEP8b', 'STEP9', 'STEP10', 'STEP11']) { runDryStep({ diagram, agent: agentOf[diagram] }); } // STEP12 run-tests with the FAIL -> STEP6 loop firing once. v = runDryStep({ diagram: 'STEP12', agent: agentOf.STEP12, verdict: 'FAIL' }); if (v === 'FAIL') { logLoop('STEP12', v, 'STEP6', 'tests FAIL: re-plan then re-test'); runDryStep({ diagram: 'STEP6', agent: agentOf.STEP6, attempt: '3' }); v = runDryStep({ diagram: 'STEP12', agent: agentOf.STEP12, attempt: '2', verdict: 'PASS' }); } // STEP13's payload mentions "deploy", which H5 correctly classifies as // high-risk → approval required. Simulate a distinct human approver so the // dry-run walks the real approval path (separation of duties holds: // actor=developer != approver=qa-lead). log('debug', 'boss', 'STEP13 is high-risk (deploy): supplying human approval approver=qa-lead for the dry-run'); runDryStep({ diagram: 'STEP13', agent: agentOf.STEP13, extraEnv: { CASAN_APPROVAL_DECISION: 'approve', CASAN_APPROVER: 'qa-lead' }, }); printSummary(); log('info', 'boss', `dry-run complete. Boss log: ${bossLog}. Context: ${contextPath}.`); process.exit(0); } // ───────────────────────────── real pipeline ───────────────────────────── const sequence = [ { id: '01-srs', agent: 'okr.srs', step: '01-srs' }, { id: '02-bd', agent: 'okr.bd', step: '02-bd' }, { id: '03-spec', agent: 'speckit.specify', step: '03-spec' }, ]; for (const item of sequence) { runHarness(item); } let reviewSpec = runHarness({ id: '04-reviewspec', agent: 'okr.reviewspec', step: '04-reviewspec' }); if (reviewSpec.verdict === 'REJECTED') { logLoop('STEP5', reviewSpec.verdict, 'STEP3', 'review-spec rejected; retrying spec with template fallback'); runHarness({ id: '04b-spec-attempt-2', agent: 'speckit.specify', step: '03-spec', attempt: '2', extraEnv: { CASAN_GEN_MODE: 'template' }, }); reviewSpec = runHarness({ id: '04c-reviewspec-attempt-2', agent: 'okr.reviewspec', step: '04-reviewspec', attempt: '2', extraEnv: { CASAN_MODEL_PRIMARY: process.env.CASAN_REVIEW_ESCALATE_MODEL || 'openai:gpt-4o-mini' }, }); if (reviewSpec.verdict === 'REJECTED') { logLoop('STEP5', reviewSpec.verdict, 'STOP', 'review-spec rejected after fallback retry'); throw new Error('review-spec rejected after fallback retry'); } } runHarness({ id: '05-plan-attempt-1', agent: 'speckit.plan', step: '05-plan', attempt: '1' }); const reviewPlan1 = runHarness({ id: '06-reviewplan-attempt-1', agent: 'okr.reviewplan', step: '06-reviewplan', attempt: '1' }); if (reviewPlan1.verdict === 'REJECTED') { logLoop('STEP7', reviewPlan1.verdict, 'STEP6', 'BACK-TO-PLAN: retrying plan with missing criteria fixed'); } appendBoss('BACK-TO-PLAN triggered by reviewplan rejection; retrying plan with missing criteria fixed.'); runHarness({ id: '07-plan-attempt-2', agent: 'speckit.plan', step: '05-plan', attempt: '2' }); const fallbackOut = `${casanDir}/model-fallback-output.txt`; log('debug', 'boss', `model-fallback invoked (real primary failure) → ${fallbackOut}`); execFileSync( `${HARNESS_BASH}/model-fallback.sh`, [ fallbackOut, // Real primary failure: reading a nonexistent path exits non-zero (not a // hardcoded `exit 9` stub) — the fallback route is driven by a genuine error. '--primary', 'cat /nonexistent/casan/primary-model-endpoint', '--fallback', 'printf "Generate a safe OKR plan for employee ***MASKED_EMAIL***.\\nExpected sections:\\n- Objective\\n- Key Results\\n- Security gate\\n- Governance decision\\n- AgentOps metrics\\n"', ], { cwd: root, stdio: 'inherit' }, ); appendBoss(`Model fallback invoked; output ${fallbackOut}`); // Drift: compare this run's fallback plan output against the committed golden // baseline. A clean run matches the golden (similarity=1.0 → no drift). The // ability to DETECT real drift (similarity<1.0 on differing docs) is proven // independently in adversarial-harness-tests.sh (H7 drift, two different files). const driftCandidate = `${casanDir}/drift-plan-candidate.txt`; copyFileSync(fallbackOut, driftCandidate); log('debug', 'boss', 'drift-detect: fallback output vs golden baseline'); execFileSync(`${HARNESS_BASH}/drift-detect.sh`, [ GOLDEN_PLAN, driftCandidate, '.specify/logs/level5/okr-plan-drift-report.json', ], { cwd: root, stdio: 'inherit' }); appendBoss('Drift detection invoked: fallback output vs golden baseline.'); const reviewPlan2 = runHarness({ id: '08-reviewplan-attempt-2', agent: 'okr.reviewplan', step: '06-reviewplan', attempt: '2', extraEnv: { CASAN_MODEL_PRIMARY: process.env.CASAN_REVIEW_ESCALATE_MODEL || 'openai:gpt-4o-mini' }, }); if (reviewPlan2.verdict === 'REJECTED') { logLoop('STEP7', reviewPlan2.verdict, 'STEP6', 'BACK-TO-PLAN attempt 2 still rejected'); throw new Error('review-plan rejected after retry'); } runHarness({ id: '09-dd', agent: 'okr.dd', step: '07-dd' }); runHarness({ id: '10-testkit', agent: 'okr.testkit', step: '08-testkit' }); runHarness({ id: '11-tasks', agent: 'speckit.tasks', step: '09-tasks' }); runHarness({ id: '12-implement', agent: 'speckit.implement', step: '10-implement' }); const reviewCode = runHarness({ id: '13-reviewcode', agent: 'okr.reviewcode', step: '11-reviewcode' }); if (reviewCode.verdict === 'REJECTED') { logLoop('STEP11', reviewCode.verdict, 'STEP10', 'review-code rejected; implement step must be re-run'); runHarness({ id: '13b-implement-attempt-2', agent: 'speckit.implement', step: '10-implement', attempt: '2' }); const reviewCode2 = runHarness({ id: '13c-reviewcode-attempt-2', agent: 'okr.reviewcode', step: '11-reviewcode', attempt: '2', extraEnv: { CASAN_MODEL_PRIMARY: process.env.CASAN_REVIEW_ESCALATE_MODEL || 'openai:gpt-4o-mini' }, }); if (reviewCode2.verdict === 'REJECTED') { logLoop('STEP11', reviewCode2.verdict, 'STOP', 'review-code rejected after implement retry'); throw new Error('review-code rejected after implement retry'); } } const runTests = runHarness({ id: '14-runtests', agent: 'okr.testkit run-tests', step: '12-runtests' }); if (runTests.verdict === 'FAIL') { logLoop('STEP12', runTests.verdict, 'STEP6', 'tests FAIL: re-plan/re-implement before acceptance'); } const rollbackDir = 'docs/output/casan/app-evidence'; mkdirSync(rollbackDir, { recursive: true }); const rollbackTarget = `${rollbackDir}/rollback-target.txt`; writeFileSync(rollbackTarget, 'original pipeline rollback content\n', 'utf8'); writeFileSync(`${rollbackDir}/rollback-before.txt`, readFileSync(rollbackTarget, 'utf8'), 'utf8'); const record = execFileSync(`${HARNESS_BASH}/rollback-manager.sh`, [ 'checkpoint', rollbackTarget, ], { cwd: root, encoding: 'utf8' }); writeFileSync(`${rollbackDir}/rollback-record.stdout`, record, 'utf8'); const tx = record.match(/transaction_id=([^\s]+)/)?.[1]; if (!tx) { throw new Error('rollback transaction id not found'); } writeFileSync(rollbackTarget, 'changed content that must be undone\n', 'utf8'); writeFileSync(`${rollbackDir}/rollback-changed.txt`, readFileSync(rollbackTarget, 'utf8'), 'utf8'); const execute = execFileSync(`${HARNESS_BASH}/rollback-manager.sh`, ['execute', tx], { cwd: root, encoding: 'utf8', }); writeFileSync(`${rollbackDir}/rollback-execute.stdout`, execute, 'utf8'); writeFileSync(`${rollbackDir}/rollback-after.txt`, readFileSync(rollbackTarget, 'utf8'), 'utf8'); appendBoss(`Rollback transaction ${tx} executed; before/changed/after evidence captured.`); log('debug', 'boss', `rollback transaction ${tx} recorded + executed (evidence under ${rollbackDir})`); printSummary(); const summary = `Pipeline complete. Context: ${contextPath}. Boss log: ${bossLog}. Last step artifacts under ${reportsDir}.\n`; writeFileSync(`${rollbackDir}/pipeline-summary.txt`, summary, 'utf8'); console.log(summary);