CASAN/backend/test/llm-judge.test.ts

/**
 * LLM-judge gate: sends live API responses to Claude and fails if the model
 * returns REJECTED. This catches semantic breakage that structural assertions miss
 * (e.g. scrambled titles, impossible status/progress combinations, data leakage).
 *
 * Requires ANTHROPIC_API_KEY to be set. The test is skipped — not failed — when
 * the key is absent, so local dev without a key still passes CI.
 *
 * REJECTED→fix cycle:
 *   1. CI logs the judge verdict and reason.
 *   2. Developer reads the reason, fixes the business logic or seed data.
 *   3. CI re-runs; the judge issues ACCEPTED once invariants are restored.
 */
import test from 'node:test';
import assert from 'node:assert/strict';
import request from 'supertest';
import { createTestApp, loginToken } from './helpers.js';

const ANTHROPIC_API_KEY = process.env.ANTHROPIC_API_KEY;
const JUDGE_MODEL = 'claude-haiku-4-5-20251001';

interface JudgeVerdict {
  verdict: 'ACCEPTED' | 'REJECTED';
  reason: string;
}

async function callLlmJudge(prompt: string): Promise<JudgeVerdict> {
  const response = await fetch('https://api.anthropic.com/v1/messages', {
    method: 'POST',
    headers: {
      'x-api-key': ANTHROPIC_API_KEY!,
      'anthropic-version': '2023-06-01',
      'content-type': 'application/json',
    },
    body: JSON.stringify({
      model: JUDGE_MODEL,
      max_tokens: 512,
      temperature: 0,
      messages: [{ role: 'user', content: prompt }],
    }),
  });

  if (!response.ok) {
    const body = await response.text();
    throw new Error(`Anthropic API error ${response.status}: ${body}`);
  }

  const result = await response.json() as { content: Array<{ type: string; text: string }> };
  const text = result.content.find((c) => c.type === 'text')?.text ?? '';

  // Extract JSON object from the model response (model may wrap it in prose)
  const match = text.match(/\{[\s\S]*"verdict"[\s\S]*\}/);
  if (!match) {
    throw new Error(`LLM judge returned unparseable response:\n${text}`);
  }
  return JSON.parse(match[0]) as JudgeVerdict;
}

function buildJudgePrompt(label: string, responseJson: unknown): string {
  const serialized = JSON.stringify(responseJson, null, 2);
  return `You are a strict quality gate for an OKR (Objectives and Key Results) management API.

Evaluate the API response below for the endpoint: ${label}

Check ALL of the following invariants:
1. \`success\` field is exactly boolean \`true\`.
2. \`data\` exists and is a non-empty array (for list endpoints) or a non-null object (for detail endpoints).
3. For list responses: \`meta.total\` equals \`data.length\`.
4. Each objective has: id (positive integer), title (non-empty string), quarter (format Q[1-4]/YYYY), status (one of NOT_STARTED, IN_PROGRESS, COMPLETED), keyResults (array).
5. Each keyResult has: id (positive integer), title (non-empty string), progress (integer 0-100 inclusive).
6. Status-progress invariant — for each objective:
   - If ALL keyResults have progress === 0 → status MUST be NOT_STARTED.
   - If ALL keyResults have progress === 100 → status MUST be COMPLETED.
   - Otherwise → status MUST be IN_PROGRESS.
7. Titles must NOT contain placeholder or test noise (e.g. "undefined", "null", "TODO", "string", "untitled", "test objective").

API response:
\`\`\`json
${serialized}
\`\`\`

Respond with a JSON object ONLY — no prose before or after:
{"verdict": "ACCEPTED" or "REJECTED", "reason": "<one sentence explaining pass or the first failing invariant>"}`;
}

// Skip gracefully when the API key is absent
const skip = !ANTHROPIC_API_KEY && 'Set ANTHROPIC_API_KEY to enable the LLM-judge gate';

test(
  'LLM judge: manager objective list passes all semantic invariants',
  { skip },
  async () => {
    const { app } = await createTestApp();
    let verdict: JudgeVerdict;
    try {
      const token = await loginToken(app, 'manager');
      const res = await request(app.getHttpServer())
        .get('/api/v1/objectives')
        .set('Authorization', `Bearer ${token}`)
        .expect(200);

      verdict = await callLlmJudge('GET /api/v1/objectives (manager)', res.body);
    } finally {
      await app.close();
    }

    if (verdict!.verdict === 'REJECTED') {
      assert.fail(`LLM judge REJECTED manager objective list: ${verdict!.reason}`);
    }
  },
);

test(
  'LLM judge: employee objective list passes all semantic invariants',
  { skip },
  async () => {
    const { app } = await createTestApp();
    let verdict: JudgeVerdict;
    try {
      const token = await loginToken(app, 'employee');
      const res = await request(app.getHttpServer())
        .get('/api/v1/objectives')
        .set('Authorization', `Bearer ${token}`)
        .expect(200);

      verdict = await callLlmJudge('GET /api/v1/objectives (employee)', res.body);
    } finally {
      await app.close();
    }

    if (verdict!.verdict === 'REJECTED') {
      assert.fail(`LLM judge REJECTED employee objective list: ${verdict!.reason}`);
    }
  },
);

test(
  'LLM judge: objective detail passes all semantic invariants',
  { skip },
  async () => {
    const { app } = await createTestApp();
    let verdict: JudgeVerdict;
    try {
      const token = await loginToken(app, 'manager');
      const res = await request(app.getHttpServer())
        .get('/api/v1/objectives/1')
        .set('Authorization', `Bearer ${token}`)
        .expect(200);

      // Normalize: present as a single-item list so the judge uses the same invariant set
      const normalized = {
        success: res.body.success,
        data: [res.body.data],
        meta: { total: 1 },
      };
      verdict = await callLlmJudge('GET /api/v1/objectives/1 (detail, normalized to list form)', normalized);
    } finally {
      await app.close();
    }

    if (verdict!.verdict === 'REJECTED') {
      assert.fail(`LLM judge REJECTED objective detail: ${verdict!.reason}`);
    }
  },
);