feat: updade workspace

apps/okr/backend/test/llm-judge.test.ts

@@ -0,0 +1,165 @@
+/**
+ * LLM-judge gate: sends live API responses to Claude and fails if the model
+ * returns REJECTED. This catches semantic breakage that structural assertions miss
+ * (e.g. scrambled titles, impossible status/progress combinations, data leakage).
+ *
+ * Requires ANTHROPIC_API_KEY to be set. The test is skipped — not failed — when
+ * the key is absent, so local dev without a key still passes CI.
+ *
+ * REJECTED→fix cycle:
+ *   1. CI logs the judge verdict and reason.
+ *   2. Developer reads the reason, fixes the business logic or seed data.
+ *   3. CI re-runs; the judge issues ACCEPTED once invariants are restored.
+ */
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import request from 'supertest';
+import { createTestApp, loginToken } from './helpers.js';
+
+const ANTHROPIC_API_KEY = process.env.ANTHROPIC_API_KEY;
+const JUDGE_MODEL = 'claude-haiku-4-5-20251001';
+
+interface JudgeVerdict {
+  verdict: 'ACCEPTED' | 'REJECTED';
+  reason: string;
+}
+
+async function callLlmJudge(prompt: string): Promise<JudgeVerdict> {
+  const response = await fetch('https://api.anthropic.com/v1/messages', {
+    method: 'POST',
+    headers: {
+      'x-api-key': ANTHROPIC_API_KEY!,
+      'anthropic-version': '2023-06-01',
+      'content-type': 'application/json',
+    },
+    body: JSON.stringify({
+      model: JUDGE_MODEL,
+      max_tokens: 512,
+      temperature: 0,
+      messages: [{ role: 'user', content: prompt }],
+    }),
+  });
+
+  if (!response.ok) {
+    const body = await response.text();
+    throw new Error(`Anthropic API error ${response.status}: ${body}`);
+  }
+
+  const result = await response.json() as { content: Array<{ type: string; text: string }> };
+  const text = result.content.find((c) => c.type === 'text')?.text ?? '';
+
+  // Extract JSON object from the model response (model may wrap it in prose)
+  const match = text.match(/\{[\s\S]*"verdict"[\s\S]*\}/);
+  if (!match) {
+    throw new Error(`LLM judge returned unparseable response:\n${text}`);
+  }
+  return JSON.parse(match[0]) as JudgeVerdict;
+}
+
+function buildJudgePrompt(label: string, responseJson: unknown): string {
+  const serialized = JSON.stringify(responseJson, null, 2);
+  return `You are a strict quality gate for an OKR (Objectives and Key Results) management API.
+
+Evaluate the API response below for the endpoint: ${label}
+
+Check ALL of the following invariants:
+1. \`success\` field is exactly boolean \`true\`.
+2. \`data\` exists and is a non-empty array (for list endpoints) or a non-null object (for detail endpoints).
+3. For list responses: \`meta.total\` equals \`data.length\`.
+4. Each objective has: id (positive integer), title (non-empty string), quarter (format Q[1-4]/YYYY), status (one of NOT_STARTED, IN_PROGRESS, COMPLETED), keyResults (array).
+5. Each keyResult has: id (positive integer), title (non-empty string), progress (integer 0-100 inclusive).
+6. Status-progress invariant — for each objective:
+   - If ALL keyResults have progress === 0 → status MUST be NOT_STARTED.
+   - If ALL keyResults have progress === 100 → status MUST be COMPLETED.
+   - Otherwise → status MUST be IN_PROGRESS.
+7. Titles must NOT contain placeholder or test noise (e.g. "undefined", "null", "TODO", "string", "untitled", "test objective").
+
+API response:
+\`\`\`json
+${serialized}
+\`\`\`
+
+Respond with a JSON object ONLY — no prose before or after:
+{"verdict": "ACCEPTED" or "REJECTED", "reason": "<one sentence explaining pass or the first failing invariant>"}`;
+}
+
+// Skip gracefully when the API key is absent
+const skip = !ANTHROPIC_API_KEY && 'Set ANTHROPIC_API_KEY to enable the LLM-judge gate';
+
+test(
+  'LLM judge: manager objective list passes all semantic invariants',
+  { skip },
+  async () => {
+    const { app } = await createTestApp();
+    let verdict: JudgeVerdict;
+    try {
+      const token = await loginToken(app, 'manager');
+      const res = await request(app.getHttpServer())
+        .get('/api/v1/objectives')
+        .set('Authorization', `Bearer ${token}`)
+        .expect(200);
+
+      verdict = await callLlmJudge('GET /api/v1/objectives (manager)', res.body);
+    } finally {
+      await app.close();
+    }
+
+    if (verdict!.verdict === 'REJECTED') {
+      assert.fail(`LLM judge REJECTED manager objective list: ${verdict!.reason}`);
+    }
+  },
+);
+
+test(
+  'LLM judge: employee objective list passes all semantic invariants',
+  { skip },
+  async () => {
+    const { app } = await createTestApp();
+    let verdict: JudgeVerdict;
+    try {
+      const token = await loginToken(app, 'employee');
+      const res = await request(app.getHttpServer())
+        .get('/api/v1/objectives')
+        .set('Authorization', `Bearer ${token}`)
+        .expect(200);
+
+      verdict = await callLlmJudge('GET /api/v1/objectives (employee)', res.body);
+    } finally {
+      await app.close();
+    }
+
+    if (verdict!.verdict === 'REJECTED') {
+      assert.fail(`LLM judge REJECTED employee objective list: ${verdict!.reason}`);
+    }
+  },
+);
+
+test(
+  'LLM judge: objective detail passes all semantic invariants',
+  { skip },
+  async () => {
+    const { app } = await createTestApp();
+    let verdict: JudgeVerdict;
+    try {
+      const token = await loginToken(app, 'manager');
+      const res = await request(app.getHttpServer())
+        .get('/api/v1/objectives/1')
+        .set('Authorization', `Bearer ${token}`)
+        .expect(200);
+
+      // Normalize: present as a single-item list so the judge uses the same invariant set
+      const normalized = {
+        success: res.body.success,
+        data: [res.body.data],
+        meta: { total: 1 },
+      };
+      verdict = await callLlmJudge('GET /api/v1/objectives/1 (detail, normalized to list form)', normalized);
+    } finally {
+      await app.close();
+    }
+
+    if (verdict!.verdict === 'REJECTED') {
+      assert.fail(`LLM judge REJECTED objective detail: ${verdict!.reason}`);
+    }
+  },
+);