feat(model): implement OpenAI/Anthropic cloud backends + cloud-aware judge gate
The cloud branch of model-call.py was a stub (cloud_backend_not_implemented, failed even with a key set); casan-step.mjs gated the judge on a hard-coded Ollama ping. Wire up the real cloud path so a model can run without Ollama. - model-call.py: add call_openai() and call_anthropic() (raw urllib, no new dependency — matches the existing call_ollama). Endpoints hard-pinned to the SSRF allowlist; keys read from env, never logged. Anthropic sends no temperature/thinking (rejected as 400 on Opus 4.8/4.7; omitting thinking keeps the terse one-word classify/judge answer). main() routes by ollama:/openai:/anthropic: prefix; key-unset still fails closed honestly. provider-usage.jsonl cost_source is per-backend, keeping ollama's exact "ollama_local_real_tokens" tag that evidence/tests key on. - casan-step.mjs: ollamaAvailable() -> modelAvailable() — when CASAN_MODEL_PRIMARY is a cloud spec with its key set, the judge runs through the cloud path; otherwise it pings local Ollama as before. Default (unset CASAN_MODEL_PRIMARY) is unchanged. - CASAN_MASTER_RUNBOOK.md: update sections 0/1/4/7/8 — cloud is now implemented (not a stub); keep the honest "untested with a real key" + CA-cert caveats. Not verified against a live API key (none available); confirmed key-set makes a real HTTPS call and key-unset fails closed. Gates unchanged: security-gate PASS=11 FAIL=0, adversarial PASS=44 FAIL=0. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
thanhnv
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
c08d119381
commit
392f190b7e
@@ -118,6 +118,87 @@ def call_ollama(model_name, prompt, role):
|
||||
}
|
||||
|
||||
|
||||
def call_openai(model_name, prompt, role):
|
||||
# Endpoint hard-pinned to the allowlisted host (no env override) — same SSRF
|
||||
# posture as call_ollama. Key read from env; never logged.
|
||||
host = "api.openai.com"
|
||||
if host not in ALLOWED_CLOUD:
|
||||
fail(f"endpoint_not_allowed openai host={host}")
|
||||
key = os.environ["OPENAI_API_KEY"]
|
||||
url = f"https://{host}/v1/chat/completions"
|
||||
body = {
|
||||
"model": model_name,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"temperature": 0 if role in ("classify", "judge") else 0.2,
|
||||
"max_tokens": 16 if role in ("classify", "judge") else 512,
|
||||
}
|
||||
data = json.dumps(body).encode()
|
||||
req = urllib.request.Request(
|
||||
url, data=data,
|
||||
headers={"Content-Type": "application/json", "Authorization": f"Bearer {key}"},
|
||||
)
|
||||
t0 = time.time()
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=180) as resp:
|
||||
payload = json.loads(resp.read().decode())
|
||||
except Exception as exc: # honest non-zero, no fake success
|
||||
fail(f"backend_unreachable {type(exc).__name__}: {str(exc)[:120]}")
|
||||
latency_ms = int((time.time() - t0) * 1000)
|
||||
usage = payload.get("usage", {})
|
||||
return {
|
||||
"text": (payload["choices"][0]["message"]["content"] or "").strip(),
|
||||
"input_tokens": int(usage.get("prompt_tokens", 0)),
|
||||
"output_tokens": int(usage.get("completion_tokens", 0)),
|
||||
"latency_ms": latency_ms,
|
||||
}
|
||||
|
||||
|
||||
def call_anthropic(model_name, prompt, role):
|
||||
# Endpoint hard-pinned to the allowlisted host (no env override). NOTE: on
|
||||
# current Claude models (Opus 4.8/4.7, Sonnet 5, ...) `temperature`/`top_p`
|
||||
# are rejected with 400 and omitting `thinking` runs without thinking — so
|
||||
# we send neither, which also keeps the terse one-word classify/judge answer
|
||||
# from being eaten by reasoning tokens. Key read from env; never logged.
|
||||
host = "api.anthropic.com"
|
||||
if host not in ALLOWED_CLOUD:
|
||||
fail(f"endpoint_not_allowed anthropic host={host}")
|
||||
key = os.environ["ANTHROPIC_API_KEY"]
|
||||
url = f"https://{host}/v1/messages"
|
||||
body = {
|
||||
"model": model_name,
|
||||
"max_tokens": 16 if role in ("classify", "judge") else 512,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
}
|
||||
data = json.dumps(body).encode()
|
||||
req = urllib.request.Request(
|
||||
url, data=data,
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"x-api-key": key,
|
||||
"anthropic-version": "2023-06-01",
|
||||
},
|
||||
)
|
||||
t0 = time.time()
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=180) as resp:
|
||||
payload = json.loads(resp.read().decode())
|
||||
except Exception as exc: # honest non-zero, no fake success
|
||||
fail(f"backend_unreachable {type(exc).__name__}: {str(exc)[:120]}")
|
||||
latency_ms = int((time.time() - t0) * 1000)
|
||||
# content is a list of blocks; concatenate text blocks. A safety refusal
|
||||
# (stop_reason=="refusal") yields empty text -> extract_verdict fails closed.
|
||||
text = "".join(
|
||||
b.get("text", "") for b in payload.get("content", []) if b.get("type") == "text"
|
||||
).strip()
|
||||
usage = payload.get("usage", {})
|
||||
return {
|
||||
"text": text,
|
||||
"input_tokens": int(usage.get("input_tokens", 0)),
|
||||
"output_tokens": int(usage.get("output_tokens", 0)),
|
||||
"latency_ms": latency_ms,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("prompt_file")
|
||||
@@ -134,17 +215,21 @@ def main():
|
||||
if model_spec.startswith("ollama:"):
|
||||
backend, model_name = "ollama", model_spec[len("ollama:"):]
|
||||
elif model_spec.startswith(("anthropic:", "openai:")):
|
||||
backend = model_spec.split(":", 1)[0]
|
||||
backend, model_name = model_spec.split(":", 1)
|
||||
key = os.environ.get("ANTHROPIC_API_KEY" if backend == "anthropic" else "OPENAI_API_KEY", "")
|
||||
if not key:
|
||||
# honest: cloud backend unavailable while key unset (do NOT fake)
|
||||
fail(f"cloud_backend_unavailable {backend} (API key unset)")
|
||||
fail(f"cloud_backend_not_implemented_in_wave1 {backend}") # no key here anyway
|
||||
else:
|
||||
fail(f"unknown_model_spec {model_spec}")
|
||||
|
||||
prompt = build_prompt(args.role, content)
|
||||
result = call_ollama(model_name, prompt, args.role)
|
||||
if backend == "ollama":
|
||||
result = call_ollama(model_name, prompt, args.role)
|
||||
elif backend == "openai":
|
||||
result = call_openai(model_name, prompt, args.role)
|
||||
else: # anthropic
|
||||
result = call_anthropic(model_name, prompt, args.role)
|
||||
|
||||
verdict, malformed = extract_verdict(args.role, result["text"])
|
||||
total = result["input_tokens"] + result["output_tokens"]
|
||||
@@ -168,14 +253,21 @@ def main():
|
||||
os.makedirs(os.path.dirname(args.out_json) or ".", exist_ok=True)
|
||||
open(args.out_json, "w", encoding="utf-8").write(json.dumps(out, indent=2) + "\n")
|
||||
|
||||
# Append REAL usage telemetry (local = $0 cost, but real token counts).
|
||||
# Append REAL usage telemetry with real token counts. cost_source is
|
||||
# per-backend so cloud tokens are not mislabeled as local (ollama keeps its
|
||||
# exact "ollama_local_real_tokens" tag that evidence/tests key on).
|
||||
cost_source = {
|
||||
"ollama": "ollama_local_real_tokens",
|
||||
"openai": "openai_api_real_tokens",
|
||||
"anthropic": "anthropic_api_real_tokens",
|
||||
}.get(backend, f"{backend}_real_tokens")
|
||||
os.makedirs(os.path.dirname(PROVIDER_LOG), exist_ok=True)
|
||||
usage = {
|
||||
"timestamp": ts, "harness": "L5-provider-telemetry", "provider": backend,
|
||||
"model": model_name, "run_id": os.environ.get("CASAN_RUN_ID", "adhoc"),
|
||||
"step": os.environ.get("CASAN_STEP_NAME", args.role), "role": args.role,
|
||||
"input_tokens": result["input_tokens"], "output_tokens": result["output_tokens"],
|
||||
"total_tokens": total, "cost_usd": 0.0, "cost_source": "ollama_local_real_tokens",
|
||||
"total_tokens": total, "cost_usd": 0.0, "cost_source": cost_source,
|
||||
"latency_ms": result["latency_ms"], "status": "success",
|
||||
}
|
||||
open(PROVIDER_LOG, "a", encoding="utf-8").write(json.dumps(usage) + "\n")
|
||||
|
||||
@@ -19,7 +19,15 @@ try {
|
||||
/* standalone copy: keep silent */
|
||||
}
|
||||
|
||||
function ollamaAvailable() {
|
||||
// Is a model backend reachable for the judge gate? Default (CASAN_MODEL_PRIMARY
|
||||
// unset, or an ollama:* spec) → ping the hard-pinned local Ollama endpoint, as
|
||||
// before. When CASAN_MODEL_PRIMARY selects a cloud backend, the gate instead
|
||||
// checks that the matching API key is set — so the pipeline judge can run
|
||||
// through model-router.sh → model-call.py's cloud path without needing Ollama.
|
||||
function modelAvailable() {
|
||||
const spec = process.env.CASAN_MODEL_PRIMARY || 'ollama:ornith:9b';
|
||||
if (spec.startsWith('openai:')) return Boolean(process.env.OPENAI_API_KEY);
|
||||
if (spec.startsWith('anthropic:')) return Boolean(process.env.ANTHROPIC_API_KEY);
|
||||
try {
|
||||
const r = spawnSync('curl', ['-sS', '-m', '3', 'http://127.0.0.1:11434/api/tags'], { timeout: 5000 });
|
||||
return r.status === 0;
|
||||
@@ -29,9 +37,9 @@ function ollamaAvailable() {
|
||||
}
|
||||
|
||||
function judgeArtifact(filePath, criteria) {
|
||||
if (!ollamaAvailable()) {
|
||||
logDebug(`judge skipped (ollama_unavailable) artifact=${filePath}`);
|
||||
return { verdict: 'SKIP', note: 'ollama_unavailable' };
|
||||
if (!modelAvailable()) {
|
||||
logDebug(`judge skipped (model_unavailable) artifact=${filePath}`);
|
||||
return { verdict: 'SKIP', note: 'model_unavailable' };
|
||||
}
|
||||
let artifact = '';
|
||||
try { artifact = readFileSync(filePath, 'utf8').slice(0, 2000); } catch { return { verdict: 'SKIP', note: 'artifact_unreadable' }; }
|
||||
|
||||
Reference in New Issue
Block a user