CASAN/AINative_OKR_CASAN5/packages/casan-harness/scripts/bash/provider-usage-fetch.sh

#!/usr/bin/env bash
set -uo pipefail

# CASAN H6 — provider usage telemetry API fetch (D2).
# Pulls usage records from a provider usage HTTP API (production: the
# OpenAI/Anthropic usage endpoints) and imports them into provider-usage.jsonl
# through the same schema gate as import-provider-telemetry.sh.
# Fail-loud by design: unreachable API or invalid schema => non-zero exit and
# NOTHING is imported (all-or-nothing, no partial/dirty telemetry).
#
# Usage: provider-usage-fetch.sh <api-url> [out-jsonl]
#
# Greppable outputs:
#   PROVIDER_TELEMETRY_FETCHED | PROVIDER_USAGE_INVALID | PROVIDER_API_UNREACHABLE

API_URL="${1:-}"
if [[ -z "$API_URL" ]]; then
  echo "Usage: provider-usage-fetch.sh <api-url> [out-jsonl]" >&2
  exit 64
fi

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/casan-paths.sh"
PROJECT_ROOT="$CASAN_APP_ROOT"
OUT="${2:-$CASAN_STATE_ROOT/logs/level5/provider-usage.jsonl}"
mkdir -p "$(dirname "$OUT")"

# SEC-13 (M-09): SSRF guard on the fetch URL. ALWAYS reject non-http(s) schemes
# (file://, gopher://, dict://, … metadata exfil). In enforced mode additionally
# require the host to be in the provider allowlist and block internal/link-local
# IPs — dev keeps loopback mocks working (http://127.0.0.1 test servers).
if ! python3 - "$API_URL" <<'PY'
import ipaddress, os, sys
from urllib.parse import urlparse

url = sys.argv[1]
u = urlparse(url)
scheme = (u.scheme or "").lower()
host = (u.hostname or "").lower()
enforced = os.environ.get("CASAN_PROFILE") == "prod" or os.environ.get("CASAN_SSRF_STRICT") == "1"
allow = [h.strip().lower() for h in os.environ.get(
    "CASAN_PROVIDER_HOST_ALLOWLIST", "api.openai.com,api.anthropic.com").split(",") if h.strip()]

def die(reason):
    sys.stderr.write(f"PROVIDER_URL_REJECTED {reason} url={url}\n")
    sys.exit(1)

if scheme not in ("http", "https"):
    die(f"scheme_not_allowed:{scheme or 'none'}")   # blocks file:// et al (all modes)
if not host:
    die("no_host")

if enforced:
    if scheme != "https":
        die("plaintext_http_not_allowed_in_prod")
    if host in ("localhost",) or host.endswith(".internal") or host.endswith(".local"):
        die(f"internal_host:{host}")
    try:
        ip = ipaddress.ip_address(host)
        if (ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved
                or ip.is_multicast or ip.is_unspecified):
            die(f"internal_ip:{host}")
    except ValueError:
        pass  # a hostname, not a literal IP
    if allow and host not in allow:
        die(f"host_not_in_allowlist:{host}")
sys.exit(0)
PY
then
  echo "PROVIDER_API_SSRF_BLOCKED url=$API_URL (telemetry NOT imported)" >&2
  exit 1
fi

BODY="$(mktemp)"
trap 'rm -f "$BODY"' EXIT

if ! curl -sS -m 10 --retry 2 --retry-delay 1 -f "$API_URL" -o "$BODY" 2>/dev/null; then
  echo "PROVIDER_API_UNREACHABLE url=$API_URL (telemetry NOT imported)" >&2
  exit 1
fi

python - "$BODY" "$OUT" "$API_URL" <<'PY'
import json
import sys
from datetime import datetime, timezone

src, out, url = sys.argv[1], sys.argv[2], sys.argv[3]
try:
    data = json.load(open(src, encoding="utf-8"))
except (json.JSONDecodeError, UnicodeDecodeError):
    raise SystemExit("PROVIDER_USAGE_INVALID response is not JSON")
records = data if isinstance(data, list) else [data]
required = ["provider", "model", "run_id", "step", "input_tokens", "output_tokens",
            "total_tokens", "cost_usd", "latency_ms", "status"]
for idx, rec in enumerate(records):
    if not isinstance(rec, dict):
        raise SystemExit(f"PROVIDER_USAGE_INVALID record[{idx}] is not an object")
    missing = [key for key in required if key not in rec]
    if missing:
        raise SystemExit(f"PROVIDER_USAGE_INVALID record[{idx}] missing={missing}")
now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
with open(out, "a", encoding="utf-8") as fh:
    for rec in records:
        fh.write(json.dumps({
            "timestamp": now,
            "harness": "L5-provider-telemetry",
            "telemetry_source": "provider_api",
            "api_endpoint": url,
            **rec,
        }) + "\n")
print(f"PROVIDER_TELEMETRY_FETCHED count={len(records)} url={url} output={out}")
PY
RC=$?
if [[ "$RC" -ne 0 ]]; then
  echo "PROVIDER_USAGE_INVALID import rejected (nothing written)" >&2
  exit 1
fi