CASAN/packages/casan-harness/scripts/bash/agent-metrics.sh

#!/usr/bin/env bash
set -euo pipefail

# CASAN H6 AgentOps Harness
# Usage:
#   agent-metrics.sh <input-file> <output-file> [-- <command> ...]
#
# If command is omitted, the script performs a pass-through copy. If command is
# provided, it runs with CASAN_INPUT and CASAN_OUTPUT environment variables.

INPUT_FILE="${1:-}"
OUTPUT_FILE="${2:-}"
shift 2 || true
if [[ "${1:-}" == "--" ]]; then
  shift
fi

if [[ -z "$INPUT_FILE" || -z "$OUTPUT_FILE" ]]; then
  echo "Usage: agent-metrics.sh <input-file> <output-file> [-- <command> ...]" >&2
  exit 64
fi

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/casan-paths.sh"
PROJECT_ROOT="$CASAN_APP_ROOT"
LOG_DIR="$CASAN_STATE_ROOT/logs"
TRACE_DIR="$LOG_DIR/trace"
# SEC-23 (MT-01): telemetry dir is tenant-scoped when CASAN_METRICS_DIR is set
# (tenant-paths.sh exports it per tenant); default is the shared path.
METRICS_DIR="${CASAN_METRICS_DIR:-$(dirname "$CASAN_TELEMETRY_METRICS_LOG")}"
ALERT_LOG="$CASAN_TELEMETRY_ALERTS_LOG"
METRICS_LOG="${CASAN_METRICS_LOG:-$CASAN_TELEMETRY_METRICS_LOG}"
mkdir -p "$TRACE_DIR" "$METRICS_DIR" "$(dirname "$OUTPUT_FILE")" "$(dirname "$ALERT_LOG")"

# shellcheck source=tool-audit-lib.sh
source "$SCRIPT_DIR/tool-audit-lib.sh"

if [[ ! -f "$INPUT_FILE" ]]; then
  echo "AGENTOPS_FAILED: input file not found: $INPUT_FILE" >&2
  exit 1
fi

timestamp() {
  date -u +"%Y-%m-%dT%H:%M:%SZ"
}

epoch_ms() {
  python -c 'import time; print(int(time.time() * 1000))' 2>/dev/null || printf '%s000\n' "$(date +%s)"
}

new_trace_id() {
  if command -v uuidgen >/dev/null 2>&1; then
    uuidgen | tr '[:upper:]' '[:lower:]'
  else
    printf 'trace-%s-%s\n' "$(date +%s)" "$$"
  fi
}

hash_text() {
  if command -v sha256sum >/dev/null 2>&1; then
    sha256sum | awk '{print $1}'
  else
    shasum -a 256 | awk '{print $1}'
  fi
}

word_count() {
  wc -w < "$1" | tr -d ' '
}

TRACE_ID="$(new_trace_id)"
START_TS="$(timestamp)"
START_MS="$(epoch_ms)"
STATUS="success"
ERROR_MSG=""
EXIT_CODE=0
RETRY_COUNT="${CASAN_RETRY_COUNT:-0}"
AGENT_NAME="${CASAN_AGENT_NAME:-unknown-agent}"
STEP_NAME="${CASAN_STEP_NAME:-unknown-step}"

INPUT_TOKENS="$(word_count "$INPUT_FILE")"

if [[ "$#" -gt 0 ]]; then
  set +e
  CASAN_INPUT="$INPUT_FILE" CASAN_OUTPUT="$OUTPUT_FILE" "$@"
  EXIT_CODE=$?
  set -e
  if [[ "$EXIT_CODE" -ne 0 ]]; then
    STATUS="failed"
    ERROR_MSG="command exited with code $EXIT_CODE"
  fi

  TOOL_AUDIT_RECORD="$(python - "$START_TS" "$TRACE_ID" "$AGENT_NAME" "$STEP_NAME" "$*" "$EXIT_CODE" "$STATUS" <<'PY'
import json, sys
ts, trace, agent, step, cmd, code, status = sys.argv[1:]
print(json.dumps({
    "timestamp": ts, "trace_id": trace, "agent": agent, "step": step,
    "tool": "Bash", "command": cmd, "exit_code": int(code), "status": status,
}))
PY
)"
  append_tool_audit "$TOOL_AUDIT_RECORD" "$PROJECT_ROOT"
else
  cp "$INPUT_FILE" "$OUTPUT_FILE"
fi

END_MS="$(epoch_ms)"
LATENCY_MS=$((END_MS - START_MS))

if [[ ! -f "$OUTPUT_FILE" ]]; then
  STATUS="failed"
  ERROR_MSG="${ERROR_MSG:-output file not produced}"
  # A zero command exit does not make the step successful when the runtime
  # contract requires an output artifact and none was produced. Telemetry can
  # record this failure successfully, but must propagate a failed step outcome.
  [[ "$EXIT_CODE" -eq 0 ]] && EXIT_CODE=1
  : > "$OUTPUT_FILE"
fi

OUTPUT_TOKENS="$(word_count "$OUTPUT_FILE")"
TOTAL_TOKENS=$((INPUT_TOKENS + OUTPUT_TOKENS))
COST_PER_1K="${CASAN_COST_PER_1K:-0.002}"
COST_ESTIMATE="$(python - "$TOTAL_TOKENS" "$COST_PER_1K" <<'PY'
import sys
tokens = int(sys.argv[1])
rate = float(sys.argv[2])
print(f"{tokens * rate / 1000:.8f}")
PY
)"
COST_SOURCE="word_count_estimate"

# Prefer real provider usage when telemetry has been imported; the word-count
# figure above is an explicit fallback, not presented as a real billed cost.
PROVIDER_LOG="$CASAN_TELEMETRY_PROVIDER_LOG"
if [[ -f "$PROVIDER_LOG" ]] && command -v python >/dev/null 2>&1; then
  # Use real provider telemetry ONLY when a record genuinely matches this step.
  # Do NOT fall back to an arbitrary record (that would reuse one sample's cost
  # across every step and misrepresent it as real per-step billing).
  PROV="$(python "$SCRIPT_DIR/provider-cost-lookup.py" "$PROVIDER_LOG" "$STEP_NAME")"
  if [[ -n "$PROV" ]]; then
    TOTAL_TOKENS="${PROV%% *}"
    COST_ESTIMATE="${PROV##* }"
    COST_SOURCE="provider_telemetry"
  fi
fi

# Real hallucination-signal detection (populates hallucination-tracking.yaml's metric).
HALLU_YAML="$CASAN_HARNESS_ROOT/agentops/hallucination-tracking.yaml"
HALLUCINATION_SIGNALS=0
HALLUCINATION_MATCHED="[]"
if command -v python >/dev/null 2>&1; then
  HSCAN="$(python "$SCRIPT_DIR/hallucination-scan.py" "$HALLU_YAML" "$OUTPUT_FILE" 2>/dev/null || printf '0\n[]')"
  HALLUCINATION_SIGNALS="$(printf '%s' "$HSCAN" | head -1)"
  HALLUCINATION_MATCHED="$(printf '%s' "$HSCAN" | tail -1)"
fi

INPUT_HASH="$(cat "$INPUT_FILE" | hash_text)"
OUTPUT_HASH="$(cat "$OUTPUT_FILE" | hash_text)"

ALERTS=()
if [[ "$LATENCY_MS" -gt "${CASAN_LATENCY_ALERT_MS:-5000}" ]]; then
  ALERTS+=("high-latency")
fi
if [[ "$RETRY_COUNT" -gt "${CASAN_RETRY_ALERT_THRESHOLD:-2}" ]]; then
  ALERTS+=("high-retry")
fi
if [[ "$STATUS" == "failed" ]]; then
  ALERTS+=("execution-failed")
fi
if [[ "$TOTAL_TOKENS" -gt "${CASAN_TOKEN_ALERT_THRESHOLD:-5000}" ]]; then
  ALERTS+=("token-overuse")
fi
if [[ "$HALLUCINATION_SIGNALS" -ge "${CASAN_HALLUCINATION_WARN:-3}" ]]; then
  ALERTS+=("hallucination-suspected")
fi

ALERTS_JSON="$(printf '%s\n' "${ALERTS[@]:-}" | python -c 'import json,sys; print(json.dumps([x for x in sys.stdin.read().splitlines() if x]))')"
TRACE_FILE="$TRACE_DIR/agentops-$TRACE_ID.json"
cat > "$TRACE_FILE" <<EOF
{
  "trace_id": "$TRACE_ID",
  "timestamp": "$START_TS",
  "harness": "H6-agentops",
  "agent": "$AGENT_NAME",
  "step": "$STEP_NAME",
  "status": "$STATUS",
  "exit_code": $EXIT_CODE,
  "latency_ms": $LATENCY_MS,
  "retry_count": $RETRY_COUNT,
  "input_tokens": $INPUT_TOKENS,
  "output_tokens": $OUTPUT_TOKENS,
  "total_tokens": $TOTAL_TOKENS,
  "cost_estimate": $COST_ESTIMATE,
  "cost_source": "$COST_SOURCE",
  "hallucination_signals": $HALLUCINATION_SIGNALS,
  "hallucination_matched": $HALLUCINATION_MATCHED,
  "alerts": $ALERTS_JSON,
  "input_hash": "$INPUT_HASH",
  "output_hash": "$OUTPUT_HASH",
  "error": "$ERROR_MSG"
}
EOF

# SEC-05 (M-05): serialize via json.dumps so a crafted AGENT_NAME/STEP_NAME cannot
# inject a forged metrics record, and an empty/non-numeric metric can't emit invalid
# JSON. String fields are quoted safely; numeric fields are coerced (fail-safe 0).
CASAN_AM_HALLU="$HALLUCINATION_SIGNALS" CASAN_AM_ALERTS="$ALERTS_JSON" python - "$METRICS_LOG" \
  "$START_TS" "$TRACE_ID" "$AGENT_NAME" "$STEP_NAME" "$STATUS" "$EXIT_CODE" "$LATENCY_MS" \
  "$RETRY_COUNT" "$INPUT_TOKENS" "$OUTPUT_TOKENS" "$TOTAL_TOKENS" "$COST_ESTIMATE" "$COST_SOURCE" \
  "$INPUT_HASH" "$OUTPUT_HASH" <<'PY'
import json, os, sys
(log, ts, trace_id, agent, step, status, exit_code, latency, retry,
 in_tok, out_tok, tot_tok, cost, cost_source, in_hash, out_hash) = sys.argv[1:]
def num(x):
    try: return int(x)
    except (ValueError, TypeError): pass
    try: return float(x)
    except (ValueError, TypeError): return 0
def jval(s, default):
    try: return json.loads(s)
    except (ValueError, TypeError): return default
rec = {"timestamp": ts, "trace_id": trace_id, "harness": "H6-agentops",
       "agent": agent, "step": step, "status": status,
       "exit_code": num(exit_code), "latency_ms": num(latency), "retry_count": num(retry),
       "input_tokens": num(in_tok), "output_tokens": num(out_tok), "total_tokens": num(tot_tok),
       "cost_estimate": jval(cost, 0), "cost_source": cost_source,
       "hallucination_signals": jval(os.environ.get("CASAN_AM_HALLU"), 0),
       "alerts": jval(os.environ.get("CASAN_AM_ALERTS"), []),
       "input_hash": in_hash, "output_hash": out_hash}
# Compact separators to match the original printf layout (regex-parsed downstream).
open(log, "a", encoding="utf-8").write(json.dumps(rec, separators=(",", ":")) + "\n")
PY

for alert in "${ALERTS[@]:-}"; do
  if [[ -n "$alert" ]]; then
    # SEC-05 (M-05): build the alert record with json.dumps (agent/alert/step raw before).
    ALERT_JSON="$(python - "$START_TS" "$TRACE_ID" "$AGENT_NAME" "$alert" "$STEP_NAME" "$LATENCY_MS" "$STATUS" <<'PY'
import json, sys
ts, trace_id, agent, alert, step, latency, status = sys.argv[1:]
def num(x):
    try: return int(x)
    except (ValueError, TypeError):
        try: return float(x)
        except (ValueError, TypeError): return 0
print(json.dumps({
    "timestamp": ts, "trace_id": trace_id, "severity": "WARN",
    "resource": {"service.name": agent, "service.version": "1.0.0"},
    "body": {"message": f"Alert triggered: {alert}", "alert.type": alert, "step.name": step},
    "attributes": {"latency_ms": num(latency), "status": status},
}, separators=(",", ":")))
PY
)"
    printf '%s\n' "$ALERT_JSON" >> "$ALERT_LOG"
    # Live dispatch (H6-D1): push to the real alert channel when configured.
    # Delivery failure is queued to the dead-letter file by alert-dispatch.sh.
    if [[ -n "${CASAN_ALERT_WEBHOOK:-}" ]]; then
      ALERT_TMP="$(mktemp)"
      printf '%s\n' "$ALERT_JSON" > "$ALERT_TMP"
      bash "$SCRIPT_DIR/alert-dispatch.sh" "$ALERT_TMP" \
        || echo "AGENTOPS_ALERT_DISPATCH_FAILED alert=$alert (queued to dead-letter)" >&2
      rm -f "$ALERT_TMP"
    fi
  fi
done

echo "AGENTOPS_RECORDED trace_id=$TRACE_ID status=$STATUS latency_ms=$LATENCY_MS tokens=$TOTAL_TOKENS cost=$COST_ESTIMATE output=$OUTPUT_FILE"
exit "$EXIT_CODE"