CASAN/AINative_OKR_CASAN5/.specify/scripts/bash/agent-metrics.sh

#!/usr/bin/env bash
set -euo pipefail

# CASAN H6 AgentOps Harness
# Usage:
#   agent-metrics.sh <input-file> <output-file> [-- <command> ...]
#
# If command is omitted, the script performs a pass-through copy. If command is
# provided, it runs with CASAN_INPUT and CASAN_OUTPUT environment variables.

INPUT_FILE="${1:-}"
OUTPUT_FILE="${2:-}"
shift 2 || true
if [[ "${1:-}" == "--" ]]; then
  shift
fi

if [[ -z "$INPUT_FILE" || -z "$OUTPUT_FILE" ]]; then
  echo "Usage: agent-metrics.sh <input-file> <output-file> [-- <command> ...]" >&2
  exit 64
fi

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
LOG_DIR="$PROJECT_ROOT/.specify/logs"
TRACE_DIR="$LOG_DIR/trace"
METRICS_DIR="$LOG_DIR/cost"
ALERT_LOG="$PROJECT_ROOT/.specify/agentops/alerts.log"
METRICS_LOG="$METRICS_DIR/metrics.jsonl"
mkdir -p "$TRACE_DIR" "$METRICS_DIR" "$(dirname "$OUTPUT_FILE")" "$(dirname "$ALERT_LOG")"

# shellcheck source=tool-audit-lib.sh
source "$SCRIPT_DIR/tool-audit-lib.sh"

if [[ ! -f "$INPUT_FILE" ]]; then
  echo "AGENTOPS_FAILED: input file not found: $INPUT_FILE" >&2
  exit 1
fi

timestamp() {
  date -u +"%Y-%m-%dT%H:%M:%SZ"
}

epoch_ms() {
  python -c 'import time; print(int(time.time() * 1000))' 2>/dev/null || printf '%s000\n' "$(date +%s)"
}

new_trace_id() {
  if command -v uuidgen >/dev/null 2>&1; then
    uuidgen | tr '[:upper:]' '[:lower:]'
  else
    printf 'trace-%s-%s\n' "$(date +%s)" "$$"
  fi
}

hash_text() {
  if command -v sha256sum >/dev/null 2>&1; then
    sha256sum | awk '{print $1}'
  else
    shasum -a 256 | awk '{print $1}'
  fi
}

word_count() {
  wc -w < "$1" | tr -d ' '
}

TRACE_ID="$(new_trace_id)"
START_TS="$(timestamp)"
START_MS="$(epoch_ms)"
STATUS="success"
ERROR_MSG=""
EXIT_CODE=0
RETRY_COUNT="${CASAN_RETRY_COUNT:-0}"
AGENT_NAME="${CASAN_AGENT_NAME:-unknown-agent}"
STEP_NAME="${CASAN_STEP_NAME:-unknown-step}"

INPUT_TOKENS="$(word_count "$INPUT_FILE")"

if [[ "$#" -gt 0 ]]; then
  set +e
  CASAN_INPUT="$INPUT_FILE" CASAN_OUTPUT="$OUTPUT_FILE" "$@"
  EXIT_CODE=$?
  set -e
  if [[ "$EXIT_CODE" -ne 0 ]]; then
    STATUS="failed"
    ERROR_MSG="command exited with code $EXIT_CODE"
  fi

  TOOL_AUDIT_RECORD="$(python - "$START_TS" "$TRACE_ID" "$AGENT_NAME" "$STEP_NAME" "$*" "$EXIT_CODE" "$STATUS" <<'PY'
import json, sys
ts, trace, agent, step, cmd, code, status = sys.argv[1:]
print(json.dumps({
    "timestamp": ts, "trace_id": trace, "agent": agent, "step": step,
    "tool": "Bash", "command": cmd, "exit_code": int(code), "status": status,
}))
PY
)"
  append_tool_audit "$TOOL_AUDIT_RECORD" "$PROJECT_ROOT"
else
  cp "$INPUT_FILE" "$OUTPUT_FILE"
fi

END_MS="$(epoch_ms)"
LATENCY_MS=$((END_MS - START_MS))

if [[ ! -f "$OUTPUT_FILE" ]]; then
  STATUS="failed"
  ERROR_MSG="${ERROR_MSG:-output file not produced}"
  : > "$OUTPUT_FILE"
fi

OUTPUT_TOKENS="$(word_count "$OUTPUT_FILE")"
TOTAL_TOKENS=$((INPUT_TOKENS + OUTPUT_TOKENS))
COST_PER_1K="${CASAN_COST_PER_1K:-0.002}"
COST_ESTIMATE="$(python - "$TOTAL_TOKENS" "$COST_PER_1K" <<'PY'
import sys
tokens = int(sys.argv[1])
rate = float(sys.argv[2])
print(f"{tokens * rate / 1000:.8f}")
PY
)"
COST_SOURCE="word_count_estimate"

# Prefer real provider usage when telemetry has been imported; the word-count
# figure above is an explicit fallback, not presented as a real billed cost.
PROVIDER_LOG="$PROJECT_ROOT/.specify/logs/level5/provider-usage.jsonl"
if [[ -f "$PROVIDER_LOG" ]] && command -v python >/dev/null 2>&1; then
  # Use real provider telemetry ONLY when a record genuinely matches this step.
  # Do NOT fall back to an arbitrary record (that would reuse one sample's cost
  # across every step and misrepresent it as real per-step billing).
  PROV="$(python "$SCRIPT_DIR/provider-cost-lookup.py" "$PROVIDER_LOG" "$STEP_NAME")"
  if [[ -n "$PROV" ]]; then
    TOTAL_TOKENS="${PROV%% *}"
    COST_ESTIMATE="${PROV##* }"
    COST_SOURCE="provider_telemetry"
  fi
fi

# Real hallucination-signal detection (populates hallucination-tracking.yaml's metric).
HALLU_YAML="$PROJECT_ROOT/.specify/agentops/hallucination-tracking.yaml"
HALLUCINATION_SIGNALS=0
HALLUCINATION_MATCHED="[]"
if command -v python >/dev/null 2>&1; then
  HSCAN="$(python "$SCRIPT_DIR/hallucination-scan.py" "$HALLU_YAML" "$OUTPUT_FILE" 2>/dev/null || printf '0\n[]')"
  HALLUCINATION_SIGNALS="$(printf '%s' "$HSCAN" | head -1)"
  HALLUCINATION_MATCHED="$(printf '%s' "$HSCAN" | tail -1)"
fi

INPUT_HASH="$(cat "$INPUT_FILE" | hash_text)"
OUTPUT_HASH="$(cat "$OUTPUT_FILE" | hash_text)"

ALERTS=()
if [[ "$LATENCY_MS" -gt "${CASAN_LATENCY_ALERT_MS:-5000}" ]]; then
  ALERTS+=("high-latency")
fi
if [[ "$RETRY_COUNT" -gt "${CASAN_RETRY_ALERT_THRESHOLD:-2}" ]]; then
  ALERTS+=("high-retry")
fi
if [[ "$STATUS" == "failed" ]]; then
  ALERTS+=("execution-failed")
fi
if [[ "$TOTAL_TOKENS" -gt "${CASAN_TOKEN_ALERT_THRESHOLD:-5000}" ]]; then
  ALERTS+=("token-overuse")
fi
if [[ "$HALLUCINATION_SIGNALS" -ge "${CASAN_HALLUCINATION_WARN:-3}" ]]; then
  ALERTS+=("hallucination-suspected")
fi

ALERTS_JSON="$(printf '%s\n' "${ALERTS[@]:-}" | python -c 'import json,sys; print(json.dumps([x for x in sys.stdin.read().splitlines() if x]))')"
TRACE_FILE="$TRACE_DIR/agentops-$TRACE_ID.json"
cat > "$TRACE_FILE" <<EOF
{
  "trace_id": "$TRACE_ID",
  "timestamp": "$START_TS",
  "harness": "H6-agentops",
  "agent": "$AGENT_NAME",
  "step": "$STEP_NAME",
  "status": "$STATUS",
  "exit_code": $EXIT_CODE,
  "latency_ms": $LATENCY_MS,
  "retry_count": $RETRY_COUNT,
  "input_tokens": $INPUT_TOKENS,
  "output_tokens": $OUTPUT_TOKENS,
  "total_tokens": $TOTAL_TOKENS,
  "cost_estimate": $COST_ESTIMATE,
  "cost_source": "$COST_SOURCE",
  "hallucination_signals": $HALLUCINATION_SIGNALS,
  "hallucination_matched": $HALLUCINATION_MATCHED,
  "alerts": $ALERTS_JSON,
  "input_hash": "$INPUT_HASH",
  "output_hash": "$OUTPUT_HASH",
  "error": "$ERROR_MSG"
}
EOF

printf '{"timestamp":"%s","trace_id":"%s","harness":"H6-agentops","agent":"%s","step":"%s","status":"%s","exit_code":%s,"latency_ms":%s,"retry_count":%s,"input_tokens":%s,"output_tokens":%s,"total_tokens":%s,"cost_estimate":%s,"cost_source":"%s","hallucination_signals":%s,"alerts":%s,"input_hash":"%s","output_hash":"%s"}\n' \
  "$START_TS" "$TRACE_ID" "$AGENT_NAME" "$STEP_NAME" "$STATUS" "$EXIT_CODE" "$LATENCY_MS" "$RETRY_COUNT" "$INPUT_TOKENS" "$OUTPUT_TOKENS" "$TOTAL_TOKENS" "$COST_ESTIMATE" "$COST_SOURCE" "$HALLUCINATION_SIGNALS" "$ALERTS_JSON" "$INPUT_HASH" "$OUTPUT_HASH" >> "$METRICS_LOG"

for alert in "${ALERTS[@]:-}"; do
  if [[ -n "$alert" ]]; then
    ALERT_JSON="$(printf '{"timestamp":"%s","trace_id":"%s","severity":"WARN","resource":{"service.name":"%s","service.version":"1.0.0"},"body":{"message":"Alert triggered: %s","alert.type":"%s","step.name":"%s"},"attributes":{"latency_ms":%s,"status":"%s"}}' \
      "$START_TS" "$TRACE_ID" "$AGENT_NAME" "$alert" "$alert" "$STEP_NAME" "$LATENCY_MS" "$STATUS")"
    printf '%s\n' "$ALERT_JSON" >> "$ALERT_LOG"
    # Live dispatch (H6-D1): push to the real alert channel when configured.
    # Delivery failure is queued to the dead-letter file by alert-dispatch.sh.
    if [[ -n "${CASAN_ALERT_WEBHOOK:-}" ]]; then
      ALERT_TMP="$(mktemp)"
      printf '%s\n' "$ALERT_JSON" > "$ALERT_TMP"
      bash "$SCRIPT_DIR/alert-dispatch.sh" "$ALERT_TMP" \
        || echo "AGENTOPS_ALERT_DISPATCH_FAILED alert=$alert (queued to dead-letter)" >&2
      rm -f "$ALERT_TMP"
    fi
  fi
done

echo "AGENTOPS_RECORDED trace_id=$TRACE_ID status=$STATUS latency_ms=$LATENCY_MS tokens=$TOTAL_TOKENS cost=$COST_ESTIMATE output=$OUTPUT_FILE"
exit "$EXIT_CODE"