#!/usr/bin/env bash set -euo pipefail # CASAN H6 AgentOps Harness # Usage: # agent-metrics.sh [-- ...] # # If command is omitted, the script performs a pass-through copy. If command is # provided, it runs with CASAN_INPUT and CASAN_OUTPUT environment variables. INPUT_FILE="${1:-}" OUTPUT_FILE="${2:-}" shift 2 || true if [[ "${1:-}" == "--" ]]; then shift fi if [[ -z "$INPUT_FILE" || -z "$OUTPUT_FILE" ]]; then echo "Usage: agent-metrics.sh [-- ...]" >&2 exit 64 fi SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/casan-paths.sh" PROJECT_ROOT="$CASAN_APP_ROOT" LOG_DIR="$CASAN_STATE_ROOT/logs" TRACE_DIR="$LOG_DIR/trace" # SEC-23 (MT-01): telemetry dir is tenant-scoped when CASAN_METRICS_DIR is set # (tenant-paths.sh exports it per tenant); default is the shared path. METRICS_DIR="${CASAN_METRICS_DIR:-$LOG_DIR/cost}" ALERT_LOG="$CASAN_HARNESS_ROOT/agentops/alerts.log" METRICS_LOG="$METRICS_DIR/metrics.jsonl" mkdir -p "$TRACE_DIR" "$METRICS_DIR" "$(dirname "$OUTPUT_FILE")" "$(dirname "$ALERT_LOG")" # shellcheck source=tool-audit-lib.sh source "$SCRIPT_DIR/tool-audit-lib.sh" if [[ ! -f "$INPUT_FILE" ]]; then echo "AGENTOPS_FAILED: input file not found: $INPUT_FILE" >&2 exit 1 fi timestamp() { date -u +"%Y-%m-%dT%H:%M:%SZ" } epoch_ms() { python -c 'import time; print(int(time.time() * 1000))' 2>/dev/null || printf '%s000\n' "$(date +%s)" } new_trace_id() { if command -v uuidgen >/dev/null 2>&1; then uuidgen | tr '[:upper:]' '[:lower:]' else printf 'trace-%s-%s\n' "$(date +%s)" "$$" fi } hash_text() { if command -v sha256sum >/dev/null 2>&1; then sha256sum | awk '{print $1}' else shasum -a 256 | awk '{print $1}' fi } word_count() { wc -w < "$1" | tr -d ' ' } TRACE_ID="$(new_trace_id)" START_TS="$(timestamp)" START_MS="$(epoch_ms)" STATUS="success" ERROR_MSG="" EXIT_CODE=0 RETRY_COUNT="${CASAN_RETRY_COUNT:-0}" AGENT_NAME="${CASAN_AGENT_NAME:-unknown-agent}" STEP_NAME="${CASAN_STEP_NAME:-unknown-step}" INPUT_TOKENS="$(word_count "$INPUT_FILE")" if [[ "$#" -gt 0 ]]; then set +e CASAN_INPUT="$INPUT_FILE" CASAN_OUTPUT="$OUTPUT_FILE" "$@" EXIT_CODE=$? set -e if [[ "$EXIT_CODE" -ne 0 ]]; then STATUS="failed" ERROR_MSG="command exited with code $EXIT_CODE" fi TOOL_AUDIT_RECORD="$(python - "$START_TS" "$TRACE_ID" "$AGENT_NAME" "$STEP_NAME" "$*" "$EXIT_CODE" "$STATUS" <<'PY' import json, sys ts, trace, agent, step, cmd, code, status = sys.argv[1:] print(json.dumps({ "timestamp": ts, "trace_id": trace, "agent": agent, "step": step, "tool": "Bash", "command": cmd, "exit_code": int(code), "status": status, })) PY )" append_tool_audit "$TOOL_AUDIT_RECORD" "$PROJECT_ROOT" else cp "$INPUT_FILE" "$OUTPUT_FILE" fi END_MS="$(epoch_ms)" LATENCY_MS=$((END_MS - START_MS)) if [[ ! -f "$OUTPUT_FILE" ]]; then STATUS="failed" ERROR_MSG="${ERROR_MSG:-output file not produced}" : > "$OUTPUT_FILE" fi OUTPUT_TOKENS="$(word_count "$OUTPUT_FILE")" TOTAL_TOKENS=$((INPUT_TOKENS + OUTPUT_TOKENS)) COST_PER_1K="${CASAN_COST_PER_1K:-0.002}" COST_ESTIMATE="$(python - "$TOTAL_TOKENS" "$COST_PER_1K" <<'PY' import sys tokens = int(sys.argv[1]) rate = float(sys.argv[2]) print(f"{tokens * rate / 1000:.8f}") PY )" COST_SOURCE="word_count_estimate" # Prefer real provider usage when telemetry has been imported; the word-count # figure above is an explicit fallback, not presented as a real billed cost. PROVIDER_LOG="$CASAN_STATE_ROOT/logs/level5/provider-usage.jsonl" if [[ -f "$PROVIDER_LOG" ]] && command -v python >/dev/null 2>&1; then # Use real provider telemetry ONLY when a record genuinely matches this step. # Do NOT fall back to an arbitrary record (that would reuse one sample's cost # across every step and misrepresent it as real per-step billing). PROV="$(python "$SCRIPT_DIR/provider-cost-lookup.py" "$PROVIDER_LOG" "$STEP_NAME")" if [[ -n "$PROV" ]]; then TOTAL_TOKENS="${PROV%% *}" COST_ESTIMATE="${PROV##* }" COST_SOURCE="provider_telemetry" fi fi # Real hallucination-signal detection (populates hallucination-tracking.yaml's metric). HALLU_YAML="$CASAN_HARNESS_ROOT/agentops/hallucination-tracking.yaml" HALLUCINATION_SIGNALS=0 HALLUCINATION_MATCHED="[]" if command -v python >/dev/null 2>&1; then HSCAN="$(python "$SCRIPT_DIR/hallucination-scan.py" "$HALLU_YAML" "$OUTPUT_FILE" 2>/dev/null || printf '0\n[]')" HALLUCINATION_SIGNALS="$(printf '%s' "$HSCAN" | head -1)" HALLUCINATION_MATCHED="$(printf '%s' "$HSCAN" | tail -1)" fi INPUT_HASH="$(cat "$INPUT_FILE" | hash_text)" OUTPUT_HASH="$(cat "$OUTPUT_FILE" | hash_text)" ALERTS=() if [[ "$LATENCY_MS" -gt "${CASAN_LATENCY_ALERT_MS:-5000}" ]]; then ALERTS+=("high-latency") fi if [[ "$RETRY_COUNT" -gt "${CASAN_RETRY_ALERT_THRESHOLD:-2}" ]]; then ALERTS+=("high-retry") fi if [[ "$STATUS" == "failed" ]]; then ALERTS+=("execution-failed") fi if [[ "$TOTAL_TOKENS" -gt "${CASAN_TOKEN_ALERT_THRESHOLD:-5000}" ]]; then ALERTS+=("token-overuse") fi if [[ "$HALLUCINATION_SIGNALS" -ge "${CASAN_HALLUCINATION_WARN:-3}" ]]; then ALERTS+=("hallucination-suspected") fi ALERTS_JSON="$(printf '%s\n' "${ALERTS[@]:-}" | python -c 'import json,sys; print(json.dumps([x for x in sys.stdin.read().splitlines() if x]))')" TRACE_FILE="$TRACE_DIR/agentops-$TRACE_ID.json" cat > "$TRACE_FILE" <> "$ALERT_LOG" # Live dispatch (H6-D1): push to the real alert channel when configured. # Delivery failure is queued to the dead-letter file by alert-dispatch.sh. if [[ -n "${CASAN_ALERT_WEBHOOK:-}" ]]; then ALERT_TMP="$(mktemp)" printf '%s\n' "$ALERT_JSON" > "$ALERT_TMP" bash "$SCRIPT_DIR/alert-dispatch.sh" "$ALERT_TMP" \ || echo "AGENTOPS_ALERT_DISPATCH_FAILED alert=$alert (queued to dead-letter)" >&2 rm -f "$ALERT_TMP" fi fi done echo "AGENTOPS_RECORDED trace_id=$TRACE_ID status=$STATUS latency_ms=$LATENCY_MS tokens=$TOTAL_TOKENS cost=$COST_ESTIMATE output=$OUTPUT_FILE" exit "$EXIT_CODE"