update first - 84
This commit is contained in:
thanhnv
+208
@@ -0,0 +1,208 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# CASAN H6 AgentOps Harness
|
||||
# Usage:
|
||||
# agent-metrics.sh <input-file> <output-file> [-- <command> ...]
|
||||
#
|
||||
# If command is omitted, the script performs a pass-through copy. If command is
|
||||
# provided, it runs with CASAN_INPUT and CASAN_OUTPUT environment variables.
|
||||
|
||||
INPUT_FILE="${1:-}"
|
||||
OUTPUT_FILE="${2:-}"
|
||||
shift 2 || true
|
||||
if [[ "${1:-}" == "--" ]]; then
|
||||
shift
|
||||
fi
|
||||
|
||||
if [[ -z "$INPUT_FILE" || -z "$OUTPUT_FILE" ]]; then
|
||||
echo "Usage: agent-metrics.sh <input-file> <output-file> [-- <command> ...]" >&2
|
||||
exit 64
|
||||
fi
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
|
||||
LOG_DIR="$PROJECT_ROOT/.specify/logs"
|
||||
TRACE_DIR="$LOG_DIR/trace"
|
||||
METRICS_DIR="$LOG_DIR/cost"
|
||||
ALERT_LOG="$PROJECT_ROOT/.specify/agentops/alerts.log"
|
||||
METRICS_LOG="$METRICS_DIR/metrics.jsonl"
|
||||
mkdir -p "$TRACE_DIR" "$METRICS_DIR" "$(dirname "$OUTPUT_FILE")" "$(dirname "$ALERT_LOG")"
|
||||
|
||||
# shellcheck source=tool-audit-lib.sh
|
||||
source "$SCRIPT_DIR/tool-audit-lib.sh"
|
||||
|
||||
if [[ ! -f "$INPUT_FILE" ]]; then
|
||||
echo "AGENTOPS_FAILED: input file not found: $INPUT_FILE" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
timestamp() {
|
||||
date -u +"%Y-%m-%dT%H:%M:%SZ"
|
||||
}
|
||||
|
||||
epoch_ms() {
|
||||
python3 -c 'import time; print(int(time.time() * 1000))' 2>/dev/null || printf '%s000\n' "$(date +%s)"
|
||||
}
|
||||
|
||||
new_trace_id() {
|
||||
if command -v uuidgen >/dev/null 2>&1; then
|
||||
uuidgen | tr '[:upper:]' '[:lower:]'
|
||||
else
|
||||
printf 'trace-%s-%s\n' "$(date +%s)" "$$"
|
||||
fi
|
||||
}
|
||||
|
||||
hash_text() {
|
||||
if command -v sha256sum >/dev/null 2>&1; then
|
||||
sha256sum | awk '{print $1}'
|
||||
else
|
||||
shasum -a 256 | awk '{print $1}'
|
||||
fi
|
||||
}
|
||||
|
||||
word_count() {
|
||||
wc -w < "$1" | tr -d ' '
|
||||
}
|
||||
|
||||
TRACE_ID="$(new_trace_id)"
|
||||
START_TS="$(timestamp)"
|
||||
START_MS="$(epoch_ms)"
|
||||
STATUS="success"
|
||||
ERROR_MSG=""
|
||||
EXIT_CODE=0
|
||||
RETRY_COUNT="${CASAN_RETRY_COUNT:-0}"
|
||||
AGENT_NAME="${CASAN_AGENT_NAME:-unknown-agent}"
|
||||
STEP_NAME="${CASAN_STEP_NAME:-unknown-step}"
|
||||
|
||||
INPUT_TOKENS="$(word_count "$INPUT_FILE")"
|
||||
|
||||
if [[ "$#" -gt 0 ]]; then
|
||||
set +e
|
||||
CASAN_INPUT="$INPUT_FILE" CASAN_OUTPUT="$OUTPUT_FILE" "$@"
|
||||
EXIT_CODE=$?
|
||||
set -e
|
||||
if [[ "$EXIT_CODE" -ne 0 ]]; then
|
||||
STATUS="failed"
|
||||
ERROR_MSG="command exited with code $EXIT_CODE"
|
||||
fi
|
||||
|
||||
TOOL_AUDIT_RECORD="$(python3 - "$START_TS" "$TRACE_ID" "$AGENT_NAME" "$STEP_NAME" "$*" "$EXIT_CODE" "$STATUS" <<'PY'
|
||||
import json, sys
|
||||
ts, trace, agent, step, cmd, code, status = sys.argv[1:]
|
||||
print(json.dumps({
|
||||
"timestamp": ts, "trace_id": trace, "agent": agent, "step": step,
|
||||
"tool": "Bash", "command": cmd, "exit_code": int(code), "status": status,
|
||||
}))
|
||||
PY
|
||||
)"
|
||||
append_tool_audit "$TOOL_AUDIT_RECORD" "$PROJECT_ROOT"
|
||||
else
|
||||
cp "$INPUT_FILE" "$OUTPUT_FILE"
|
||||
fi
|
||||
|
||||
END_MS="$(epoch_ms)"
|
||||
LATENCY_MS=$((END_MS - START_MS))
|
||||
|
||||
if [[ ! -f "$OUTPUT_FILE" ]]; then
|
||||
STATUS="failed"
|
||||
ERROR_MSG="${ERROR_MSG:-output file not produced}"
|
||||
: > "$OUTPUT_FILE"
|
||||
fi
|
||||
|
||||
OUTPUT_TOKENS="$(word_count "$OUTPUT_FILE")"
|
||||
TOTAL_TOKENS=$((INPUT_TOKENS + OUTPUT_TOKENS))
|
||||
COST_PER_1K="${CASAN_COST_PER_1K:-0.002}"
|
||||
COST_ESTIMATE="$(python3 - "$TOTAL_TOKENS" "$COST_PER_1K" <<'PY'
|
||||
import sys
|
||||
tokens = int(sys.argv[1])
|
||||
rate = float(sys.argv[2])
|
||||
print(f"{tokens * rate / 1000:.8f}")
|
||||
PY
|
||||
)"
|
||||
COST_SOURCE="word_count_estimate"
|
||||
|
||||
# Prefer real provider usage when telemetry has been imported; the word-count
|
||||
# figure above is an explicit fallback, not presented as a real billed cost.
|
||||
PROVIDER_LOG="$PROJECT_ROOT/.specify/logs/level5/provider-usage.jsonl"
|
||||
if [[ -f "$PROVIDER_LOG" ]] && command -v python3 >/dev/null 2>&1; then
|
||||
# Use real provider telemetry ONLY when a record genuinely matches this step.
|
||||
# Do NOT fall back to an arbitrary record (that would reuse one sample's cost
|
||||
# across every step and misrepresent it as real per-step billing).
|
||||
PROV="$(python3 "$SCRIPT_DIR/provider-cost-lookup.py" "$PROVIDER_LOG" "$STEP_NAME")"
|
||||
if [[ -n "$PROV" ]]; then
|
||||
TOTAL_TOKENS="${PROV%% *}"
|
||||
COST_ESTIMATE="${PROV##* }"
|
||||
COST_SOURCE="provider_telemetry"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Real hallucination-signal detection (populates hallucination-tracking.yaml's metric).
|
||||
HALLU_YAML="$PROJECT_ROOT/.specify/agentops/hallucination-tracking.yaml"
|
||||
HALLUCINATION_SIGNALS=0
|
||||
HALLUCINATION_MATCHED="[]"
|
||||
if command -v python3 >/dev/null 2>&1; then
|
||||
HSCAN="$(python3 "$SCRIPT_DIR/hallucination-scan.py" "$HALLU_YAML" "$OUTPUT_FILE" 2>/dev/null || printf '0\n[]')"
|
||||
HALLUCINATION_SIGNALS="$(printf '%s' "$HSCAN" | head -1)"
|
||||
HALLUCINATION_MATCHED="$(printf '%s' "$HSCAN" | tail -1)"
|
||||
fi
|
||||
|
||||
INPUT_HASH="$(cat "$INPUT_FILE" | hash_text)"
|
||||
OUTPUT_HASH="$(cat "$OUTPUT_FILE" | hash_text)"
|
||||
|
||||
ALERTS=()
|
||||
if [[ "$LATENCY_MS" -gt "${CASAN_LATENCY_ALERT_MS:-5000}" ]]; then
|
||||
ALERTS+=("high-latency")
|
||||
fi
|
||||
if [[ "$RETRY_COUNT" -gt "${CASAN_RETRY_ALERT_THRESHOLD:-2}" ]]; then
|
||||
ALERTS+=("high-retry")
|
||||
fi
|
||||
if [[ "$STATUS" == "failed" ]]; then
|
||||
ALERTS+=("execution-failed")
|
||||
fi
|
||||
if [[ "$TOTAL_TOKENS" -gt "${CASAN_TOKEN_ALERT_THRESHOLD:-5000}" ]]; then
|
||||
ALERTS+=("token-overuse")
|
||||
fi
|
||||
if [[ "$HALLUCINATION_SIGNALS" -ge "${CASAN_HALLUCINATION_WARN:-3}" ]]; then
|
||||
ALERTS+=("hallucination-suspected")
|
||||
fi
|
||||
|
||||
ALERTS_JSON="$(printf '%s\n' "${ALERTS[@]:-}" | python3 -c 'import json,sys; print(json.dumps([x for x in sys.stdin.read().splitlines() if x]))')"
|
||||
TRACE_FILE="$TRACE_DIR/agentops-$TRACE_ID.json"
|
||||
cat > "$TRACE_FILE" <<EOF
|
||||
{
|
||||
"trace_id": "$TRACE_ID",
|
||||
"timestamp": "$START_TS",
|
||||
"harness": "H6-agentops",
|
||||
"agent": "$AGENT_NAME",
|
||||
"step": "$STEP_NAME",
|
||||
"status": "$STATUS",
|
||||
"exit_code": $EXIT_CODE,
|
||||
"latency_ms": $LATENCY_MS,
|
||||
"retry_count": $RETRY_COUNT,
|
||||
"input_tokens": $INPUT_TOKENS,
|
||||
"output_tokens": $OUTPUT_TOKENS,
|
||||
"total_tokens": $TOTAL_TOKENS,
|
||||
"cost_estimate": $COST_ESTIMATE,
|
||||
"cost_source": "$COST_SOURCE",
|
||||
"hallucination_signals": $HALLUCINATION_SIGNALS,
|
||||
"hallucination_matched": $HALLUCINATION_MATCHED,
|
||||
"alerts": $ALERTS_JSON,
|
||||
"input_hash": "$INPUT_HASH",
|
||||
"output_hash": "$OUTPUT_HASH",
|
||||
"error": "$ERROR_MSG"
|
||||
}
|
||||
EOF
|
||||
|
||||
printf '{"timestamp":"%s","trace_id":"%s","harness":"H6-agentops","agent":"%s","step":"%s","status":"%s","exit_code":%s,"latency_ms":%s,"retry_count":%s,"input_tokens":%s,"output_tokens":%s,"total_tokens":%s,"cost_estimate":%s,"cost_source":"%s","hallucination_signals":%s,"alerts":%s,"input_hash":"%s","output_hash":"%s"}\n' \
|
||||
"$START_TS" "$TRACE_ID" "$AGENT_NAME" "$STEP_NAME" "$STATUS" "$EXIT_CODE" "$LATENCY_MS" "$RETRY_COUNT" "$INPUT_TOKENS" "$OUTPUT_TOKENS" "$TOTAL_TOKENS" "$COST_ESTIMATE" "$COST_SOURCE" "$HALLUCINATION_SIGNALS" "$ALERTS_JSON" "$INPUT_HASH" "$OUTPUT_HASH" >> "$METRICS_LOG"
|
||||
|
||||
for alert in "${ALERTS[@]:-}"; do
|
||||
if [[ -n "$alert" ]]; then
|
||||
printf '{"timestamp":"%s","trace_id":"%s","severity":"WARN","resource":{"service.name":"%s","service.version":"1.0.0"},"body":{"message":"Alert triggered: %s","alert.type":"%s","step.name":"%s"},"attributes":{"latency_ms":%s,"status":"%s"}}\n' \
|
||||
"$START_TS" "$TRACE_ID" "$AGENT_NAME" "$alert" "$alert" "$STEP_NAME" "$LATENCY_MS" "$STATUS" >> "$ALERT_LOG"
|
||||
fi
|
||||
done
|
||||
|
||||
echo "AGENTOPS_RECORDED trace_id=$TRACE_ID status=$STATUS latency_ms=$LATENCY_MS tokens=$TOTAL_TOKENS cost=$COST_ESTIMATE output=$OUTPUT_FILE"
|
||||
exit "$EXIT_CODE"
|
||||
Reference in New Issue
Block a user