#!/usr/bin/env pwsh # CASAN H6 AgentOps Harness - PowerShell port of agent-metrics.sh # Usage: # agent-metrics.ps1 [-- [args...]] # # If command omitted: pass-through copy. # If command provided: runs under timing/cost wrapper. # Exit codes mirror the wrapped command's exit code. param( [Parameter(Mandatory=$true, Position=0)][string]$InputFile, [Parameter(Mandatory=$true, Position=1)][string]$OutputFile, [Parameter(ValueFromRemainingArguments=$true)][string[]]$RemainingArgs ) $ErrorActionPreference = "Stop" $scriptDir = Split-Path $MyInvocation.MyCommand.Path -Parent $projectRoot = (Resolve-Path (Join-Path $scriptDir "../../..")).Path $logDir = Join-Path $projectRoot ".specify/logs" $traceDir = Join-Path $logDir "trace" $metricsDir = Join-Path $logDir "cost" $alertLog = Join-Path $projectRoot ".specify/agentops/alerts.log" $metricsLog = Join-Path $metricsDir "metrics.jsonl" $toolAudit = Join-Path $logDir "audit/tool-calls.jsonl" foreach ($d in @($traceDir, $metricsDir, (Split-Path $OutputFile -Parent), (Split-Path $alertLog -Parent), (Split-Path $toolAudit -Parent))) { if ($d -and !(Test-Path $d)) { New-Item -ItemType Directory -Force -Path $d | Out-Null } } if (!(Test-Path $InputFile)) { Write-Error "AGENTOPS_FAILED: input file not found: $InputFile" exit 1 } function New-TraceId { try { return [System.Guid]::NewGuid().ToString("D") } catch { return "trace-$(Get-Date -Format 'yyyyMMddHHmmss')-$PID" } } function Get-Sha256 ([string]$text) { $bytes = [System.Text.Encoding]::UTF8.GetBytes($text) $hash = [System.Security.Cryptography.SHA256]::Create().ComputeHash($bytes) return ($hash | ForEach-Object { $_.ToString("x2") }) -join "" } function Get-WordCount ([string]$text) { return ($text -split '\s+' | Where-Object { $_ -ne "" }).Count } function ConvertTo-JsonArray ([string[]]$arr) { if (!$arr -or $arr.Count -eq 0) { return "[]" } $escaped = $arr | ForEach-Object { '"' + ($_ -replace '"','\"') + '"' } return "[" + ($escaped -join ",") + "]" } $traceId = New-TraceId $startTs = (Get-Date).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ") $startMs = [long]([System.DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds()) $status = "success" $errorMsg = "" $exitCode = 0 $retryCount = if ($env:CASAN_RETRY_COUNT) { [int]$env:CASAN_RETRY_COUNT } else { 0 } $agentName = if ($env:CASAN_AGENT_NAME) { $env:CASAN_AGENT_NAME } else { "unknown-agent" } $stepName = if ($env:CASAN_STEP_NAME) { $env:CASAN_STEP_NAME } else { "unknown-step" } $modelName = if ($env:CASAN_MODEL_NAME) { $env:CASAN_MODEL_NAME } else { "claude-opus-4-8" } $inputContent = Get-Content $InputFile -Raw -Encoding UTF8 if (!$inputContent) { $inputContent = "" } $inputTokens = Get-WordCount $inputContent # ── Strip leading "--" separator from remaining args ────────────────────── $cmdArgs = $RemainingArgs if ($cmdArgs -and $cmdArgs[0] -eq "--") { $cmdArgs = $cmdArgs[1..($cmdArgs.Count-1)] } # ── Execute wrapped command ──────────────────────────────────────────────── if ($cmdArgs -and $cmdArgs.Count -gt 0) { $env:CASAN_INPUT = $InputFile $env:CASAN_OUTPUT = $OutputFile try { & $cmdArgs[0] $cmdArgs[1..($cmdArgs.Count-1)] $exitCode = $LASTEXITCODE } catch { $exitCode = 1 $errorMsg = $_.Exception.Message } if ($exitCode -ne 0) { $status = "failed" if (!$errorMsg) { $errorMsg = "command exited with code $exitCode" } } # Tool call audit log (H2 per-call audit) $cmdStr = ($cmdArgs -join " ") -replace '"','\"' $toolLine = "{`"timestamp`":`"$startTs`",`"trace_id`":`"$traceId`",`"agent`":`"$agentName`",`"step`":`"$stepName`",`"tool`":`"Bash`",`"command`":`"$cmdStr`",`"exit_code`":$exitCode,`"status`":`"$status`"}" Add-Content -Path $toolAudit -Value $toolLine -Encoding UTF8 } else { Copy-Item -Path $InputFile -Destination $OutputFile -Force } $endMs = [long]([System.DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds()) $latencyMs = $endMs - $startMs if (!(Test-Path $OutputFile)) { $status = "failed" if (!$errorMsg) { $errorMsg = "output file not produced" } Set-Content -Path $OutputFile -Value "" -Encoding UTF8 } $outputContent = Get-Content $OutputFile -Raw -Encoding UTF8 if (!$outputContent) { $outputContent = "" } $outputTokens = Get-WordCount $outputContent $totalTokens = $inputTokens + $outputTokens # ── Token estimate v2: word-ratio model (more accurate than pure word count) ─ $modelRatios = @{ "claude-opus" = 1.35; "claude-sonnet" = 1.32; "gpt-4" = 1.30 } $modelFamily = if ($modelName -match "opus") { "claude-opus" } elseif ($modelName -match "sonnet") { "claude-sonnet" } else { "claude-opus" } $tokenRatio = $modelRatios[$modelFamily] $tokensEstimated = [int]($totalTokens * $tokenRatio) # Cost per 1M tokens (blended input+output estimate) $costPer1M = @{ "claude-opus" = 45.0; "claude-sonnet" = 9.0 } $costRate = $costPer1M[$modelFamily] $costEstimate = [math]::Round($tokensEstimated / 1000000 * $costRate, 8) $costPerKOverride = if ($env:CASAN_COST_PER_1K) { [double]$env:CASAN_COST_PER_1K } else { $null } if ($costPerKOverride) { $costEstimate = [math]::Round($totalTokens * $costPerKOverride / 1000, 8) } $inputHash = Get-Sha256 $inputContent $outputHash = Get-Sha256 $outputContent # ── Alerts ───────────────────────────────────────────────────────────────── $latencyAlertMs = if ($env:CASAN_LATENCY_ALERT_MS) { [int]$env:CASAN_LATENCY_ALERT_MS } else { 5000 } $retryAlertThresh = if ($env:CASAN_RETRY_ALERT_THRESHOLD) { [int]$env:CASAN_RETRY_ALERT_THRESHOLD } else { 2 } $tokenAlertThresh = if ($env:CASAN_TOKEN_ALERT_THRESHOLD) { [int]$env:CASAN_TOKEN_ALERT_THRESHOLD } else { 5000 } $alerts = [System.Collections.Generic.List[string]]::new() if ($latencyMs -gt $latencyAlertMs) { $alerts.Add("high-latency") } if ($retryCount -gt $retryAlertThresh) { $alerts.Add("high-retry") } if ($status -eq "failed") { $alerts.Add("execution-failed") } if ($totalTokens -gt $tokenAlertThresh){ $alerts.Add("token-overuse") } $alertsJson = ConvertTo-JsonArray ($alerts.ToArray()) # ── Trace JSON ───────────────────────────────────────────────────────────── $traceFile = Join-Path $traceDir "agentops-$traceId.json" @" { "trace_id": "$traceId", "timestamp": "$startTs", "harness": "H6-agentops", "agent": "$agentName", "step": "$stepName", "model": "$modelName", "status": "$status", "exit_code": $exitCode, "latency_ms": $latencyMs, "retry_count": $retryCount, "input_tokens": $inputTokens, "output_tokens": $outputTokens, "total_tokens": $totalTokens, "tokens_estimated": $tokensEstimated, "token_estimate_method": "word_ratio_v2", "cost_estimate": $costEstimate, "alerts": $alertsJson, "input_hash": "$inputHash", "output_hash": "$outputHash", "error": "$errorMsg" } "@ | Set-Content -Path $traceFile -Encoding UTF8 # ── Metrics JSONL ────────────────────────────────────────────────────────── $metricsLine = "{`"timestamp`":`"$startTs`",`"trace_id`":`"$traceId`",`"harness`":`"H6-agentops`",`"agent`":`"$agentName`",`"step`":`"$stepName`",`"status`":`"$status`",`"exit_code`":$exitCode,`"latency_ms`":$latencyMs,`"retry_count`":$retryCount,`"input_tokens`":$inputTokens,`"output_tokens`":$outputTokens,`"total_tokens`":$totalTokens,`"tokens_estimated`":$tokensEstimated,`"cost_estimate`":$costEstimate,`"alerts`":$alertsJson,`"input_hash`":`"$inputHash`",`"output_hash`":`"$outputHash`"}" Add-Content -Path $metricsLog -Value $metricsLine -Encoding UTF8 # ── Alert log + multi-channel notification ──────────────────────────────── foreach ($alert in $alerts) { $alertEntry = "{`"timestamp`":`"$startTs`",`"trace_id`":`"$traceId`",`"severity`":`"WARN`",`"resource`":{`"service.name`":`"$agentName`",`"service.version`":`"1.0.0`"},`"body`":{`"message`":`"Alert triggered: $alert`",`"alert.type`":`"$alert`",`"step.name`":`"$stepName`"},`"attributes`":{`"latency_ms`":$latencyMs,`"status`":`"$status`"}}" Add-Content -Path $alertLog -Value $alertEntry -Encoding UTF8 # Console notification (always on for interactive runs) Write-Warning "[CASAN ALERT] $alert | step=$stepName agent=$agentName latency=${latencyMs}ms" # Webhook (if configured) if ($env:CASAN_ALERT_WEBHOOK_URL) { try { $body = @{ text = "CASAN Alert [$alert]: $stepName — $agentName" } | ConvertTo-Json Invoke-RestMethod -Uri $env:CASAN_ALERT_WEBHOOK_URL -Method POST -Body $body -ContentType "application/json" -ErrorAction SilentlyContinue } catch { <# fire-and-forget #> } } } Write-Output "AGENTOPS_RECORDED trace_id=$traceId status=$status latency_ms=$latencyMs tokens=$totalTokens estimated_tokens=$tokensEstimated cost=$costEstimate output=$OutputFile" exit $exitCode