197 lines
9.6 KiB
PowerShell
197 lines
9.6 KiB
PowerShell
#!/usr/bin/env pwsh
|
|
# CASAN H6 AgentOps Harness - PowerShell port of agent-metrics.sh
|
|
# Usage:
|
|
# agent-metrics.ps1 <input-file> <output-file> [-- <command> [args...]]
|
|
#
|
|
# If command omitted: pass-through copy.
|
|
# If command provided: runs under timing/cost wrapper.
|
|
# Exit codes mirror the wrapped command's exit code.
|
|
|
|
param(
|
|
[Parameter(Mandatory=$true, Position=0)][string]$InputFile,
|
|
[Parameter(Mandatory=$true, Position=1)][string]$OutputFile,
|
|
[Parameter(ValueFromRemainingArguments=$true)][string[]]$RemainingArgs
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
$scriptDir = Split-Path $MyInvocation.MyCommand.Path -Parent
|
|
$projectRoot = (Resolve-Path (Join-Path $scriptDir "../../..")).Path
|
|
$logDir = Join-Path $projectRoot ".specify/logs"
|
|
$traceDir = Join-Path $logDir "trace"
|
|
$metricsLog = if ($env:CASAN_TELEMETRY_METRICS_LOG) { $env:CASAN_TELEMETRY_METRICS_LOG } elseif ($env:CASAN_DASHBOARD_METRICS) { $env:CASAN_DASHBOARD_METRICS } else { Join-Path $logDir "cost/metrics.jsonl" }
|
|
$metricsDir = Split-Path $metricsLog -Parent
|
|
$alertLog = if ($env:CASAN_TELEMETRY_ALERTS_LOG) { $env:CASAN_TELEMETRY_ALERTS_LOG } elseif ($env:CASAN_DASHBOARD_ALERTS) { $env:CASAN_DASHBOARD_ALERTS } else { Join-Path $projectRoot ".specify/agentops/alerts.log" }
|
|
$toolAudit = Join-Path $logDir "audit/tool-calls.jsonl"
|
|
|
|
foreach ($d in @($traceDir, $metricsDir, (Split-Path $OutputFile -Parent), (Split-Path $alertLog -Parent), (Split-Path $toolAudit -Parent))) {
|
|
if ($d -and !(Test-Path $d)) { New-Item -ItemType Directory -Force -Path $d | Out-Null }
|
|
}
|
|
|
|
if (!(Test-Path $InputFile)) {
|
|
Write-Error "AGENTOPS_FAILED: input file not found: $InputFile"
|
|
exit 1
|
|
}
|
|
|
|
function New-TraceId {
|
|
try { return [System.Guid]::NewGuid().ToString("D") } catch { return "trace-$(Get-Date -Format 'yyyyMMddHHmmss')-$PID" }
|
|
}
|
|
|
|
function Get-Sha256 ([string]$text) {
|
|
$bytes = [System.Text.Encoding]::UTF8.GetBytes($text)
|
|
$hash = [System.Security.Cryptography.SHA256]::Create().ComputeHash($bytes)
|
|
return ($hash | ForEach-Object { $_.ToString("x2") }) -join ""
|
|
}
|
|
|
|
function Get-WordCount ([string]$text) {
|
|
return ($text -split '\s+' | Where-Object { $_ -ne "" }).Count
|
|
}
|
|
|
|
function ConvertTo-JsonArray ([string[]]$arr) {
|
|
if (!$arr -or $arr.Count -eq 0) { return "[]" }
|
|
$escaped = $arr | ForEach-Object { '"' + ($_ -replace '"','\"') + '"' }
|
|
return "[" + ($escaped -join ",") + "]"
|
|
}
|
|
|
|
$traceId = New-TraceId
|
|
$startTs = (Get-Date).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ")
|
|
$startMs = [long]([System.DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds())
|
|
$status = "success"
|
|
$errorMsg = ""
|
|
$exitCode = 0
|
|
|
|
$retryCount = if ($env:CASAN_RETRY_COUNT) { [int]$env:CASAN_RETRY_COUNT } else { 0 }
|
|
$agentName = if ($env:CASAN_AGENT_NAME) { $env:CASAN_AGENT_NAME } else { "unknown-agent" }
|
|
$stepName = if ($env:CASAN_STEP_NAME) { $env:CASAN_STEP_NAME } else { "unknown-step" }
|
|
$modelName = if ($env:CASAN_MODEL_NAME) { $env:CASAN_MODEL_NAME } else { "claude-opus-4-8" }
|
|
|
|
$inputContent = Get-Content $InputFile -Raw -Encoding UTF8
|
|
if (!$inputContent) { $inputContent = "" }
|
|
$inputTokens = Get-WordCount $inputContent
|
|
|
|
# ── Strip leading "--" separator from remaining args ──────────────────────
|
|
$cmdArgs = $RemainingArgs
|
|
if ($cmdArgs -and $cmdArgs[0] -eq "--") { $cmdArgs = $cmdArgs[1..($cmdArgs.Count-1)] }
|
|
|
|
# ── Execute wrapped command ────────────────────────────────────────────────
|
|
if ($cmdArgs -and $cmdArgs.Count -gt 0) {
|
|
$env:CASAN_INPUT = $InputFile
|
|
$env:CASAN_OUTPUT = $OutputFile
|
|
|
|
try {
|
|
& $cmdArgs[0] $cmdArgs[1..($cmdArgs.Count-1)]
|
|
$exitCode = $LASTEXITCODE
|
|
} catch {
|
|
$exitCode = 1
|
|
$errorMsg = $_.Exception.Message
|
|
}
|
|
|
|
if ($exitCode -ne 0) {
|
|
$status = "failed"
|
|
if (!$errorMsg) { $errorMsg = "command exited with code $exitCode" }
|
|
}
|
|
|
|
# Tool call audit log (H2 per-call audit)
|
|
$cmdStr = ($cmdArgs -join " ") -replace '"','\"'
|
|
$toolLine = "{`"timestamp`":`"$startTs`",`"trace_id`":`"$traceId`",`"agent`":`"$agentName`",`"step`":`"$stepName`",`"tool`":`"Bash`",`"command`":`"$cmdStr`",`"exit_code`":$exitCode,`"status`":`"$status`"}"
|
|
Add-Content -Path $toolAudit -Value $toolLine -Encoding UTF8
|
|
} else {
|
|
Copy-Item -Path $InputFile -Destination $OutputFile -Force
|
|
}
|
|
|
|
$endMs = [long]([System.DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds())
|
|
$latencyMs = $endMs - $startMs
|
|
|
|
if (!(Test-Path $OutputFile)) {
|
|
$status = "failed"
|
|
if (!$errorMsg) { $errorMsg = "output file not produced" }
|
|
Set-Content -Path $OutputFile -Value "" -Encoding UTF8
|
|
}
|
|
|
|
$outputContent = Get-Content $OutputFile -Raw -Encoding UTF8
|
|
if (!$outputContent) { $outputContent = "" }
|
|
$outputTokens = Get-WordCount $outputContent
|
|
$totalTokens = $inputTokens + $outputTokens
|
|
|
|
# ── Token estimate v2: word-ratio model (more accurate than pure word count) ─
|
|
$modelRatios = @{ "claude-opus" = 1.35; "claude-sonnet" = 1.32; "gpt-4" = 1.30 }
|
|
$modelFamily = if ($modelName -match "opus") { "claude-opus" }
|
|
elseif ($modelName -match "sonnet") { "claude-sonnet" }
|
|
else { "claude-opus" }
|
|
$tokenRatio = $modelRatios[$modelFamily]
|
|
$tokensEstimated = [int]($totalTokens * $tokenRatio)
|
|
|
|
# Cost per 1M tokens (blended input+output estimate)
|
|
$costPer1M = @{ "claude-opus" = 45.0; "claude-sonnet" = 9.0 }
|
|
$costRate = $costPer1M[$modelFamily]
|
|
$costEstimate = [math]::Round($tokensEstimated / 1000000 * $costRate, 8)
|
|
|
|
$costPerKOverride = if ($env:CASAN_COST_PER_1K) { [double]$env:CASAN_COST_PER_1K } else { $null }
|
|
if ($costPerKOverride) { $costEstimate = [math]::Round($totalTokens * $costPerKOverride / 1000, 8) }
|
|
|
|
$inputHash = Get-Sha256 $inputContent
|
|
$outputHash = Get-Sha256 $outputContent
|
|
|
|
# ── Alerts ─────────────────────────────────────────────────────────────────
|
|
$latencyAlertMs = if ($env:CASAN_LATENCY_ALERT_MS) { [int]$env:CASAN_LATENCY_ALERT_MS } else { 5000 }
|
|
$retryAlertThresh = if ($env:CASAN_RETRY_ALERT_THRESHOLD) { [int]$env:CASAN_RETRY_ALERT_THRESHOLD } else { 2 }
|
|
$tokenAlertThresh = if ($env:CASAN_TOKEN_ALERT_THRESHOLD) { [int]$env:CASAN_TOKEN_ALERT_THRESHOLD } else { 5000 }
|
|
|
|
$alerts = [System.Collections.Generic.List[string]]::new()
|
|
if ($latencyMs -gt $latencyAlertMs) { $alerts.Add("high-latency") }
|
|
if ($retryCount -gt $retryAlertThresh) { $alerts.Add("high-retry") }
|
|
if ($status -eq "failed") { $alerts.Add("execution-failed") }
|
|
if ($totalTokens -gt $tokenAlertThresh){ $alerts.Add("token-overuse") }
|
|
$alertsJson = ConvertTo-JsonArray ($alerts.ToArray())
|
|
|
|
# ── Trace JSON ─────────────────────────────────────────────────────────────
|
|
$traceFile = Join-Path $traceDir "agentops-$traceId.json"
|
|
@"
|
|
{
|
|
"trace_id": "$traceId",
|
|
"timestamp": "$startTs",
|
|
"harness": "H6-agentops",
|
|
"agent": "$agentName",
|
|
"step": "$stepName",
|
|
"model": "$modelName",
|
|
"status": "$status",
|
|
"exit_code": $exitCode,
|
|
"latency_ms": $latencyMs,
|
|
"retry_count": $retryCount,
|
|
"input_tokens": $inputTokens,
|
|
"output_tokens": $outputTokens,
|
|
"total_tokens": $totalTokens,
|
|
"tokens_estimated": $tokensEstimated,
|
|
"token_estimate_method": "word_ratio_v2",
|
|
"cost_estimate": $costEstimate,
|
|
"alerts": $alertsJson,
|
|
"input_hash": "$inputHash",
|
|
"output_hash": "$outputHash",
|
|
"error": "$errorMsg"
|
|
}
|
|
"@ | Set-Content -Path $traceFile -Encoding UTF8
|
|
|
|
# ── Metrics JSONL ──────────────────────────────────────────────────────────
|
|
$metricsLine = "{`"timestamp`":`"$startTs`",`"trace_id`":`"$traceId`",`"harness`":`"H6-agentops`",`"agent`":`"$agentName`",`"step`":`"$stepName`",`"status`":`"$status`",`"exit_code`":$exitCode,`"latency_ms`":$latencyMs,`"retry_count`":$retryCount,`"input_tokens`":$inputTokens,`"output_tokens`":$outputTokens,`"total_tokens`":$totalTokens,`"tokens_estimated`":$tokensEstimated,`"cost_estimate`":$costEstimate,`"alerts`":$alertsJson,`"input_hash`":`"$inputHash`",`"output_hash`":`"$outputHash`"}"
|
|
Add-Content -Path $metricsLog -Value $metricsLine -Encoding UTF8
|
|
|
|
# ── Alert log + multi-channel notification ────────────────────────────────
|
|
foreach ($alert in $alerts) {
|
|
$alertEntry = "{`"timestamp`":`"$startTs`",`"trace_id`":`"$traceId`",`"severity`":`"WARN`",`"resource`":{`"service.name`":`"$agentName`",`"service.version`":`"1.0.0`"},`"body`":{`"message`":`"Alert triggered: $alert`",`"alert.type`":`"$alert`",`"step.name`":`"$stepName`"},`"attributes`":{`"latency_ms`":$latencyMs,`"status`":`"$status`"}}"
|
|
Add-Content -Path $alertLog -Value $alertEntry -Encoding UTF8
|
|
|
|
# Console notification (always on for interactive runs)
|
|
Write-Warning "[CASAN ALERT] $alert | step=$stepName agent=$agentName latency=${latencyMs}ms"
|
|
|
|
# Webhook (if configured)
|
|
if ($env:CASAN_ALERT_WEBHOOK_URL) {
|
|
try {
|
|
$body = @{ text = "CASAN Alert [$alert]: $stepName — $agentName" } | ConvertTo-Json
|
|
Invoke-RestMethod -Uri $env:CASAN_ALERT_WEBHOOK_URL -Method POST -Body $body -ContentType "application/json" -ErrorAction SilentlyContinue
|
|
} catch { <# fire-and-forget #> }
|
|
}
|
|
}
|
|
|
|
Write-Output "AGENTOPS_RECORDED trace_id=$traceId status=$status latency_ms=$latencyMs tokens=$totalTokens estimated_tokens=$tokensEstimated cost=$costEstimate output=$OutputFile"
|
|
exit $exitCode
|