gitea-admin/CASAN
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update report h6

Browse Source
This commit is contained in:
thanhnv
2026-07-20 23:47:09 +07:00
parent f462079435
commit 4b6819f578
38 changed files with 1244 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/casan-harness/scripts/bash/agent-metrics.sh
+4 -4
View File
@@ -27,9 +27,9 @@ LOG_DIR="$CASAN_STATE_ROOT/logs"
TRACE_DIR="$LOG_DIR/trace"
# SEC-23 (MT-01): telemetry dir is tenant-scoped when CASAN_METRICS_DIR is set
# (tenant-paths.sh exports it per tenant); default is the shared path.
METRICS_DIR="${CASAN_METRICS_DIR:-$LOG_DIR/cost}"
ALERT_LOG="$CASAN_HARNESS_ROOT/agentops/alerts.log"
METRICS_LOG="$METRICS_DIR/metrics.jsonl"
METRICS_DIR="${CASAN_METRICS_DIR:-$(dirname "$CASAN_TELEMETRY_METRICS_LOG")}"
ALERT_LOG="$CASAN_TELEMETRY_ALERTS_LOG"
METRICS_LOG="${CASAN_METRICS_LOG:-$CASAN_TELEMETRY_METRICS_LOG}"
mkdir -p "$TRACE_DIR" "$METRICS_DIR" "$(dirname "$OUTPUT_FILE")" "$(dirname "$ALERT_LOG")"
# shellcheck source=tool-audit-lib.sh
@@ -127,7 +127,7 @@ COST_SOURCE="word_count_estimate"
# Prefer real provider usage when telemetry has been imported; the word-count
# figure above is an explicit fallback, not presented as a real billed cost.
PROVIDER_LOG="$CASAN_STATE_ROOT/logs/level5/provider-usage.jsonl"
PROVIDER_LOG="$CASAN_TELEMETRY_PROVIDER_LOG"
if [[ -f "$PROVIDER_LOG" ]] && command -v python >/dev/null 2>&1; then
# Use real provider telemetry ONLY when a record genuinely matches this step.
# Do NOT fall back to an arbitrary record (that would reuse one sample's cost
packages/casan-harness/scripts/bash/casan-paths.sh
+14
View File
@@ -61,6 +61,20 @@ if [[ -z "${CASAN_STATE_ROOT:-}" ]]; then
CASAN_STATE_ROOT="$CASAN_APP_ROOT/.specify"
fi
# Canonical H6 telemetry files. Writers and readers must use these three names so
# runtime, provider usage and alerts cannot silently drift into different trees.
# The older dashboard/control-panel variables remain accepted as compatibility
# aliases while downstream deployments move to the CASAN_TELEMETRY_* contract.
if [[ -z "${CASAN_TELEMETRY_METRICS_LOG:-}" ]]; then
CASAN_TELEMETRY_METRICS_LOG="${CASAN_DASHBOARD_METRICS:-$CASAN_STATE_ROOT/logs/cost/metrics.jsonl}"
fi
if [[ -z "${CASAN_TELEMETRY_PROVIDER_LOG:-}" ]]; then
CASAN_TELEMETRY_PROVIDER_LOG="${CASAN_CP_PROVIDER_USAGE:-${CASAN_PROVIDER_LOG:-$CASAN_STATE_ROOT/logs/level5/provider-usage.jsonl}}"
fi
if [[ -z "${CASAN_TELEMETRY_ALERTS_LOG:-}" ]]; then
CASAN_TELEMETRY_ALERTS_LOG="${CASAN_DASHBOARD_ALERTS:-$CASAN_STATE_ROOT/agentops/alerts.log}"
fi
# Governance root: central-governance mixes harness pub-keys/registries, runtime
# policy-manifest state, and a private key. Rooted under STATE (not HARNESS) so the
# runtime-regenerated policy-manifest.{json,sig} and the private key never land inside
packages/casan-harness/scripts/bash/chat-operator.py
+2 -1
View File
@@ -75,7 +75,8 @@ def head_path() -> str:
def metrics_path() -> str:
return guarded_override(os.environ["CASAN_CHAT_METRICS_LOG"]) if os.environ.get("CASAN_CHAT_METRICS_LOG") else tenant_path("telemetry/cost/metrics.jsonl", "logs/cost/metrics.jsonl")
override = os.environ.get("CASAN_TELEMETRY_METRICS_LOG") or os.environ.get("CASAN_CHAT_METRICS_LOG")
return guarded_override(override) if override else tenant_path("telemetry/cost/metrics.jsonl", "logs/cost/metrics.jsonl")
def artifact_dir() -> str:
packages/casan-harness/scripts/bash/chat-readonly.py
+2 -1
View File
@@ -90,7 +90,8 @@ def head_path() -> str:
def metrics_path() -> str:
return guarded_override(os.environ["CASAN_CHAT_METRICS_LOG"]) if os.environ.get("CASAN_CHAT_METRICS_LOG") else tenant_path("telemetry/cost/metrics.jsonl", "logs/cost/metrics.jsonl")
override = os.environ.get("CASAN_TELEMETRY_METRICS_LOG") or os.environ.get("CASAN_CHAT_METRICS_LOG")
return guarded_override(override) if override else tenant_path("telemetry/cost/metrics.jsonl", "logs/cost/metrics.jsonl")
def trace_events_path(trace_id: str) -> str:
packages/casan-harness/scripts/bash/circuit-breaker-check.sh
+1 -1
View File
@@ -24,7 +24,7 @@ set -uo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/casan-paths.sh"
ROOT="$CASAN_APP_ROOT"
PROVIDER_LOG="${CASAN_PROVIDER_LOG:-$CASAN_STATE_ROOT/logs/level5/provider-usage.jsonl}"
PROVIDER_LOG="$CASAN_TELEMETRY_PROVIDER_LOG"
CIRCUIT_BREAKER_THRESHOLD="${CIRCUIT_BREAKER_THRESHOLD:-5}"
CIRCUIT_WINDOW="${CIRCUIT_WINDOW:-10}"
CIRCUIT_WINDOW_FAIL_PCT="${CIRCUIT_WINDOW_FAIL_PCT:-50}"
packages/casan-harness/scripts/bash/cost-spike-detect.sh
+1 -1
View File
@@ -30,7 +30,7 @@ elif [[ -n "${CASAN_TENANT_ID:-}" ]]; then
LOG="$(bash "$SCRIPT_DIR/tenant-store.sh" resolve telemetry/provider-usage.jsonl 2>/dev/null)" \
|| { echo "COST_SPIKE_TENANT_DENIED" >&2; exit 3; }
else
LOG="$CASAN_STATE_ROOT/logs/level5/provider-usage.jsonl"
LOG="$CASAN_TELEMETRY_PROVIDER_LOG"
fi
MULT="${2:-3.0}"
packages/casan-harness/scripts/bash/dashboard-server.py
+4 -2
View File
@@ -35,9 +35,11 @@ PORT = int(sys.argv[1]) if len(sys.argv) > 1 else 8787
DASH = pathlib.Path(os.environ.get(
"CASAN_DASHBOARD_HTML", ROOT / "docs" / "output" / "casan" / "central-agentops-dashboard.html"))
METRICS = pathlib.Path(os.environ.get(
"CASAN_DASHBOARD_METRICS", ROOT / ".specify" / "logs" / "cost" / "metrics.jsonl"))
"CASAN_TELEMETRY_METRICS_LOG", os.environ.get(
"CASAN_DASHBOARD_METRICS", ROOT / ".specify" / "logs" / "cost" / "metrics.jsonl")))
ALERTS = pathlib.Path(os.environ.get(
"CASAN_DASHBOARD_ALERTS", ROOT / ".specify" / "agentops" / "alerts.log"))
"CASAN_TELEMETRY_ALERTS_LOG", os.environ.get(
"CASAN_DASHBOARD_ALERTS", ROOT / ".specify" / "agentops" / "alerts.log")))
STALE_S = int(os.environ.get("CASAN_DASHBOARD_STALE_S", "3600"))
packages/casan-harness/scripts/bash/evidence-pack-build.py
+11 -3
View File
@@ -133,9 +133,17 @@ def main():
"telemetry_integrity": tel_text, "telemetry_ok": tel_rc == 0,
}
# H6 cost telemetry
prov = read_jsonl(os.path.join(logs, "level5", "provider-usage.jsonl"))
metrics = read_jsonl(os.path.join(logs, "cost", "metrics.jsonl"))
# H6 telemetry — use the same canonical sources as the Control Panel/report API.
provider_path = os.environ.get(
"CASAN_TELEMETRY_PROVIDER_LOG",
os.environ.get("CASAN_CP_PROVIDER_USAGE", os.path.join(logs, "level5", "provider-usage.jsonl")),
)
metrics_path = os.environ.get(
"CASAN_TELEMETRY_METRICS_LOG",
os.environ.get("CASAN_DASHBOARD_METRICS", os.path.join(logs, "cost", "metrics.jsonl")),
)
prov = read_jsonl(provider_path)
metrics = read_jsonl(metrics_path)
cost_rc = int(os.environ.get("CASAN_EP_COST_RC", "3") or "3")
total_tokens = sum(int(r.get("total_tokens", 0)) for r in prov if str(r.get("total_tokens", "")).isdigit())
reports["h6-cost-telemetry.json"] = {
packages/casan-harness/scripts/bash/goal-orchestrator.py
+5 -1
View File
@@ -1063,7 +1063,11 @@ def audit(job: dict, status: str) -> str:
def metric(job: dict, status: str, started: float, local_meta: dict, cloud_meta: dict) -> None:
append_jsonl(os.path.join(STATE_ROOT, "logs", "cost", "metrics.jsonl"), {
metrics_path = os.environ.get(
"CASAN_TELEMETRY_METRICS_LOG",
os.environ.get("CASAN_DASHBOARD_METRICS", os.path.join(STATE_ROOT, "logs", "cost", "metrics.jsonl")),
)
append_jsonl(metrics_path, {
"timestamp": now(),
"trace_id": job["id"],
"harness": "H6-agentops",
packages/casan-harness/scripts/bash/import-provider-telemetry.sh
+2 -2
View File
@@ -14,8 +14,8 @@ fi
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/casan-paths.sh"
PROJECT_ROOT="$CASAN_APP_ROOT"
LOG_DIR="$CASAN_STATE_ROOT/logs/level5"
OUT="$LOG_DIR/provider-usage.jsonl"
OUT="$CASAN_TELEMETRY_PROVIDER_LOG"
LOG_DIR="$(dirname "$OUT")"
mkdir -p "$LOG_DIR"
python - "$INPUT_JSON" "$OUT" <<'PY'
packages/casan-harness/scripts/bash/local-full.sh
+83 -1
View File
@@ -22,6 +22,85 @@ AUTH_BRIDGE_AUDIT="$AUTH_BRIDGE_DIR/model-audit.jsonl"
AUTH_BRIDGE="$ROOT/packages/casan-control-panel/scripts/provider-auth-bridge.py"
CMD="${1:-status}"
resolve_python() {
if [[ -n "${CASAN_PYTHON_BIN:-}" ]]; then
if command -v "$CASAN_PYTHON_BIN" >/dev/null 2>&1 && "$CASAN_PYTHON_BIN" --version >/dev/null 2>&1; then
printf '%s\n' "$CASAN_PYTHON_BIN"
return 0
fi
echo "CASAN_LOCAL_PYTHON_INVALID path=$CASAN_PYTHON_BIN" >&2
return 1
fi
# Prefer macOS' universal system Python over stale framework installs that may
# appear first in PATH but are terminated by Gatekeeper/Rosetta on Apple Silicon.
local candidate
for candidate in /usr/bin/python3 python3 python; do
if command -v "$candidate" >/dev/null 2>&1 && "$candidate" --version >/dev/null 2>&1; then
printf '%s\n' "$candidate"
return 0
fi
done
echo "CASAN_LOCAL_PYTHON_MISSING" >&2
return 1
}
prepare_docker_cli() {
local original_config="${DOCKER_CONFIG:-$HOME/.docker}"
local config_file="$original_config/config.json"
[[ -f "$config_file" ]] || return 0
local python_bin credential_store helper context_name docker_host fallback_config
python_bin="$(resolve_python)" || return 1
credential_store="$("$python_bin" - "$config_file" <<'PY'
import json
import sys
try:
with open(sys.argv[1], encoding="utf-8") as handle:
print(json.load(handle).get("credsStore", ""))
except (OSError, ValueError):
print("")
PY
)"
[[ -n "$credential_store" ]] || return 0
helper="$(command -v "docker-credential-$credential_store" 2>/dev/null || true)"
if [[ -n "$helper" ]] && "$python_bin" - "$helper" <<'PY' >/dev/null 2>&1
import subprocess
import sys
raise SystemExit(subprocess.run(
[sys.argv[1], "list"],
stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
).returncode)
PY
then
return 0
fi
# CASAN's local compose files use public images only. If Docker Desktop's
# credential helper is broken, isolate this process from it without changing
# ~/.docker/config.json or touching any stored login credentials.
context_name="$(docker context show)"
docker_host="$(docker context inspect --format '{{(index .Endpoints "docker").Host}}' "$context_name")"
[[ -n "$docker_host" ]] || { echo "CASAN_LOCAL_DOCKER_CONTEXT_INVALID context=$context_name" >&2; return 1; }
fallback_config="${TMPDIR:-/tmp}/casan-docker-public-$UID"
mkdir -p "$fallback_config"
if [[ -d "$original_config/cli-plugins" && ! -e "$fallback_config/cli-plugins" ]]; then
ln -s "$original_config/cli-plugins" "$fallback_config/cli-plugins"
fi
umask 077
printf '%s\n' '{"auths":{"https://index.docker.io/v1/":{},"quay.io":{}}}' > "$fallback_config/config.json"
export DOCKER_CONFIG="$fallback_config"
export DOCKER_HOST="$docker_host"
export DOCKER_BUILDKIT=0
export COMPOSE_DOCKER_CLI_BUILD=0
echo "CASAN_LOCAL_DOCKER_CREDENTIAL_FALLBACK helper=$credential_store context=$context_name builder=classic" >&2
}
cp_compose() {
if [[ -f "$AUTH_BRIDGE_TOKEN_FILE" ]]; then
export CASAN_AUTH_BRIDGE_TOKEN
@@ -35,6 +114,8 @@ cp_compose() {
}
start_auth_bridge() {
local python_bin
python_bin="$(resolve_python)" || return 1
mkdir -p "$AUTH_BRIDGE_DIR"
if [[ ! -s "$AUTH_BRIDGE_TOKEN_FILE" ]]; then
openssl rand -hex 32 > "$AUTH_BRIDGE_TOKEN_FILE"
@@ -52,7 +133,7 @@ start_auth_bridge() {
rm -f "$AUTH_BRIDGE_PID_FILE"
fi
[[ -f "$AUTH_BRIDGE" ]] || { echo "CASAN_AUTH_BRIDGE_MISSING" >&2; return 1; }
nohup python3 "$AUTH_BRIDGE" --bind 0.0.0.0 --port 20130 --token-file "$AUTH_BRIDGE_TOKEN_FILE" --audit-log "$AUTH_BRIDGE_AUDIT" > "$AUTH_BRIDGE_LOG" 2>&1 &
nohup "$python_bin" "$AUTH_BRIDGE" --bind 0.0.0.0 --port 20130 --token-file "$AUTH_BRIDGE_TOKEN_FILE" --audit-log "$AUTH_BRIDGE_AUDIT" > "$AUTH_BRIDGE_LOG" 2>&1 &
echo "$!" > "$AUTH_BRIDGE_PID_FILE"
chmod 600 "$AUTH_BRIDGE_PID_FILE" "$AUTH_BRIDGE_LOG" "$AUTH_BRIDGE_AUDIT" 2>/dev/null || true
if ! wait_url "http://127.0.0.1:20130/healthz"; then
@@ -74,6 +155,7 @@ stop_auth_bridge() {
need_docker() {
command -v docker >/dev/null 2>&1 || { echo "CASAN_LOCAL_DOCKER_MISSING" >&2; exit 1; }
prepare_docker_cli
docker compose version >/dev/null 2>&1 || { echo "CASAN_LOCAL_COMPOSE_MISSING" >&2; exit 1; }
}
packages/casan-harness/scripts/bash/model-call.py
+4 -1
View File
@@ -48,7 +48,10 @@ OLLAMA_HOST = "127.0.0.1:11434" # the only allowed ollama endpoint
DOCKER_OLLAMA_HOST = "host.docker.internal:11434"
ALLOWED_CLOUD = {"api.anthropic.com", "api.openai.com"}
REPO_ROOT = _casan_app_root()
PROVIDER_LOG = os.path.join(REPO_ROOT, ".specify/logs/level5/provider-usage.jsonl")
PROVIDER_LOG = os.environ.get(
"CASAN_TELEMETRY_PROVIDER_LOG",
os.environ.get("CASAN_CP_PROVIDER_USAGE", os.path.join(REPO_ROOT, ".specify/logs/level5/provider-usage.jsonl")),
)
# SEC-21 (ARCH-07): a 180s-per-call timeout across many pipeline steps let a hung
# model stall a run for tens of minutes. Use a lower, configurable per-call timeout,
packages/casan-harness/scripts/bash/provider-usage-fetch.sh
+1 -1
View File
@@ -22,7 +22,7 @@ fi
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/casan-paths.sh"
PROJECT_ROOT="$CASAN_APP_ROOT"
OUT="${2:-$CASAN_STATE_ROOT/logs/level5/provider-usage.jsonl}"
OUT="${2:-$CASAN_TELEMETRY_PROVIDER_LOG}"
mkdir -p "$(dirname "$OUT")"
# SEC-13 (M-09): SSRF guard on the fetch URL. ALWAYS reject non-http(s) schemes
packages/casan-harness/scripts/powershell/agent-metrics.ps1
+3 -3
View File
@@ -19,9 +19,9 @@ $scriptDir = Split-Path $MyInvocation.MyCommand.Path -Parent
$projectRoot = (Resolve-Path (Join-Path $scriptDir "../../..")).Path
$logDir = Join-Path $projectRoot ".specify/logs"
$traceDir = Join-Path $logDir "trace"
$metricsDir = Join-Path $logDir "cost"
$alertLog = Join-Path $projectRoot ".specify/agentops/alerts.log"
$metricsLog = Join-Path $metricsDir "metrics.jsonl"
$metricsLog = if ($env:CASAN_TELEMETRY_METRICS_LOG) { $env:CASAN_TELEMETRY_METRICS_LOG } elseif ($env:CASAN_DASHBOARD_METRICS) { $env:CASAN_DASHBOARD_METRICS } else { Join-Path $logDir "cost/metrics.jsonl" }
$metricsDir = Split-Path $metricsLog -Parent
$alertLog = if ($env:CASAN_TELEMETRY_ALERTS_LOG) { $env:CASAN_TELEMETRY_ALERTS_LOG } elseif ($env:CASAN_DASHBOARD_ALERTS) { $env:CASAN_DASHBOARD_ALERTS } else { Join-Path $projectRoot ".specify/agentops/alerts.log" }
$toolAudit = Join-Path $logDir "audit/tool-calls.jsonl"
foreach ($d in @($traceDir, $metricsDir, (Split-Path $OutputFile -Parent), (Split-Path $alertLog -Parent), (Split-Path $toolAudit -Parent))) {
packages/casan-harness/scripts/powershell/generate-agentops-dashboard.ps1
+2 -2
View File
@@ -10,8 +10,8 @@ param(
$scriptDir = Split-Path $MyInvocation.MyCommand.Path -Parent
$projectRoot = (Resolve-Path (Join-Path $scriptDir "../../..")).Path
$metricsLog = Join-Path $projectRoot ".specify/logs/cost/metrics.jsonl"
$alertLog = Join-Path $projectRoot ".specify/agentops/alerts.log"
$metricsLog = if ($env:CASAN_TELEMETRY_METRICS_LOG) { $env:CASAN_TELEMETRY_METRICS_LOG } elseif ($env:CASAN_DASHBOARD_METRICS) { $env:CASAN_DASHBOARD_METRICS } else { Join-Path $projectRoot ".specify/logs/cost/metrics.jsonl" }
$alertLog = if ($env:CASAN_TELEMETRY_ALERTS_LOG) { $env:CASAN_TELEMETRY_ALERTS_LOG } elseif ($env:CASAN_DASHBOARD_ALERTS) { $env:CASAN_DASHBOARD_ALERTS } else { Join-Path $projectRoot ".specify/agentops/alerts.log" }
if (!$OutputHtml) { $OutputHtml = Join-Path $projectRoot "docs/output/casan/agentops-dashboard.html" }
if (!(Test-Path (Split-Path $OutputHtml -Parent))) { New-Item -ItemType Directory -Force -Path (Split-Path $OutputHtml -Parent) | Out-Null }