feat: harden CASAN production install lifecycle
This commit is contained in:
thanhnv
@@ -8,9 +8,9 @@ Layout:
|
||||
- `scripts/` — bash + powershell gate logic (H1→H7), path resolver `casan-paths.sh`
|
||||
- `security/` — filter/policy rules (prompt-filter, pii-rules, output-policy, ...)
|
||||
- `governance/`, `agentops/` — H5/H6 code
|
||||
- `level5/` — L5 config (drift/kpi/model-fallback/tool-registry YAMLs)
|
||||
- `templates/`, `config/` — spec/plan templates, loop-policy
|
||||
- `tests/` — reproducible harness test suites + integrity manifest
|
||||
- `config/` — runtime policy, registry, model fallback, and tool metadata
|
||||
- `templates/` — spec/plan templates
|
||||
- `tests/` — source-hub verification only; excluded from production installs
|
||||
|
||||
Runtime state (logs, audit chain, tenant state) is NOT part of this package — it stays with
|
||||
the app under `CASAN_STATE_ROOT`. Domain data (golden-runs, corpus, input) lives in
|
||||
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
version: 1.0
|
||||
description: CASAN Level 5 drift detection policy
|
||||
description: CASAN production drift detection policy
|
||||
|
||||
golden_dir: apps/okr/domain/golden-runs
|
||||
evidence_dir: docs/output/casan/level5-evidence
|
||||
+7
-7
@@ -5,11 +5,11 @@
|
||||
"H4-security",
|
||||
"H5-governance",
|
||||
"H6-agentops",
|
||||
"L5-drift-detection",
|
||||
"L5-model-fallback",
|
||||
"L5-tool-registry",
|
||||
"L5-rollback",
|
||||
"L5-business-feedback"
|
||||
"drift-detection",
|
||||
"model-fallback",
|
||||
"tool-registry",
|
||||
"rollback",
|
||||
"business-feedback"
|
||||
],
|
||||
"compatibility": {
|
||||
"speckit": ">=1.0.0",
|
||||
@@ -18,7 +18,7 @@
|
||||
},
|
||||
"entrypoints": {
|
||||
"wrapper": "packages/casan-harness/scripts/bash/casan-harness.sh",
|
||||
"test": "packages/casan-harness/tests/run-casan4-harness-tests.sh",
|
||||
"dashboard": "docs/output/casan/agentops-dashboard.html"
|
||||
"project_gate": "packages/casan-harness/scripts/bash/project-gate.sh",
|
||||
"agentic_bridge": "packages/casan-harness/scripts/python/agentic_bridge.py"
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
version: 1.0
|
||||
description: CASAN Level 5 business KPI feedback schema
|
||||
description: CASAN business KPI feedback schema
|
||||
|
||||
kpis:
|
||||
- id: cycle_time_minutes
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
version: 1.0
|
||||
description: CASAN Level 5 model/tool fallback policy
|
||||
description: CASAN model/tool fallback policy
|
||||
|
||||
routes:
|
||||
- step_pattern: "review|test|security"
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
version: 1.0
|
||||
description: CASAN Level 5 tool registry with ownership, schema, risk, idempotency, and rollback metadata
|
||||
description: CASAN production tool registry with ownership, schema, risk, idempotency, and rollback metadata
|
||||
|
||||
tools:
|
||||
- id: agent_step
|
||||
@@ -41,8 +41,8 @@ _casan_find_app_root() {
|
||||
return 1
|
||||
}
|
||||
|
||||
# Harness (CODE) root: the tree that holds scripts/, security/, templates/, tests/,
|
||||
# config/, level5/*.yaml. Derived from this file's location so it follows the move.
|
||||
# Harness (CODE) root: the production tree that holds scripts/, security/,
|
||||
# templates/, config/, adapters/, and schemas/. Derived from this file's location.
|
||||
if [[ -z "${CASAN_HARNESS_ROOT:-}" ]]; then
|
||||
CASAN_HARNESS_ROOT="$(cd "$_casan_paths_self/../.." && pwd)"
|
||||
fi
|
||||
|
||||
@@ -36,7 +36,7 @@ BIN = os.path.join(ROOT, "packages", "casan-harness", "scripts", "bash")
|
||||
MODEL_ROUTER = os.environ.get("CASAN_GOAL_MODEL_ROUTER") or os.path.join(BIN, "model-router.sh")
|
||||
SECURITY = os.path.join(BIN, "security-check.sh")
|
||||
STATE_ROOT = os.environ.get("CASAN_STATE_ROOT") or os.path.join(ROOT, ".specify")
|
||||
PROJECT_REGISTRY = os.path.join(ROOT, "packages", "casan-harness", "level5", "project-registry.json")
|
||||
PROJECT_REGISTRY = os.path.join(ROOT, "packages", "casan-harness", "config", "project-registry.json")
|
||||
APPROVAL_INBOX = os.path.join(BIN, "approval-inbox.py")
|
||||
_MANIFEST_SPEC = importlib.util.spec_from_file_location(
|
||||
"casan_goal_project_manifest", os.path.join(BIN, "project_manifest.py")
|
||||
|
||||
@@ -77,7 +77,7 @@ def _manifest_for_files(files: list[str]) -> dict:
|
||||
if selected or project:
|
||||
return PROJECT_MANIFEST.load(ROOT, selected, project)
|
||||
|
||||
registry = load(os.path.join(ROOT, "packages", "casan-harness", "level5", "project-registry.json"))
|
||||
registry = load(os.path.join(ROOT, "packages", "casan-harness", "config", "project-registry.json"))
|
||||
candidates = []
|
||||
for entry in registry.get("projects", []):
|
||||
manifest_path = entry.get("manifest")
|
||||
|
||||
@@ -70,7 +70,7 @@ def load(root: str, manifest_path: str | None = None, project_id: str | None = N
|
||||
if selected:
|
||||
path = within(root, selected, "manifest")
|
||||
elif requested:
|
||||
registry = _json(os.path.join(root, "packages/casan-harness/level5/project-registry.json"), "project registry")
|
||||
registry = _json(os.path.join(root, "packages/casan-harness/config/project-registry.json"), "project registry")
|
||||
entry = next((item for item in registry.get("projects", []) if item.get("project_id") == requested), None)
|
||||
if not entry:
|
||||
_fail(f"project is not registered: {requested}")
|
||||
|
||||
@@ -14,7 +14,7 @@ fi
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$SCRIPT_DIR/casan-paths.sh"
|
||||
PROJECT_ROOT="$CASAN_APP_ROOT"
|
||||
REGISTRY="$CASAN_HARNESS_ROOT/level5/tool-registry.yaml"
|
||||
REGISTRY="$CASAN_HARNESS_ROOT/config/tool-registry.yaml"
|
||||
LOG_DIR="$CASAN_STATE_ROOT/logs/level5"
|
||||
AUDIT_DIR="$CASAN_STATE_ROOT/logs/audit"
|
||||
mkdir -p "$LOG_DIR" "$AUDIT_DIR"
|
||||
|
||||
@@ -6,8 +6,8 @@ set -euo pipefail
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$SCRIPT_DIR/casan-paths.sh"
|
||||
PROJECT_ROOT="$CASAN_APP_ROOT"
|
||||
REGISTRY="$CASAN_HARNESS_ROOT/level5/project-registry.json"
|
||||
PACKAGE="$CASAN_HARNESS_ROOT/level5/harness-package.json"
|
||||
REGISTRY="$CASAN_HARNESS_ROOT/config/project-registry.json"
|
||||
PACKAGE="$CASAN_HARNESS_ROOT/config/harness-package.json"
|
||||
|
||||
python - "$REGISTRY" "$PACKAGE" <<'PY'
|
||||
import json
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env pwsh
|
||||
# CASAN L5 Tool Registry Gate - PowerShell port of tool-registry-gate.sh
|
||||
# CASAN Tool Registry Gate - PowerShell port of tool-registry-gate.sh
|
||||
# Usage:
|
||||
# tool-registry-gate.ps1 <tool-id> [idempotency-key]
|
||||
#
|
||||
@@ -13,8 +13,9 @@ param(
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path $MyInvocation.MyCommand.Path -Parent
|
||||
$projectRoot = (Resolve-Path (Join-Path $scriptDir "../../..")).Path
|
||||
$registry = Join-Path $projectRoot ".specify/level5/tool-registry.yaml"
|
||||
$harnessRoot = if ($env:CASAN_HARNESS_ROOT) { $env:CASAN_HARNESS_ROOT } else { (Resolve-Path (Join-Path $scriptDir "../..")).Path }
|
||||
$projectRoot = if ($env:CASAN_APP_ROOT) { $env:CASAN_APP_ROOT } else { (Resolve-Path (Join-Path $scriptDir "../../../..")).Path }
|
||||
$registry = Join-Path $harnessRoot "config/tool-registry.yaml"
|
||||
$logDir = Join-Path $projectRoot ".specify/logs/level5"
|
||||
$txLog = Join-Path $logDir "tool-registry.jsonl"
|
||||
$toolCallLog = Join-Path $projectRoot ".specify/logs/audit/tool-calls.jsonl"
|
||||
|
||||
@@ -11,7 +11,7 @@ guarantee from Plan-16 even when the harness lives outside the repo.
|
||||
Only GATE-relevant trees are hashed (the code that makes security/governance
|
||||
decisions), never runtime state/logs:
|
||||
|
||||
scripts/bash scripts/python security level5 adapters schemas
|
||||
scripts/bash scripts/python security config adapters schemas
|
||||
|
||||
stdlib-only, deterministic (sorted paths), text-mode agnostic (hashes raw bytes).
|
||||
|
||||
@@ -30,10 +30,9 @@ import sys
|
||||
|
||||
ALGO = "sha256"
|
||||
HASHED_SUBTREES = (
|
||||
"scripts/bash", "scripts/python", "security", "level5", "adapters", "schemas")
|
||||
"scripts/bash", "scripts/python", "security", "config", "adapters", "schemas")
|
||||
SKIP_DIR_NAMES = {"__pycache__", ".git", "node_modules"}
|
||||
SKIP_SUFFIXES = (".pyc", ".pyo", ".log", ".tmp", ".DS_Store")
|
||||
# Within level5, only policy/config, not regenerated runtime artifacts.
|
||||
SKIP_BASENAMES = {"policy-manifest.json", "policy-manifest.sig", "project-registry.json"}
|
||||
|
||||
|
||||
|
||||
@@ -141,7 +141,7 @@ HALLU_COUNT="$(tail -n 1 "$CASAN_STATE_ROOT/logs/cost/metrics.jsonl" | sed -n 's
|
||||
[[ "${HALLU_COUNT:-0}" -ge 3 ]] && pass "H6 detects hallucination signals (count=$HALLU_COUNT)" || fail "H6 did not detect hallucination signals (count=${HALLU_COUNT:-0})"
|
||||
|
||||
# H6: imported provider telemetry becomes the authoritative cost source
|
||||
"$SCRIPTS/import-provider-telemetry.sh" "$CASAN_HARNESS_ROOT/level5/provider-usage-sample.json" > "$EVIDENCE_DIR/05c-provider-import.stdout"
|
||||
"$SCRIPTS/import-provider-telemetry.sh" "$CASAN_HARNESS_ROOT/tests/fixtures/provider-usage-sample.json" > "$EVIDENCE_DIR/05c-provider-import.stdout"
|
||||
CASAN_AGENT_NAME=demo.agent CASAN_STEP_NAME=speckit.implement \
|
||||
"$SCRIPTS/agent-metrics.sh" "$METRICS_IN" "$EVIDENCE_DIR/05c-provider-output.txt" > "$EVIDENCE_DIR/05c-provider-metrics.stdout"
|
||||
assert_contains "$CASAN_STATE_ROOT/logs/cost/metrics.jsonl" '"cost_source":"provider_telemetry"'
|
||||
@@ -261,7 +261,7 @@ assert_contains "$LEVEL5_DIR/14-business-kpi.stdout" "status=pass"
|
||||
assert_contains "$LEVEL5_DIR/16-policy-verify.stdout" "POLICY_SIGNATURE_VALID"
|
||||
|
||||
# L5: real provider usage telemetry import path
|
||||
"$SCRIPTS/import-provider-telemetry.sh" "$CASAN_HARNESS_ROOT/level5/provider-usage-sample.json" > "$LEVEL5_DIR/17-provider-telemetry.stdout"
|
||||
"$SCRIPTS/import-provider-telemetry.sh" "$CASAN_HARNESS_ROOT/tests/fixtures/provider-usage-sample.json" > "$LEVEL5_DIR/17-provider-telemetry.stdout"
|
||||
assert_contains "$LEVEL5_DIR/17-provider-telemetry.stdout" "PROVIDER_TELEMETRY_IMPORTED"
|
||||
|
||||
# L5: shared harness package is registered by multiple projects
|
||||
|
||||
Reference in New Issue
Block a user