gitea-admin/CASAN
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(plan-01): Phase 0.5 — path indirection via casan-paths.sh (no file moves)

Browse Source 
Task 1.2: introduce a single path resolver so no harness script hardcodes
`.specify/...` scattered across the tree. casan-paths.sh resolves four roots
(HARNESS/STATE/GOVERNANCE/APP) by marker-based walk-up from its own location —
never `git rev-parse` (git root is the repo PARENT here, not the app dir).

- 101 bash scripts/tests: 238 hardcoded `$PROJECT_ROOT/.specify/...` refs rewritten
  to CASAN_HARNESS_ROOT (code) / CASAN_STATE_ROOT (logs,state) / CASAN_GOVERNANCE_ROOT.
  Sandbox test vars ($WORK/$TP/$FP/$T1_WORK) left untouched.
- Roots are NOT exported: each script/subprocess self-resolves from its own tree,
  matching the original per-script semantics and preserving hermetic sandbox isolation
  (node casan-step.mjs, copied telemetry/rollback scripts must not inherit real roots).
- Sandbox tests that copy a harness script now also copy casan-paths.sh (its new
  sibling dependency): adversarial (verify-audit-chain/verify-tool-audit/rollback) +
  track-a (security-check/telemetry-integrity).
- control-plane-settings.json reclassified as STATE (untracked runtime store).

Roots all still resolve to `.specify` in this monolithic layout, so behavior is
unchanged. Full gate: PASS=64 FAIL=0 SKIP=3 (adversarial 44/0, track-a 25/0).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
thanhnv
2026-07-07 22:07:41 +09:00
co-authored by Claude Opus 4.8
parent 61c3a7c648
commit 2c765c9a45
102 changed files with 1605 additions and 1429 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AINative_OKR_CASAN5/.specify/tests/run-casan4-harness-tests.sh
+31 -30
View File
@@ -2,7 +2,8 @@
set -euo pipefail
PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
SCRIPTS="$PROJECT_ROOT/.specify/scripts/bash"
source "$PROJECT_ROOT/.specify/scripts/bash/casan-paths.sh"
SCRIPTS="$CASAN_HARNESS_ROOT/scripts/bash"
EVIDENCE_DIR="$PROJECT_ROOT/docs/output/casan/evidence"
REPORT="$EVIDENCE_DIR/harness-test-report.md"
@@ -28,7 +29,7 @@ assert_contains() {
}
assert_json_files_valid() {
python - "$PROJECT_ROOT/.specify/logs/trace" <<'PY'
python - "$CASAN_STATE_ROOT/logs/trace" <<'PY'
import json
import pathlib
import sys
@@ -53,16 +54,16 @@ PY
# Preserve retention-gap stub traces before clearing logs (WV4-B)
RETENTION_STUBS_DIR="$(mktemp -d)"
if ls "$PROJECT_ROOT/.specify/logs/trace"/agentops-*.json >/dev/null 2>&1; then
for f in "$PROJECT_ROOT/.specify/logs/trace"/agentops-*.json; do
if ls "$CASAN_STATE_ROOT/logs/trace"/agentops-*.json >/dev/null 2>&1; then
for f in "$CASAN_STATE_ROOT/logs/trace"/agentops-*.json; do
grep -q '"retention_gap": true' "$f" 2>/dev/null && cp "$f" "$RETENTION_STUBS_DIR/"
done
fi
rm -rf "$PROJECT_ROOT/.specify/logs"
rm -f "$PROJECT_ROOT/.specify/agentops/alerts.log"
mkdir -p "$PROJECT_ROOT/.specify/logs/trace" "$PROJECT_ROOT/.specify/logs/audit" "$PROJECT_ROOT/.specify/logs/cost"
rm -rf "$CASAN_STATE_ROOT/logs"
rm -f "$CASAN_HARNESS_ROOT/agentops/alerts.log"
mkdir -p "$CASAN_STATE_ROOT/logs/trace" "$CASAN_STATE_ROOT/logs/audit" "$CASAN_STATE_ROOT/logs/cost"
# Restore retention-gap stubs so context-validate.sh can verify pipeline-context.yaml
cp "$RETENTION_STUBS_DIR"/agentops-*.json "$PROJECT_ROOT/.specify/logs/trace/" 2>/dev/null || true
cp "$RETENTION_STUBS_DIR"/agentops-*.json "$CASAN_STATE_ROOT/logs/trace/" 2>/dev/null || true
rm -rf "$RETENTION_STUBS_DIR"
# H4: prompt injection blocked
@@ -124,9 +125,9 @@ METRICS_OUT="$EVIDENCE_DIR/05-metrics-output.txt"
printf 'Approved request for OKR document generation.\n' > "$METRICS_IN"
CASAN_AGENT_NAME=demo.agent CASAN_STEP_NAME=demo-step \
"$SCRIPTS/agent-metrics.sh" "$METRICS_IN" "$METRICS_OUT" > "$EVIDENCE_DIR/05-agentops.stdout"
assert_contains "$PROJECT_ROOT/.specify/logs/cost/metrics.jsonl" '"latency_ms"'
assert_contains "$PROJECT_ROOT/.specify/logs/cost/metrics.jsonl" '"cost_estimate"'
assert_contains "$PROJECT_ROOT/.specify/logs/cost/metrics.jsonl" '"cost_source"'
assert_contains "$CASAN_STATE_ROOT/logs/cost/metrics.jsonl" '"latency_ms"'
assert_contains "$CASAN_STATE_ROOT/logs/cost/metrics.jsonl" '"cost_estimate"'
assert_contains "$CASAN_STATE_ROOT/logs/cost/metrics.jsonl" '"cost_source"'
# H6: hallucination signals are actually detected and populated (not empty config)
HALLU_IN="$EVIDENCE_DIR/05b-hallucination-input.txt"
@@ -134,14 +135,14 @@ HALLU_OUT="$EVIDENCE_DIR/05b-hallucination-output.txt"
printf 'I assume the user typically wants this; I believe it might be incorrect.\n' > "$HALLU_IN"
CASAN_AGENT_NAME=demo.agent CASAN_STEP_NAME=step-1-srs \
"$SCRIPTS/agent-metrics.sh" "$HALLU_IN" "$HALLU_OUT" -- bash -c 'cp "$CASAN_INPUT" "$CASAN_OUTPUT"' > "$EVIDENCE_DIR/05b-hallucination.stdout"
HALLU_COUNT="$(tail -n 1 "$PROJECT_ROOT/.specify/logs/cost/metrics.jsonl" | sed -n 's/.*"hallucination_signals":\([0-9]*\).*/\1/p')"
HALLU_COUNT="$(tail -n 1 "$CASAN_STATE_ROOT/logs/cost/metrics.jsonl" | sed -n 's/.*"hallucination_signals":\([0-9]*\).*/\1/p')"
[[ "${HALLU_COUNT:-0}" -ge 3 ]] && pass "H6 detects hallucination signals (count=$HALLU_COUNT)" || fail "H6 did not detect hallucination signals (count=${HALLU_COUNT:-0})"
# H6: imported provider telemetry becomes the authoritative cost source
"$SCRIPTS/import-provider-telemetry.sh" "$PROJECT_ROOT/.specify/level5/provider-usage-sample.json" > "$EVIDENCE_DIR/05c-provider-import.stdout"
"$SCRIPTS/import-provider-telemetry.sh" "$CASAN_HARNESS_ROOT/level5/provider-usage-sample.json" > "$EVIDENCE_DIR/05c-provider-import.stdout"
CASAN_AGENT_NAME=demo.agent CASAN_STEP_NAME=speckit.implement \
"$SCRIPTS/agent-metrics.sh" "$METRICS_IN" "$EVIDENCE_DIR/05c-provider-output.txt" > "$EVIDENCE_DIR/05c-provider-metrics.stdout"
assert_contains "$PROJECT_ROOT/.specify/logs/cost/metrics.jsonl" '"cost_source":"provider_telemetry"'
assert_contains "$CASAN_STATE_ROOT/logs/cost/metrics.jsonl" '"cost_source":"provider_telemetry"'
# H6: failed execution emits alert
FAIL_OUT="$EVIDENCE_DIR/06-failure-output.txt"
@@ -151,11 +152,11 @@ CASAN_AGENT_NAME=demo.agent CASAN_STEP_NAME=failing-step \
FAIL_RC=$?
set -e
[[ "$FAIL_RC" -eq 7 ]] && pass "H6 preserves failing command exit code" || fail "H6 did not preserve failing command exit code"
assert_contains "$PROJECT_ROOT/.specify/agentops/alerts.log" "execution-failed"
assert_contains "$PROJECT_ROOT/.specify/logs/audit/tool-calls.jsonl" '"tool": "Bash"'
assert_contains "$CASAN_HARNESS_ROOT/agentops/alerts.log" "execution-failed"
assert_contains "$CASAN_STATE_ROOT/logs/audit/tool-calls.jsonl" '"tool": "Bash"'
# H5: audit hash-chain validates
"$SCRIPTS/verify-audit-chain.sh" "$PROJECT_ROOT/.specify/logs/audit/audit.jsonl" > "$EVIDENCE_DIR/06b-audit-chain.stdout"
"$SCRIPTS/verify-audit-chain.sh" "$CASAN_STATE_ROOT/logs/audit/audit.jsonl" > "$EVIDENCE_DIR/06b-audit-chain.stdout"
assert_contains "$EVIDENCE_DIR/06b-audit-chain.stdout" "AUDIT_CHAIN_VALID"
# Wrapper: complete H4 -> H5 -> H6 -> H4 pipeline
@@ -166,22 +167,22 @@ CASAN_AGENT_NAME=wrapper.demo CASAN_STEP_NAME=wrapper-step \
"$SCRIPTS/casan-harness.sh" "$WRAP_IN" "$WRAP_OUT" agent_step > "$EVIDENCE_DIR/07-wrapper.stdout"
assert_contains "$WRAP_OUT" "***MASKED_EMAIL***"
TRACE_COUNT_BEFORE_CACHE="$(find "$PROJECT_ROOT/.specify/logs/trace" -type f | wc -l | tr -d ' ')"
TRACE_COUNT_BEFORE_CACHE="$(find "$CASAN_STATE_ROOT/logs/trace" -type f | wc -l | tr -d ' ')"
CASAN_AGENT_NAME=wrapper.demo CASAN_STEP_NAME=wrapper-step \
"$SCRIPTS/casan-harness.sh" "$WRAP_IN" "$WRAP_OUT" agent_step > "$EVIDENCE_DIR/07b-wrapper-cache.stdout"
TRACE_COUNT_AFTER_CACHE="$(find "$PROJECT_ROOT/.specify/logs/trace" -type f | wc -l | tr -d ' ')"
TRACE_COUNT_AFTER_CACHE="$(find "$CASAN_STATE_ROOT/logs/trace" -type f | wc -l | tr -d ' ')"
assert_contains "$EVIDENCE_DIR/07b-wrapper-cache.stdout" "cache=cached"
[[ "$TRACE_COUNT_AFTER_CACHE" -gt "$TRACE_COUNT_BEFORE_CACHE" ]] && pass "H2 cache hit still records CASAN traces" || fail "H2 cache hit did not record new CASAN traces"
assert_json_files_valid | tee -a "$REPORT"
python "$PROJECT_ROOT/.specify/tests/generate-casan-demo-context.py" > "$EVIDENCE_DIR/08-demo-context.stdout"
python "$CASAN_HARNESS_ROOT/tests/generate-casan-demo-context.py" > "$EVIDENCE_DIR/08-demo-context.stdout"
assert_contains "$PROJECT_ROOT/docs/output/output_logs/casan-demo/pipeline-context.yaml" "step-13-launch"
# L5: drift detection against golden output
LEVEL5_DIR="$PROJECT_ROOT/docs/output/casan/level5-evidence"
mkdir -p "$LEVEL5_DIR"
GOLDEN="$PROJECT_ROOT/.specify/level5/golden-runs/okr-plan.golden.txt"
GOLDEN="$CASAN_HARNESS_ROOT/level5/golden-runs/okr-plan.golden.txt"
DRIFT_CANDIDATE="$LEVEL5_DIR/09-drift-candidate.txt"
DRIFT_REPORT="$LEVEL5_DIR/09-drift-report.json"
cp "$GOLDEN" "$DRIFT_CANDIDATE"
@@ -203,7 +204,7 @@ set -e
[[ "$TOOL_DENY_RC" -eq 2 ]] && pass "L5 tool registry denies deploy without idempotency key" || fail "L5 tool registry did not deny missing idempotency key"
CASAN_AGENT=release-manager CASAN_IDEMPOTENCY_KEY=deploy-demo-001 "$SCRIPTS/tool-registry-gate.sh" deploy > "$LEVEL5_DIR/12-tool-approve.stdout"
assert_contains "$LEVEL5_DIR/12-tool-approve.stdout" "TOOL_APPROVED"
assert_contains "$PROJECT_ROOT/.specify/logs/audit/tool-calls.jsonl" '"tool": "deploy"'
assert_contains "$CASAN_STATE_ROOT/logs/audit/tool-calls.jsonl" '"tool": "deploy"'
# H2: per-agent least privilege — an unauthorized agent is denied
set +e
@@ -217,10 +218,10 @@ assert_contains "$LEVEL5_DIR/11b-tool-unauthorized.stderr" "unauthorized_agent"
# and verify-audit-chain.sh both report anchor=signed regardless of how audit-public.pem
# was set by a previous CI step (Vault KMS overwrites it; re-signing with the same key
# makes verify-tool-audit.sh match).
bash "$SCRIPTS/sign-audit-head.sh" "$PROJECT_ROOT/.specify/logs/audit/audit.jsonl" >/dev/null 2>&1 || true
bash "$SCRIPTS/sign-audit-head.sh" "$CASAN_STATE_ROOT/logs/audit/audit.jsonl" >/dev/null 2>&1 || true
# H2: central tool-call audit is a tamper-evident, signed hash chain
"$SCRIPTS/verify-tool-audit.sh" "$PROJECT_ROOT/.specify/logs/audit/tool-calls.jsonl" > "$LEVEL5_DIR/11c-tool-audit-verify.stdout"
"$SCRIPTS/verify-tool-audit.sh" "$CASAN_STATE_ROOT/logs/audit/tool-calls.jsonl" > "$LEVEL5_DIR/11c-tool-audit-verify.stdout"
assert_contains "$LEVEL5_DIR/11c-tool-audit-verify.stdout" "TOOL_AUDIT_VALID"
# L5: rollback transaction — checkpoint a file and execute a genuine restore.
@@ -258,14 +259,14 @@ assert_contains "$LEVEL5_DIR/14-business-kpi.stdout" "status=pass"
assert_contains "$LEVEL5_DIR/16-policy-verify.stdout" "POLICY_SIGNATURE_VALID"
# L5: real provider usage telemetry import path
"$SCRIPTS/import-provider-telemetry.sh" "$PROJECT_ROOT/.specify/level5/provider-usage-sample.json" > "$LEVEL5_DIR/17-provider-telemetry.stdout"
"$SCRIPTS/import-provider-telemetry.sh" "$CASAN_HARNESS_ROOT/level5/provider-usage-sample.json" > "$LEVEL5_DIR/17-provider-telemetry.stdout"
assert_contains "$LEVEL5_DIR/17-provider-telemetry.stdout" "PROVIDER_TELEMETRY_IMPORTED"
# L5: shared harness package is registered by multiple projects
"$SCRIPTS/verify-harness-reuse.sh" > "$LEVEL5_DIR/18-harness-reuse.stdout"
assert_contains "$LEVEL5_DIR/18-harness-reuse.stdout" "HARNESS_REUSE_VALID"
python "$PROJECT_ROOT/.specify/tests/generate-agentops-dashboard.py" > "$LEVEL5_DIR/15-dashboard.stdout"
python "$CASAN_HARNESS_ROOT/tests/generate-agentops-dashboard.py" > "$LEVEL5_DIR/15-dashboard.stdout"
assert_contains "$PROJECT_ROOT/docs/output/casan/central-agentops-dashboard.html" "CASAN Level 4 Central AgentOps Dashboard"
{
@@ -274,13 +275,13 @@ assert_contains "$PROJECT_ROOT/docs/output/casan/central-agentops-dashboard.html
find "$EVIDENCE_DIR" -type f | sort
echo
echo "## Trace Files"
find "$PROJECT_ROOT/.specify/logs/trace" -type f | sort
find "$CASAN_STATE_ROOT/logs/trace" -type f | sort
echo
echo "## Audit Files"
find "$PROJECT_ROOT/.specify/logs/audit" -type f | sort
find "$CASAN_STATE_ROOT/logs/audit" -type f | sort
echo
echo "## Metrics Files"
find "$PROJECT_ROOT/.specify/logs/cost" -type f | sort
find "$CASAN_STATE_ROOT/logs/cost" -type f | sort
echo
echo "## Demo Pipeline Context"
echo "$PROJECT_ROOT/docs/output/output_logs/casan-demo/pipeline-context.yaml"
@@ -291,7 +292,7 @@ assert_contains "$PROJECT_ROOT/docs/output/casan/central-agentops-dashboard.html
echo "$PROJECT_ROOT/docs/output/casan/central-agentops-dashboard.html"
echo
echo "## Level 5 Logs"
find "$PROJECT_ROOT/.specify/logs/level5" -type f | sort
find "$CASAN_STATE_ROOT/logs/level5" -type f | sort
} >> "$REPORT"
echo "CASAN4 harness tests completed: $REPORT"