chore(freeze): snapshot demo state before Plan-07 hardening work
Freeze current submission/demo baseline: - casan-next-plans/: full task-level plan set (Plan 00 index + 02/04/06/07/08/09/12, QA, slide deck) - optimize-docs/video-steps/: per-vector scene breakdown (commands/screen-text/script) + start-tmux - run-all.sh / scorecard.sh / map-live.sh: REAL=1 live-battery wiring - regenerated evidence + audit/telemetry logs from live REAL=1 run - submission README + video recording guide updates - dry-run pipeline logs for 001-okr-web-app Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
fabd5f8783
commit
fbcef967e5
@@ -57,10 +57,15 @@ bash $S/secrets-scan.sh >/dev/null 2>&1; [ $? -eq 0 ] && h5_5=1 || h5_5=0
|
||||
H5=$(( (h5_1+h5_2+h5_3+h5_4+h5_5)*20 ))
|
||||
|
||||
# ══════════════════════════ H6 — AGENTOPS ═════════════════════════
|
||||
# h6_1 cost-spike: DISCRIMINATING (fail-able) — phải BẮT spike (positive→exit 2)
|
||||
# VÀ KHÔNG báo động giả (negative→exit 0). Nếu detector luôn trả 1 giá trị → h6_1=0.
|
||||
printf '%s\n' '{"step":"a","total_tokens":210}' '{"step":"b","total_tokens":195}' \
|
||||
'{"step":"c","total_tokens":230}' '{"step":"plan","total_tokens":710}' > /tmp/s_usage.jsonl
|
||||
bash $S/cost-spike-detect.sh /tmp/s_usage.jsonl 3.0 >/dev/null 2>&1; crc=$?
|
||||
h6_1=0; { [ $crc -eq 2 ] || [ $crc -eq 0 ]; } && h6_1=1
|
||||
bash $S/cost-spike-detect.sh /tmp/s_usage.jsonl 3.0 >/dev/null 2>&1; crc_pos=$?
|
||||
printf '%s\n' '{"step":"a","total_tokens":210}' '{"step":"b","total_tokens":195}' \
|
||||
'{"step":"c","total_tokens":230}' '{"step":"plan","total_tokens":240}' > /tmp/s_nospike.jsonl
|
||||
bash $S/cost-spike-detect.sh /tmp/s_nospike.jsonl 3.0 >/dev/null 2>&1; crc_neg=$?
|
||||
h6_1=0; { [ "$crc_pos" -eq 2 ] && [ "$crc_neg" -eq 0 ]; } && h6_1=1
|
||||
# hallucination RATE: scanner PHẢI phân biệt output có marker (dirty) vs output sạch (clean).
|
||||
# Fail-able: nếu scanner không phân biệt được (dirty<=clean) → h6_2=0.
|
||||
HALLU_Y=".specify/agentops/hallucination-tracking.yaml"
|
||||
@@ -70,10 +75,25 @@ sc_dirty=$(python $S/hallucination-scan.py "$HALLU_Y" /tmp/s_hallu_dirty.txt 2>/
|
||||
sc_clean=$(python $S/hallucination-scan.py "$HALLU_Y" /tmp/s_hallu_clean.txt 2>/dev/null | head -1)
|
||||
h6_2=0; [ "${sc_dirty:-0}" -gt "${sc_clean:-0}" ] 2>/dev/null && h6_2=1
|
||||
bash $S/circuit-breaker-check.sh >/dev/null 2>&1; [ $? -eq 0 ] && h6_3=1 || h6_3=0
|
||||
# h6_4 drift: DISCRIMINATING — 2 artifact khác → similarity<1.0; artifact giống → DRIFT_PASS 1.0.
|
||||
# Chỉ "file tồn tại" là KHÔNG fail-able → đo bằng similarity thật (difflib).
|
||||
printf 'a\nb\nc\n' > /tmp/s_gold.txt; printf 'a\nb X\nd\n' > /tmp/s_cand.txt
|
||||
bash $S/drift-detect.sh /tmp/s_gold.txt /tmp/s_cand.txt /tmp/s_drift.json >/dev/null 2>&1
|
||||
h6_4=0; [ -f /tmp/s_drift.json ] && h6_4=1
|
||||
h6_5=0; [ -f "$T/generate-agentops-dashboard.py" ] && [ -f "$S/agent-metrics.sh" ] && h6_5=1
|
||||
d_diff=$(bash $S/drift-detect.sh /tmp/s_gold.txt /tmp/s_cand.txt /tmp/s_drift.json 2>&1)
|
||||
d_same=$(bash $S/drift-detect.sh /tmp/s_gold.txt /tmp/s_gold.txt /tmp/s_drift2.json 2>&1)
|
||||
sim_diff=$(printf '%s' "$d_diff" | sed -n 's/.*similarity=\([0-9.]*\).*/\1/p' | head -1)
|
||||
h6_4=0
|
||||
{ awk "BEGIN{exit !(${sim_diff:-1} < 1.0)}" && printf '%s' "$d_same" | grep -q 'similarity=1.0'; } && h6_4=1
|
||||
# h6_5 throughput/latency: agent-metrics PHẢI đo & ghi latency_ms + total_tokens THẬT của 1 run
|
||||
# (không chỉ "file dashboard tồn tại"). Fail-able: nếu record thiếu số đo → h6_5=0.
|
||||
printf 'agentops throughput and latency probe input\n' > /tmp/s_metric_in.txt
|
||||
CASAN_AGENT_NAME=scorecard CASAN_STEP_NAME=sc-metric \
|
||||
bash $S/agent-metrics.sh /tmp/s_metric_in.txt /tmp/s_metric_out.txt -- \
|
||||
bash -c 'cp "$CASAN_INPUT" "$CASAN_OUTPUT"' >/dev/null 2>&1
|
||||
mrec=$(tail -1 .specify/logs/cost/metrics.jsonl 2>/dev/null)
|
||||
h6_5=0
|
||||
printf '%s' "$mrec" | grep -Eq '"latency_ms":[0-9]+' \
|
||||
&& printf '%s' "$mrec" | grep -Eq '"total_tokens":[1-9][0-9]*' \
|
||||
&& [ -f "$T/generate-agentops-dashboard.py" ] && h6_5=1
|
||||
H6=$(( (h6_1+h6_2+h6_3+h6_4+h6_5)*20 ))
|
||||
|
||||
# ── baseline (assessment doc, mục 5) ────────────────────────────────────────
|
||||
@@ -94,7 +114,9 @@ IFS='|' read AVG_NOW LEVEL LOWEST < <(awk -v h1=$H1 -v h2=$H2 -v h3=$H3 -v h4=$H
|
||||
AVG_BEFORE=$(awk -v h1=$H1 -v h2=$H2 -v h3=$H3 -v b4=$B4 -v b5=$B5 -v b6=$B6 -v h7=$H7 'BEGIN{printf "%.1f",(h1+h2+h3+b4+b5+b6+h7)/7}')
|
||||
|
||||
# ── in kết quả ──────────────────────────────────────────────────────────────
|
||||
echo "${B}${CY}══════ CHECKLIST CHẤM ĐIỂM (mỗi ✓ = 1 gate chạy thật) ══════${R}"
|
||||
echo "${B}${CY}══════ CHECKLIST CHẤM ĐIỂM ══════${R}"
|
||||
echo " ${DIM}Mỗi ✓ = 1 test đối kháng CHẠY LIVE và FAIL-ABLE (control hỏng → ✗, điểm tụt).${R}"
|
||||
echo " ${DIM}Điểm 0–100 chỉ là (số ✓/5)×100 — con số đáng tin là SỐ TEST ĐỐI KHÁNG PASS, không phải điểm.${R}"
|
||||
echo
|
||||
echo "${B}H4 · Security${R} → ${B}$H4/100${R} (từ 20)"
|
||||
item $h4_1 "Scan prompt injection trong input [security-check → rc=2]"
|
||||
@@ -111,11 +133,11 @@ item $h5_4 "Policy engine / no-bypass [circuit-breaker-check
|
||||
item $h5_5 "Báo cáo compliance (secret lifecycle) [secrets-scan → PASS]"
|
||||
echo
|
||||
echo "${B}H6 · AgentOps${R} → ${B}$H6/100${R} (từ 30)"
|
||||
item $h6_1 "Đo cost/token per step [cost-spike-detect trên telemetry]"
|
||||
item $h6_1 "Cost-spike phân biệt (positive+negative) [spike→exit2 VÀ no-spike→exit0]"
|
||||
item $h6_2 "Hallucination RATE trực tiếp [hallucination-scan: dirty=$sc_dirty > clean=$sc_clean]"
|
||||
item $h6_3 "Alerting khi step fail > N [circuit-breaker threshold]"
|
||||
item $h6_4 "Drift detection [drift-detect → report]"
|
||||
item $h6_5 "Dashboard throughput/latency [agent-metrics + generate-agentops-dashboard]"
|
||||
item $h6_4 "Drift phân biệt (khác→<1.0, giống→1.0) [drift-detect similarity=$sim_diff < 1.0]"
|
||||
item $h6_5 "Throughput/latency đo THẬT 1 run [agent-metrics ghi latency_ms + total_tokens]"
|
||||
echo
|
||||
ASSESS_DATE="2026-06-26"
|
||||
DOC="${DIM}chuẩn: assessment $ASSESS_DATE (KHÔNG đo lại)${R}"
|
||||
|
||||
Reference in New Issue
Block a user