chore(freeze): snapshot demo state before Plan-07 hardening work

Freeze current submission/demo baseline:
- casan-next-plans/: full task-level plan set (Plan 00 index + 02/04/06/07/08/09/12, QA, slide deck)
- optimize-docs/video-steps/: per-vector scene breakdown (commands/screen-text/script) + start-tmux
- run-all.sh / scorecard.sh / map-live.sh: REAL=1 live-battery wiring
- regenerated evidence + audit/telemetry logs from live REAL=1 run
- submission README + video recording guide updates
- dry-run pipeline logs for 001-okr-web-app

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
thanhnv
2026-07-03 22:03:44 +09:00
co-authored by Claude Opus 4.8
parent fabd5f8783
commit fbcef967e5
182 changed files with 3865 additions and 341 deletions
+31 -9
View File
@@ -57,10 +57,15 @@ bash $S/secrets-scan.sh >/dev/null 2>&1; [ $? -eq 0 ] && h5_5=1 || h5_5=0
H5=$(( (h5_1+h5_2+h5_3+h5_4+h5_5)*20 ))
# ══════════════════════════ H6 — AGENTOPS ═════════════════════════
# h6_1 cost-spike: DISCRIMINATING (fail-able) — phải BẮT spike (positive→exit 2)
# VÀ KHÔNG báo động giả (negative→exit 0). Nếu detector luôn trả 1 giá trị → h6_1=0.
printf '%s\n' '{"step":"a","total_tokens":210}' '{"step":"b","total_tokens":195}' \
'{"step":"c","total_tokens":230}' '{"step":"plan","total_tokens":710}' > /tmp/s_usage.jsonl
bash $S/cost-spike-detect.sh /tmp/s_usage.jsonl 3.0 >/dev/null 2>&1; crc=$?
h6_1=0; { [ $crc -eq 2 ] || [ $crc -eq 0 ]; } && h6_1=1
bash $S/cost-spike-detect.sh /tmp/s_usage.jsonl 3.0 >/dev/null 2>&1; crc_pos=$?
printf '%s\n' '{"step":"a","total_tokens":210}' '{"step":"b","total_tokens":195}' \
'{"step":"c","total_tokens":230}' '{"step":"plan","total_tokens":240}' > /tmp/s_nospike.jsonl
bash $S/cost-spike-detect.sh /tmp/s_nospike.jsonl 3.0 >/dev/null 2>&1; crc_neg=$?
h6_1=0; { [ "$crc_pos" -eq 2 ] && [ "$crc_neg" -eq 0 ]; } && h6_1=1
# hallucination RATE: scanner PHẢI phân biệt output có marker (dirty) vs output sạch (clean).
# Fail-able: nếu scanner không phân biệt được (dirty<=clean) → h6_2=0.
HALLU_Y=".specify/agentops/hallucination-tracking.yaml"
@@ -70,10 +75,25 @@ sc_dirty=$(python $S/hallucination-scan.py "$HALLU_Y" /tmp/s_hallu_dirty.txt 2>/
sc_clean=$(python $S/hallucination-scan.py "$HALLU_Y" /tmp/s_hallu_clean.txt 2>/dev/null | head -1)
h6_2=0; [ "${sc_dirty:-0}" -gt "${sc_clean:-0}" ] 2>/dev/null && h6_2=1
bash $S/circuit-breaker-check.sh >/dev/null 2>&1; [ $? -eq 0 ] && h6_3=1 || h6_3=0
# h6_4 drift: DISCRIMINATING — 2 artifact khác → similarity<1.0; artifact giống → DRIFT_PASS 1.0.
# Chỉ "file tồn tại" là KHÔNG fail-able → đo bằng similarity thật (difflib).
printf 'a\nb\nc\n' > /tmp/s_gold.txt; printf 'a\nb X\nd\n' > /tmp/s_cand.txt
bash $S/drift-detect.sh /tmp/s_gold.txt /tmp/s_cand.txt /tmp/s_drift.json >/dev/null 2>&1
h6_4=0; [ -f /tmp/s_drift.json ] && h6_4=1
h6_5=0; [ -f "$T/generate-agentops-dashboard.py" ] && [ -f "$S/agent-metrics.sh" ] && h6_5=1
d_diff=$(bash $S/drift-detect.sh /tmp/s_gold.txt /tmp/s_cand.txt /tmp/s_drift.json 2>&1)
d_same=$(bash $S/drift-detect.sh /tmp/s_gold.txt /tmp/s_gold.txt /tmp/s_drift2.json 2>&1)
sim_diff=$(printf '%s' "$d_diff" | sed -n 's/.*similarity=\([0-9.]*\).*/\1/p' | head -1)
h6_4=0
{ awk "BEGIN{exit !(${sim_diff:-1} < 1.0)}" && printf '%s' "$d_same" | grep -q 'similarity=1.0'; } && h6_4=1
# h6_5 throughput/latency: agent-metrics PHẢI đo & ghi latency_ms + total_tokens THẬT của 1 run
# (không chỉ "file dashboard tồn tại"). Fail-able: nếu record thiếu số đo → h6_5=0.
printf 'agentops throughput and latency probe input\n' > /tmp/s_metric_in.txt
CASAN_AGENT_NAME=scorecard CASAN_STEP_NAME=sc-metric \
bash $S/agent-metrics.sh /tmp/s_metric_in.txt /tmp/s_metric_out.txt -- \
bash -c 'cp "$CASAN_INPUT" "$CASAN_OUTPUT"' >/dev/null 2>&1
mrec=$(tail -1 .specify/logs/cost/metrics.jsonl 2>/dev/null)
h6_5=0
printf '%s' "$mrec" | grep -Eq '"latency_ms":[0-9]+' \
&& printf '%s' "$mrec" | grep -Eq '"total_tokens":[1-9][0-9]*' \
&& [ -f "$T/generate-agentops-dashboard.py" ] && h6_5=1
H6=$(( (h6_1+h6_2+h6_3+h6_4+h6_5)*20 ))
# ── baseline (assessment doc, mục 5) ────────────────────────────────────────
@@ -94,7 +114,9 @@ IFS='|' read AVG_NOW LEVEL LOWEST < <(awk -v h1=$H1 -v h2=$H2 -v h3=$H3 -v h4=$H
AVG_BEFORE=$(awk -v h1=$H1 -v h2=$H2 -v h3=$H3 -v b4=$B4 -v b5=$B5 -v b6=$B6 -v h7=$H7 'BEGIN{printf "%.1f",(h1+h2+h3+b4+b5+b6+h7)/7}')
# ── in kết quả ──────────────────────────────────────────────────────────────
echo "${B}${CY}══════ CHECKLIST CHẤM ĐIỂM (mỗi ✓ = 1 gate chạy thật) ══════${R}"
echo "${B}${CY}══════ CHECKLIST CHẤM ĐIỂM ══════${R}"
echo " ${DIM}Mỗi ✓ = 1 test đối kháng CHẠY LIVE và FAIL-ABLE (control hỏng → ✗, điểm tụt).${R}"
echo " ${DIM}Điểm 0–100 chỉ là (số ✓/5)×100 — con số đáng tin là SỐ TEST ĐỐI KHÁNG PASS, không phải điểm.${R}"
echo
echo "${B}H4 · Security${R} → ${B}$H4/100${R} (từ 20)"
item $h4_1 "Scan prompt injection trong input [security-check → rc=2]"
@@ -111,11 +133,11 @@ item $h5_4 "Policy engine / no-bypass [circuit-breaker-check
item $h5_5 "Báo cáo compliance (secret lifecycle) [secrets-scan → PASS]"
echo
echo "${B}H6 · AgentOps${R} → ${B}$H6/100${R} (từ 30)"
item $h6_1 "Đo cost/token per step [cost-spike-detect trên telemetry]"
item $h6_1 "Cost-spike phân biệt (positive+negative) [spike→exit2 VÀ no-spike→exit0]"
item $h6_2 "Hallucination RATE trực tiếp [hallucination-scan: dirty=$sc_dirty > clean=$sc_clean]"
item $h6_3 "Alerting khi step fail > N [circuit-breaker threshold]"
item $h6_4 "Drift detection [drift-detect → report]"
item $h6_5 "Dashboard throughput/latency [agent-metrics + generate-agentops-dashboard]"
item $h6_4 "Drift phân biệt (khác→<1.0, giống→1.0) [drift-detect similarity=$sim_diff < 1.0]"
item $h6_5 "Throughput/latency đo THẬT 1 run [agent-metrics ghi latency_ms + total_tokens]"
echo
ASSESS_DATE="2026-06-26"
DOC="${DIM}chuẩn: assessment $ASSESS_DATE (KHÔNG đo lại)${R}"