test(ui): five verification rounds, and two things they found

Five rounds, each looking from an angle the previous one cannot:

  1. text     — every "Thay đổi" bullet on the page vs a probe (existing)
  2. geometry — check_layout_geometry.py: reading order of the rail, section
                order down Monitoring, which side each pane is on, and the size
                relationships the design names (hero card, 26px dot)
  3. inventory— check_controls_alive.py: every control in controls.json still
                present in the built app, attributed to its owning CLASS via the
                baseline commit (a file holds several classes, so a per-file
                check reported eight dialog controls as missing)
  4. eyes     — rendered screens read against the wireframes
  5. adversarial — check_probes_bite.py: break one feature at a time and fail if
                the matching check still passes

What rounds 4 and 5 caught, which 1-3 could not:

  * Schedule showed six of seven lanes; the seventh needed a horizontal
    scroll. The design says "giữ đủ 7 lane, thu hẹp cho vừa một màn". Lane
    minimum width 190 → 150, so 7 × 150 + gaps fits a 1280 window. Round 2 now
    measures this instead of relying on someone noticing.

  * check_design_parity ALWAYS returned 0. It was a report, not a check: every
    probe in it was incapable of failing, so a regression would print on screen
    and still exit green. It now exits non-zero when anything is CHUA — which
    is what let round 5 detect the two mutations it had been sleeping through.

Also fixed in the harness itself: it rewrote line endings while restoring
mutated files (read_text/write_text translate both ways), and it compared the
tree against "clean" rather than against its own starting state.

All ten checkers green by exit code.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
NamPDT
2026-08-17 13:55:47 +09:00
co-authored by Claude Opus 5
parent 62bec1f984
commit f381fd298a
5 changed files with 535 additions and 2 deletions
+130
View File
@@ -0,0 +1,130 @@
"""Round 5: do the checks actually bite?
Rounds 1–4 all report green. That is only worth something if the checks would
have turned red had the work not been done. So this round breaks the app on
purpose, one feature at a time, and fails if the corresponding check still
passes — a check that cannot fail is not evidence.
Each mutation is applied by monkey-patching the module BEFORE the checker
builds its own window, then undone.
Run: python tools/check_probes_bite.py
"""
from __future__ import annotations
import io
import os
import runpy
import subprocess
import sys
from pathlib import Path
REPO = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(REPO.parent))
sys.path.insert(0, str(Path(__file__).resolve().parent))
os.environ.setdefault("QT_QPA_PLATFORM", "offscreen")
# (name, file, find, replace, checker that must FAIL because of it)
MUTATIONS = [
("bo cham tro ly 26px -> 64px",
"ui/help_agent_widget.py", "_DOT = 26", "_DOT = 64",
"check_layout_geometry.py"),
("tra lane Running ve khong vien",
"ui/schedule_task_tab.py",
'if status == "running" and counts[status]:',
'if False:',
"check_design_parity.py"),
("bo cot muc luc cua Cai dat",
"ui/settings_dialog.py",
"self.section_list, self.section_stack = section_panels(pages)",
"self.section_list, self.section_stack = section_panels(pages[:1])",
"check_dialogs.py"),
("noi lai dai tab flow Co4E",
"ui/co4e_tab.py",
"self.flow_scroll.setVisible(False)",
"self.flow_scroll.setVisible(True)",
"check_co4e.py"),
("bo dong 'Tat ca project...' khoi GAN DAY",
"app.py",
'more.setData(0, Qt.UserRole, {"all": True})',
'more.setData(0, Qt.UserRole, {})',
"check_design_parity.py"),
("tra thanh menu ve accordion (bo nhom day)",
"app.py",
'rows.append((self.nav_bottom, self._ROW_DASHBOARD, None,',
'rows.append((self.nav, self._ROW_DASHBOARD, None,',
"check_layout_geometry.py"),
]
def run_checker(script: str) -> int:
"""Run a checker in a fresh process; return its exit code."""
proc = subprocess.run(
[sys.executable, str(REPO / "tools" / script)],
cwd=REPO, capture_output=True, text=True, encoding="utf-8",
errors="replace", env={**os.environ, "QT_QPA_PLATFORM": "offscreen",
"PYTHONIOENCODING": "utf-8"})
return proc.returncode
def tree_state() -> str:
return subprocess.run(["git", "status", "--short"], cwd=REPO,
capture_output=True, text=True).stdout.strip()
def main() -> int:
fails: list[str] = []
# Compare the tree BEFORE and AFTER, not against a clean tree: work in
# progress is legitimately uncommitted, and demanding a clean tree made this
# round fail for a reason that has nothing to do with the mutations.
before = tree_state()
print(f"{'hong gi':44} {'phep do':26} ket qua")
print("-" * 88)
for name, rel, find, repl, checker in MUTATIONS:
path = REPO / rel
# newline="" both ways: the default translates on read AND write, so a
# LF file came back as CRLF and every mutated file was left "modified"
# even after being restored.
with io.open(path, "r", encoding="utf-8", newline="") as fh:
original = fh.read()
if find not in original:
fails.append(f"{name}: khong tim thay doan can sua trong {rel}")
print(f"{name:44} {checker:26} *** KHONG AP DUNG DUOC ***")
continue
def write(text: str) -> None:
with io.open(path, "w", encoding="utf-8", newline="") as fh:
fh.write(text)
write(original.replace(find, repl, 1))
try:
code = run_checker(checker)
finally:
write(original) # always restore
bit = code != 0
print(f"{name:44} {checker:26} {'BAT DUOC' if bit else '*** KHONG BAT ***'}")
if not bit:
fails.append(f"{name}: {checker} van bao xanh du da lam hong")
# Everything must be back exactly as it was before this run.
after = tree_state()
same = after == before
print()
print("cay lam viec sau khi thu giong het truoc:", "co" if same else "*** KHAC ***")
if not same:
print(" truoc:", before.replace("\n", " | ") or "(sach)")
print(" sau :", after.replace("\n", " | ") or "(sach)")
fails.append("file chua duoc khoi phuc sau khi thu")
print()
if fails:
print("*** VONG 5 THAT BAI ***")
for f in fails:
print(" " + f)
return 1
print(f"KET QUA VONG 5: ca {len(MUTATIONS)} phep do deu bat duoc loi khi co tinh lam hong")
return 0
if __name__ == "__main__":
raise SystemExit(main())