test(ui): five verification rounds, and two things they found
Five rounds, each looking from an angle the previous one cannot:
1. text — every "Thay đổi" bullet on the page vs a probe (existing)
2. geometry — check_layout_geometry.py: reading order of the rail, section
order down Monitoring, which side each pane is on, and the size
relationships the design names (hero card, 26px dot)
3. inventory— check_controls_alive.py: every control in controls.json still
present in the built app, attributed to its owning CLASS via the
baseline commit (a file holds several classes, so a per-file
check reported eight dialog controls as missing)
4. eyes — rendered screens read against the wireframes
5. adversarial — check_probes_bite.py: break one feature at a time and fail if
the matching check still passes
What rounds 4 and 5 caught, which 1-3 could not:
* Schedule showed six of seven lanes; the seventh needed a horizontal
scroll. The design says "giữ đủ 7 lane, thu hẹp cho vừa một màn". Lane
minimum width 190 → 150, so 7 × 150 + gaps fits a 1280 window. Round 2 now
measures this instead of relying on someone noticing.
* check_design_parity ALWAYS returned 0. It was a report, not a check: every
probe in it was incapable of failing, so a regression would print on screen
and still exit green. It now exits non-zero when anything is CHUA — which
is what let round 5 detect the two mutations it had been sleeping through.
Also fixed in the harness itself: it rewrote line endings while restoring
mutated files (read_text/write_text translate both ways), and it compared the
tree against "clean" rather than against its own starting state.
All ten checkers green by exit code.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,130 @@
|
||||
"""Round 5: do the checks actually bite?
|
||||
|
||||
Rounds 1–4 all report green. That is only worth something if the checks would
|
||||
have turned red had the work not been done. So this round breaks the app on
|
||||
purpose, one feature at a time, and fails if the corresponding check still
|
||||
passes — a check that cannot fail is not evidence.
|
||||
|
||||
Each mutation is applied by monkey-patching the module BEFORE the checker
|
||||
builds its own window, then undone.
|
||||
|
||||
Run: python tools/check_probes_bite.py
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import os
|
||||
import runpy
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(REPO.parent))
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
os.environ.setdefault("QT_QPA_PLATFORM", "offscreen")
|
||||
|
||||
# (name, file, find, replace, checker that must FAIL because of it)
|
||||
MUTATIONS = [
|
||||
("bo cham tro ly 26px -> 64px",
|
||||
"ui/help_agent_widget.py", "_DOT = 26", "_DOT = 64",
|
||||
"check_layout_geometry.py"),
|
||||
("tra lane Running ve khong vien",
|
||||
"ui/schedule_task_tab.py",
|
||||
'if status == "running" and counts[status]:',
|
||||
'if False:',
|
||||
"check_design_parity.py"),
|
||||
("bo cot muc luc cua Cai dat",
|
||||
"ui/settings_dialog.py",
|
||||
"self.section_list, self.section_stack = section_panels(pages)",
|
||||
"self.section_list, self.section_stack = section_panels(pages[:1])",
|
||||
"check_dialogs.py"),
|
||||
("noi lai dai tab flow Co4E",
|
||||
"ui/co4e_tab.py",
|
||||
"self.flow_scroll.setVisible(False)",
|
||||
"self.flow_scroll.setVisible(True)",
|
||||
"check_co4e.py"),
|
||||
("bo dong 'Tat ca project...' khoi GAN DAY",
|
||||
"app.py",
|
||||
'more.setData(0, Qt.UserRole, {"all": True})',
|
||||
'more.setData(0, Qt.UserRole, {})',
|
||||
"check_design_parity.py"),
|
||||
("tra thanh menu ve accordion (bo nhom day)",
|
||||
"app.py",
|
||||
'rows.append((self.nav_bottom, self._ROW_DASHBOARD, None,',
|
||||
'rows.append((self.nav, self._ROW_DASHBOARD, None,',
|
||||
"check_layout_geometry.py"),
|
||||
]
|
||||
|
||||
|
||||
def run_checker(script: str) -> int:
|
||||
"""Run a checker in a fresh process; return its exit code."""
|
||||
proc = subprocess.run(
|
||||
[sys.executable, str(REPO / "tools" / script)],
|
||||
cwd=REPO, capture_output=True, text=True, encoding="utf-8",
|
||||
errors="replace", env={**os.environ, "QT_QPA_PLATFORM": "offscreen",
|
||||
"PYTHONIOENCODING": "utf-8"})
|
||||
return proc.returncode
|
||||
|
||||
|
||||
def tree_state() -> str:
|
||||
return subprocess.run(["git", "status", "--short"], cwd=REPO,
|
||||
capture_output=True, text=True).stdout.strip()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
fails: list[str] = []
|
||||
# Compare the tree BEFORE and AFTER, not against a clean tree: work in
|
||||
# progress is legitimately uncommitted, and demanding a clean tree made this
|
||||
# round fail for a reason that has nothing to do with the mutations.
|
||||
before = tree_state()
|
||||
print(f"{'hong gi':44} {'phep do':26} ket qua")
|
||||
print("-" * 88)
|
||||
for name, rel, find, repl, checker in MUTATIONS:
|
||||
path = REPO / rel
|
||||
# newline="" both ways: the default translates on read AND write, so a
|
||||
# LF file came back as CRLF and every mutated file was left "modified"
|
||||
# even after being restored.
|
||||
with io.open(path, "r", encoding="utf-8", newline="") as fh:
|
||||
original = fh.read()
|
||||
if find not in original:
|
||||
fails.append(f"{name}: khong tim thay doan can sua trong {rel}")
|
||||
print(f"{name:44} {checker:26} *** KHONG AP DUNG DUOC ***")
|
||||
continue
|
||||
|
||||
def write(text: str) -> None:
|
||||
with io.open(path, "w", encoding="utf-8", newline="") as fh:
|
||||
fh.write(text)
|
||||
|
||||
write(original.replace(find, repl, 1))
|
||||
try:
|
||||
code = run_checker(checker)
|
||||
finally:
|
||||
write(original) # always restore
|
||||
bit = code != 0
|
||||
print(f"{name:44} {checker:26} {'BAT DUOC' if bit else '*** KHONG BAT ***'}")
|
||||
if not bit:
|
||||
fails.append(f"{name}: {checker} van bao xanh du da lam hong")
|
||||
|
||||
# Everything must be back exactly as it was before this run.
|
||||
after = tree_state()
|
||||
same = after == before
|
||||
print()
|
||||
print("cay lam viec sau khi thu giong het truoc:", "co" if same else "*** KHAC ***")
|
||||
if not same:
|
||||
print(" truoc:", before.replace("\n", " | ") or "(sach)")
|
||||
print(" sau :", after.replace("\n", " | ") or "(sach)")
|
||||
fails.append("file chua duoc khoi phuc sau khi thu")
|
||||
|
||||
print()
|
||||
if fails:
|
||||
print("*** VONG 5 THAT BAI ***")
|
||||
for f in fails:
|
||||
print(" " + f)
|
||||
return 1
|
||||
print(f"KET QUA VONG 5: ca {len(MUTATIONS)} phep do deu bat duoc loi khi co tinh lam hong")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user