diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 600774dc96f..26a851d4a3b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,7 +50,19 @@ jobs: - name: Test run: | set -x - scripts/rip-environment runtests -p + scripts/rip-environment runtests -p -d + - name: Upload UI smoke screenshots + if: always() + uses: actions/upload-artifact@v7 + with: + name: ui-smoke-screenshots-gcc + path: | + tests/ui-smoke/**/screenshot.png + tests/ui-smoke/**/confirm.png + tests/ui-smoke/**/diff.png + tests/ui-smoke/**/diff-abs.png + if-no-files-found: ignore + retention-days: 14 - name: Verify no untracked or modified files after test run: | .github/scripts/verify-clean-repo.sh @@ -111,7 +123,19 @@ jobs: - name: Test run: | set -x - scripts/rip-environment runtests -p + scripts/rip-environment runtests -p -d + - name: Upload UI smoke screenshots + if: always() + uses: actions/upload-artifact@v7 + with: + name: ui-smoke-screenshots-clang + path: | + tests/ui-smoke/**/screenshot.png + tests/ui-smoke/**/confirm.png + tests/ui-smoke/**/diff.png + tests/ui-smoke/**/diff-abs.png + if-no-files-found: ignore + retention-days: 14 - name: Verify no untracked or modified files after test run: | .github/scripts/verify-clean-repo.sh @@ -237,7 +261,7 @@ jobs: passwd -d testrunner adduser testrunner sudo chmod 0777 $(find tests/ -type d) # make test dirs world-writable for the testrunner - su -c "./scripts/runtests -p ./tests" testrunner + su -c "./scripts/runtests -p -d ./tests" testrunner - name: Verify no untracked or modified files after test run: | .github/scripts/verify-clean-repo.sh ':(exclude)VERSION' ':(exclude)debian/changelog' diff --git a/debian/control.top.in b/debian/control.top.in index 596063ea377..b174e09e326 100644 --- a/debian/control.top.in +++ b/debian/control.top.in @@ -47,6 +47,9 @@ Build-Depends: tk@TCLTK_VERSION@-dev, xvfb , x11-xserver-utils , + x11-utils , + gdb , + imagemagick , python3-opengl , python3-pyqt5 , python3-pyqt5.qsci , diff --git a/scripts/runtests.in b/scripts/runtests.in index 1ec571fe19a..833cf00c2eb 100755 --- a/scripts/runtests.in +++ b/scripts/runtests.in @@ -357,6 +357,12 @@ Usage: $P -v Show stdout and stderr (normally it's hidden). + + $P -d + Enable crash dumps: on a crashing test, point kernel.core_pattern + at a temporary dir via sudo and print a gdb backtrace of the core. + Off by default since it changes a global system setting; CI passes + it. Needs sudo, so it cannot be combined with -u. EOF } @@ -365,7 +371,7 @@ NOCLEAN=0 NOSUDO=false STOP=0 PRINT=0 -while getopts cnuvsph opt; do +while getopts cnuvsphd opt; do case "$opt" in c) CLEAN_ONLY=1 ;; n) NOCLEAN=1 ;; @@ -373,12 +379,18 @@ while getopts cnuvsph opt; do v) VERBOSE=1 ;; s) STOP=1 ;; p) PRINT=1 ;; + d) export ENABLE_CRASHDUMPS=1 ;; h|?) usage; exit 0 ;; *) usage; exit 1 ;; esac done shift $((OPTIND-1)) +if [ "${ENABLE_CRASHDUMPS:-0}" = "1" ] && $NOSUDO; then + echo "$0: -d (crash dumps) needs sudo and cannot be combined with -u" >&2 + exit 1 +fi + if [ $# -eq 0 ]; then if [ -f test.hal ] || [ -f test.sh ]; then set -- . diff --git a/tests/ui-smoke/.gitignore b/tests/ui-smoke/.gitignore index 05fea704e9d..f5726b435a5 100644 --- a/tests/ui-smoke/.gitignore +++ b/tests/ui-smoke/.gitignore @@ -5,3 +5,11 @@ linuxcnc.err linuxcnc.pid ui-smoke.out ui-smoke.err +# confirm.png is the per-run shot; diff.png and diff-abs.png are its +# comparison to the committed reference.png (which IS tracked, so it is +# not listed here). +screenshot.png +confirm.png +ui-smoke-qt.png +diff.png +diff-abs.png diff --git a/tests/ui-smoke/README b/tests/ui-smoke/README index eb9ea8ef0ff..3a816fdcdb0 100644 --- a/tests/ui-smoke/README +++ b/tests/ui-smoke/README @@ -19,6 +19,35 @@ Shared helpers live in _lib/: checkresult.sh shared pass/fail predicate skip-if-missing.sh shared skip predicate +Failure diagnostics (failure path only, no cost on a green run): + crashdump.sh arms a core dump and prints a native backtrace if a GUI + segfaults (the Qt/dbus/GL frames PYTHONFAULTHANDLER misses) + screenshot.sh photographs the Xvfb root window before teardown. On a + failure it captures the cause (a GUI hung on a blocking + modal leaves no core and no traceback); on a clean Phase 2 + run it captures confirm.png, the GUI in its post-movement + idle state (final DRO / toolpath) for visual confirmation. + CI uploads both as build artifacts (ui-smoke-screenshots-* + in ci.yml). + compare.sh on a clean run, compares confirm.png to the committed + known-good reference.png (ImageMagick) and writes a + highlighted diff.png, also uploaded as an artifact. + +Reference images: + reference.png committed per-GUI known-good shot (in a --run-program test + directory); diff.png is confirm.png compared against it. + The comparison NEVER fails a test: freetype/font versions + differ across distros, so some drift is expected. The diff + is only recorded, not used to gate the test. + + Create or refresh the references on a built run-in-place tree: + . scripts/rip-environment + tests/ui-smoke/_lib/make-references.sh # all run-program GUIs + tests/ui-smoke/_lib/make-references.sh axis # just one + This sets UI_SMOKE_UPDATE_REFERENCE=1, which makes compare.sh save each + clean confirm shot as that test's reference.png. Review the PNGs before + committing; they are baselines from the generating machine's fonts. + Skip vs fail policy: the only condition we skip on is xvfb-run absence (rare local dev env). Python and gi typelib deps the GUIs need are declared in debian/control under !nocheck so apt-get build-dep diff --git a/tests/ui-smoke/_lib/compare.sh b/tests/ui-smoke/_lib/compare.sh new file mode 100644 index 00000000000..134f91e5e4f --- /dev/null +++ b/tests/ui-smoke/_lib/compare.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# Known-good image comparison for the UI smoke confirm shots. Complements +# screenshot.sh: that grabs confirm.png on a clean run; this compares it to +# a committed reference.png and writes two visual diffs. Like screenshot.sh +# it carries no state and is a logged no-op whenever it cannot run, so it can +# never turn a pass into a fail. +# +# Two diff images, both at 0% fuzz so nothing is hidden: +# diff.png red highlight over a faded copy of the reference (where it +# changed, ImageMagick "compare" style) +# diff-abs.png absolute per-channel difference |b - a|, black where equal +# and bright where it changed (unbiased, magnitude preserved) +# +# Policy: we never fail a test on the image difference. freetype/font +# versions differ across distros, so some drift is expected; the diffs are +# here to record what changed, not to gate. The function always returns 0. +# +# Local "make a known-good image" workflow: run a test with +# UI_SMOKE_UPDATE_REFERENCE=1 (see make-references.sh) and the freshly +# captured shot is saved as the committed reference instead of compared. + +# Set IM_COMPARE and IM_CONVERT to the IM7 or IM6 entry points; return 1 +# if ImageMagick is absent. +_im_tools() { + if command -v magick >/dev/null 2>&1; then + IM_COMPARE="magick compare"; IM_CONVERT="magick" + elif command -v compare >/dev/null 2>&1; then + IM_COMPARE="compare"; IM_CONVERT="convert" + else + return 1 + fi +} + +# compare_to_reference +# Compare the captured shot to the committed reference, writing a red-highlight +# diff and an absolute-difference diff. Always returns 0. +compare_to_reference() { + shot="$1" + reference="$2" + diff="$3" + diff_abs="$4" + + if [ ! -s "$shot" ]; then + echo "compare: no shot at $shot, skipping" + return 0 + fi + + # Update mode: adopt this shot as the new known-good reference. + if [ "${UI_SMOKE_UPDATE_REFERENCE:-}" = "1" ]; then + if cp -f "$shot" "$reference"; then + echo "compare: saved reference $reference (UI_SMOKE_UPDATE_REFERENCE=1)" + else + echo "compare: failed to save reference $reference" + fi + return 0 + fi + + if [ ! -s "$reference" ]; then + echo "compare: no reference at $reference yet, skipping (run with UI_SMOKE_UPDATE_REFERENCE=1 to create one)" + return 0 + fi + + _im_tools || { + echo "compare: no ImageMagick available, skipping" + return 0 + } + + # Red-highlight diff at 0% fuzz: -metric AE counts every differing pixel + # (interpretable), and the same call writes diff.png. compare exits 0 + # (identical), 1 (differ) or 2 (error, e.g. the shots are different + # sizes). We log the outcome and always succeed. + metric=$($IM_COMPARE -metric AE -fuzz 0% "$reference" "$shot" "$diff" 2>&1) + rc=$? + case "$rc" in + 0) echo "compare: $shot matches $reference (AE=$metric)" ;; + 1) echo "compare: $shot differs from $reference (AE=$metric differing pixels); diff at $diff" ;; + *) echo "compare: could not compare $shot to $reference (rc=$rc): $metric" ;; + esac + + # Absolute-difference diff |b - a|: black where equal, bright where it + # changed. Unbiased (direction does not matter) and keeps magnitude. + if $IM_CONVERT "$reference" "$shot" -compose difference -composite "$diff_abs" 2>/dev/null; then + echo "compare: absolute-difference diff at $diff_abs" + else + echo "compare: could not write absolute-difference diff $diff_abs" + fi + return 0 +} diff --git a/tests/ui-smoke/_lib/crashdump.sh b/tests/ui-smoke/_lib/crashdump.sh index dd8b84894ce..967480f1b50 100644 --- a/tests/ui-smoke/_lib/crashdump.sh +++ b/tests/ui-smoke/_lib/crashdump.sh @@ -1,45 +1,66 @@ #!/bin/bash -# Native crash capture for the UI smoke launchers. A GUI segfault is the -# failure these tests most need to explain, and it lands in C/C++ (Qt, -# dbus, GL) where PYTHONFAULTHANDLER stops at the event-loop frame. Arm a -# core dump before launch; after the run, if the GUI left a core, print a -# native backtrace into the log so CI shows the faulting frame directly. -# Source with LIB_DIR set; runs only on the failure path, so green runs -# pay nothing. +# Native crash capture for the UI smoke launchers. A GUI segfault lands in +# C/C++ (Qt, dbus, GL); PYTHONFAULTHANDLER (set in launch-env.sh) prints a +# Python traceback to linuxcnc.err naming the frame that called in, which +# is the reliable, environment-independent crash signal and is surfaced in +# every failure log. This helper adds a best-effort native backtrace on +# top, but only when runtests is given -d (ENABLE_CRASHDUMPS=1): arm a core +# dump before launch, and after the run, if a readable core from this run is +# present, gdb-print its backtrace. The core only materialises when we can +# point kernel.core_pattern at a writable dir, which needs root, so we do it +# via sudo (CI has passwordless sudo; the suite never runs as root since +# linuxcnc refuses to). It is opt-in because that sudo changes a global +# system setting (runtests rejects -d together with -u). Without -d the +# native backtrace is off and only the Python traceback remains. Source with +# LIB_DIR set; runs only on the failure path, so green runs pay nothing. crashdump_arm() { + # Off unless runtests was given -d: arming changes a global system + # setting (kernel.core_pattern) via sudo, so it is opt-in. The Python + # faulthandler traceback does not depend on this and is always present. + [ "${ENABLE_CRASHDUMPS:-0}" = "1" ] || return 0 CORE_DIR="$(mktemp -d -t ui-smoke-cores.XXXXXX)" export CORE_DIR ulimit -c unlimited 2>/dev/null || true - # core_pattern is global; only set it if already root. Never sudo: - # the suite must run unattended. Non-root falls back to a cwd "core". - if [ "$(id -u)" = 0 ]; then - sysctl -w "kernel.core_pattern=$CORE_DIR/core.%e.%p" >/dev/null 2>&1 || true - fi + # Point the global core_pattern at our writable dir so the kernel writes + # a core we can read. -d implies sudo is allowed (runtests rejects -d with + # -u), and tests never run as root, so no id check; if sudo still fails the + # core lands wherever the system pattern says and we fall back to the + # Python traceback. + sudo sysctl -w "kernel.core_pattern=$CORE_DIR/core.%e.%p" >/dev/null 2>&1 || true } crashdump_report() { + [ "${ENABLE_CRASHDUMPS:-0}" = "1" ] || return 0 [ -n "${CORE_DIR:-}" ] || return 0 - local core - # shellcheck disable=SC2012 # mktemp dir, no odd filenames - core=$(ls -t "$CORE_DIR"/core* ./core* /tmp/core* 2>/dev/null | head -1) - if [ -n "$core" ]; then + local c core="" + # Only trust a core we know is from this run and can actually read: + # one the kernel wrote into our fresh CORE_DIR (root path, where we set + # core_pattern), or a relative "core" in the cwd that postdates arming. + # A broad /tmp glob would pick up a stale or foreign core (often root- + # owned), and gdb would just print "Permission denied". + for c in "$CORE_DIR"/core*; do + [ -e "$c" ] && [ -r "$c" ] && { core="$c"; break; } + done + if [ -z "$core" ]; then + for c in ./core*; do + [ -e "$c" ] && [ -r "$c" ] && [ "$c" -nt "$CORE_DIR" ] && { core="$c"; break; } + done + fi + if [ -n "$core" ] && command -v gdb >/dev/null 2>&1; then echo "=== crash: native backtrace ($core) ===" - # gdb reads the core; pull it in if missing, only when root. - if ! command -v gdb >/dev/null 2>&1 && [ "$(id -u)" = 0 ]; then - apt-get install -y -q gdb >/dev/null 2>&1 || true - fi - if command -v gdb >/dev/null 2>&1; then - # "bt" first: gdb auto-selects the faulting thread on a SIGSEGV - # core. "thread apply all bt" after gives the rest. - gdb -batch -nx \ - -ex "bt" \ - -ex "echo \n=== all threads ===\n" \ - -ex "thread apply all bt" \ - "$(command -v python3)" "$core" 2>&1 | head -400 - else - echo "(gdb unavailable; core left at $core)" - fi + # "bt" first: gdb auto-selects the faulting thread on a SIGSEGV + # core. "thread apply all bt" after gives the rest. + gdb -batch -nx \ + -ex "bt" \ + -ex "echo \n=== all threads ===\n" \ + -ex "thread apply all bt" \ + "$(command -v python3)" "$core" 2>&1 | head -400 + else + # No readable core (the common non-root case). The Python + # faulthandler traceback in linuxcnc.err already names the crash + # site; the native backtrace is only a best-effort extra. + echo "=== crash: no readable core dump; see the Python traceback in linuxcnc.err above ===" fi rm -rf "$CORE_DIR" } diff --git a/tests/ui-smoke/_lib/drive.py b/tests/ui-smoke/_lib/drive.py index ad7e89a01fe..35deb1d3d7c 100755 --- a/tests/ui-smoke/_lib/drive.py +++ b/tests/ui-smoke/_lib/drive.py @@ -32,6 +32,13 @@ # stability check catches that. STATE_STABILITY_S = 0.5 STATE_RETRY_BUDGET = 6 +# Pause after homing before requesting AUTO. gmoccapy only enables AUTO +# once it has processed the all-homed signal in its own event loop (and +# re-asserts MANUAL itself on that signal). Requesting AUTO before then is +# rejected: it bounces back to MANUAL with an "It is not possible to +# change to Auto Mode" warning. ensure_mode would retry and win, but the +# warning lingers on screen; this settle lets the GUI catch up first. +POST_HOME_SETTLE_S = 2.0 # linuxcnc launcher PID, written to linuxcnc.pid by the launcher and read # once at startup. The driver watches it so a GUI crash, which tears @@ -297,6 +304,11 @@ def run_program(cmd, stat, ngc_path, expect_delta_mm, tol, run_timeout): if not home_all(cmd, stat, timeout=60.0): return False + # Let the GUI react to the all-homed transition before requesting AUTO, + # so it does not reject the mode change (see POST_HOME_SETTLE_S). + time.sleep(POST_HOME_SETTLE_S) + stat.poll() + if not ensure_mode(cmd, stat, linuxcnc.MODE_AUTO, "MODE_AUTO"): return False diff --git a/tests/ui-smoke/_lib/gmoccapy-prepare.sh b/tests/ui-smoke/_lib/gmoccapy-prepare.sh new file mode 100644 index 00000000000..08bfaffc5ee --- /dev/null +++ b/tests/ui-smoke/_lib/gmoccapy-prepare.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# Sourced by the gmoccapy ui-smoke tests (smoke and quit) to run gmoccapy +# against a writable copy of its sim config. Sets GMOCCAPY_INI to the +# mirrored ini path; the caller then execs run-gui.sh or quit-launch.sh +# with "$GMOCCAPY_INI". Must be sourced with LIB_DIR already set. +# +# gmoccapy writes its preferences file next to the config: with no +# PREFERENCE_FILE_PATH in the ini, getiniinfo falls back to +# /.pref. CI mounts the workspace read-only for the +# runtime user, so that write raises PermissionError partway through +# __init__ (during _get_pref_data, before the MDIHistory widget's +# _hal_init runs). gmoccapy pops an error dialog and limps on in a +# half-initialised state: the interp-idle handler then hits a widget with +# no .stat and throws a second dialog. Both vanish once the config dir is +# writable. Mirror it to tmp, same fix qtdragon-prepare.sh uses. + +: "${LIB_DIR:?gmoccapy-prepare.sh must be sourced with LIB_DIR set}" + +SRC_DIR="$(cd "$LIB_DIR/../../../configs/sim/gmoccapy" && pwd)" + +WORK_DIR="$(mktemp -d -t ui-smoke-gmoccapy.XXXXXX)" +trap 'rm -rf "$WORK_DIR"' EXIT +cp -r "$SRC_DIR/." "$WORK_DIR/" + +# Seed the preference file (config dir + .pref; MACHINE=gmoccapy) +# so the first-run "Important change(s)" modal stays hidden. That dialog +# runs a nested gtk loop, so under xvfb it never gets dismissed: it sits +# on top of the UI in the confirm shot and, worse, swallows the SIGTERM +# in the quit test (the loop keeps running after main_quit). A real user +# ticks "Don't show this again" once; hide_startup_messsage replicates +# that. The triple-s key matches gmoccapy's own (sic). +cat >"$WORK_DIR/gmoccapy.pref" <<'PREF' +[DEFAULT] +hide_startup_messsage = 99 +PREF + +# Consumed by the sourcing test.sh, which execs the launcher with it. +# shellcheck disable=SC2034 +GMOCCAPY_INI="$WORK_DIR/gmoccapy.ini" diff --git a/tests/ui-smoke/_lib/launch-env.sh b/tests/ui-smoke/_lib/launch-env.sh index 110319195be..0de0dca2ac3 100644 --- a/tests/ui-smoke/_lib/launch-env.sh +++ b/tests/ui-smoke/_lib/launch-env.sh @@ -29,3 +29,10 @@ export SDL_AUDIODRIVER=dummy # names the line; for a C/C++ crash (Qt, dbus, GL) it shows the Python # frame that called in. The native side is captured by crashdump.sh. export PYTHONFAULTHANDLER=1 + +# Xvfb virtual screen for the launchers' xvfb-run. There is no window +# manager under xvfb-run, so a GUI's maximize() is a no-op and it renders +# at its natural size; a screen smaller than the window clips the grab +# (the failure/confirm screenshot then misses panels). 1920x1080 fits +# every sim GUI so the whole window is captured. +export UI_SMOKE_XVFB_SCREEN="${UI_SMOKE_XVFB_SCREEN:-1920x1080x24}" diff --git a/tests/ui-smoke/_lib/launch.sh b/tests/ui-smoke/_lib/launch.sh index b66b4af1d00..fafefbb30ca 100755 --- a/tests/ui-smoke/_lib/launch.sh +++ b/tests/ui-smoke/_lib/launch.sh @@ -43,6 +43,12 @@ DRIVER_TIMEOUT=180 . "$LIB_DIR/crashdump.sh" crashdump_arm +# Absolute path the offscreen-Qt self-grab writes to. An offscreen GUI +# runs with its cwd at the (writable) config mirror, not the test dir, so +# a relative name would land out of reach; pin it to the test dir, which +# is this shell's cwd. Harmless for the GTK GUIs, which ignore it. +export UI_SMOKE_QT_SHOT="$PWD/ui-smoke-qt.png" + # Export the per-invocation values so the inner bash -c receives them # as proper env vars (avoids embedding paths into the inner script # via quoting, which breaks on apostrophes / spaces). @@ -52,7 +58,7 @@ export CONFIG_INI LIB_DIR DRIVER_TIMEOUT # LIB_DIR and DRIVER_TIMEOUT are expanded by the inner bash (which sees # them via the exported env), not by the outer shell. # shellcheck disable=SC2016 -xvfb-run -a --server-args="-screen 0 1024x768x24" \ +xvfb-run -a --server-args="-screen 0 $UI_SMOKE_XVFB_SCREEN" \ timeout "$LINUXCNC_TIMEOUT" \ bash -c ' # Run linuxcnc in its own process group so we can signal the @@ -70,6 +76,39 @@ xvfb-run -a --server-args="-screen 0 1024x768x24" \ timeout "$DRIVER_TIMEOUT" python3 "$LIB_DIR/drive.py" "$@" >ui-smoke.out 2>ui-smoke.err DRIVE_RC=$? + # Optional window-fit regression check: fail if the GUI window is + # larger than the screen (controls pushed off-display). Runs only + # when the GUI came up, and folds a failure into DRIVE_RC so the + # offending window is photographed below. + if [ -n "${UI_SMOKE_FIT_CLASS:-}" ] && [ "$DRIVE_RC" -eq 0 ]; then + . "$LIB_DIR/window-fit.sh" + window_fit_check "$UI_SMOKE_FIT_CLASS" || DRIVE_RC=1 + fi + + # Photograph the root window before teardown, while DISPLAY is the + # Xvfb server and the GUI is still up. On failure the picture shows + # the cause (a hung GUI on a blocking modal leaves no core for + # crashdump.sh and no Python traceback). On a clean Phase 2 run it + # is a confirmation shot of the GUI in its post-movement idle state + # (final DRO / toolpath), so a reviewer can eyeball the result; the + # confirm grab settles until the UI stops changing first. + . "$LIB_DIR/screenshot.sh" + if [ "$DRIVE_RC" -ne 0 ]; then + screenshot_grab screenshot.png + else + case " $* " in + *" --run-program "*) + # Wait for the UI to stop changing before keeping the + # shot, so a slow startup is not captured half-built. + screenshot_grab_settled confirm.png + # Compare the confirm shot to the committed known-good + # reference and write the diffs. Never fails the test. + . "$LIB_DIR/compare.sh" + compare_to_reference confirm.png reference.png diff.png diff-abs.png + ;; + esac + fi + # Clean shutdown: GUI-specific quit first (lets linuxcnc end # its own SIGTERM trap run Cleanup which unloads halrun and # reaps shared memory). axis-remote works only for axis but is @@ -109,4 +148,11 @@ echo "=== ui-smoke.err ===" # If the GUI dumped a core, print its native backtrace. crashdump_report +# Note any screenshot so the CI artifact step and reviewer know it is +# there to download: screenshot.png on failure, confirm.png on a clean run. +[ -f screenshot.png ] && echo "=== screenshot: $TEST_DIR/screenshot.png ===" +[ -f confirm.png ] && echo "=== confirm: $TEST_DIR/confirm.png ===" +[ -f diff.png ] && echo "=== diff: $TEST_DIR/diff.png ===" +[ -f diff-abs.png ] && echo "=== diff-abs: $TEST_DIR/diff-abs.png ===" + exit "$RC" diff --git a/tests/ui-smoke/_lib/make-references.sh b/tests/ui-smoke/_lib/make-references.sh new file mode 100755 index 00000000000..c8cd2136e25 --- /dev/null +++ b/tests/ui-smoke/_lib/make-references.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# Generate (or refresh) the committed known-good reference.png images by +# running the ui-smoke tests with UI_SMOKE_UPDATE_REFERENCE=1, which makes +# compare.sh save each clean confirm shot as that test's reference.png +# instead of comparing against it. +# +# Run from a built run-in-place tree with the rip environment sourced: +# . scripts/rip-environment +# tests/ui-smoke/_lib/make-references.sh # all run-program GUIs +# tests/ui-smoke/_lib/make-references.sh axis # just one +# +# Only the GUIs that capture a confirm shot (the --run-program tests) have a +# reference. Review the resulting PNGs before committing; they are baselines +# from this machine's fonts and will drift on other distros (which is why the +# comparison never fails a test, only records a diff). + +set -u + +LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SMOKE_DIR="$(cd "$LIB_DIR/.." && pwd)" +ROOT="$(cd "$SMOKE_DIR/../.." && pwd)" + +if ! command -v runtests >/dev/null 2>&1; then + echo "make-references: 'runtests' not found; source scripts/rip-environment first" >&2 + exit 1 +fi + +# GUIs whose tests grab a confirm shot (i.e. run a program). +GUIS=("$@") +if [ "${#GUIS[@]}" -eq 0 ]; then + GUIS=(axis gmoccapy touchy qtdragon) +fi + +export UI_SMOKE_UPDATE_REFERENCE=1 + +rc=0 +for gui in "${GUIS[@]}"; do + dir="$SMOKE_DIR/$gui" + if [ ! -x "$dir/test.sh" ]; then + echo "make-references: no test at $dir, skipping" + continue + fi + echo "=== make-references: $gui ===" + runtests "$dir" || rc=1 + if [ -s "$dir/reference.png" ]; then + echo "make-references: wrote $dir/reference.png" + else + echo "make-references: WARNING no reference.png produced for $gui" >&2 + rc=1 + fi +done + +exit "$rc" diff --git a/tests/ui-smoke/_lib/qtdragon-prepare.sh b/tests/ui-smoke/_lib/qtdragon-prepare.sh index 19741978d73..19404a998db 100644 --- a/tests/ui-smoke/_lib/qtdragon-prepare.sh +++ b/tests/ui-smoke/_lib/qtdragon-prepare.sh @@ -71,6 +71,73 @@ class _BlockFinder(MetaPathFinder): return None sys.meta_path.insert(0, _BlockFinder()) + + +# Native screenshot for the offscreen Qt platform. An X root grab gets a +# black frame (offscreen never draws to the X server), so the launcher +# signals this process instead: on SIGUSR1 we grab the main qtvcp window +# to ui-smoke-qt.png in the cwd (the test dir). The grab is deferred into +# the Qt event loop via singleShot so it never runs in signal context. +import os +import signal + +def _ui_smoke_find_main(app): + target, best = None, -1 + for w in app.topLevelWidgets(): + if not w.isVisible(): + continue + area = w.width() * w.height() + if area > best: + target, best = w, area + return target + +def _ui_smoke_normalize(target): + # Pin qtdragon's startup nondeterminism before the grab: re-fit the + # preview zoom at the now-final widget size and scroll the gcode view + # to the top. + from qtpy.QtWidgets import QWidget + for w in target.findChildren(QWidget): + try: + if hasattr(w, 'set_current_view') and hasattr(w, 'current_view'): + w.set_current_view() + elif hasattr(w, 'ensureLineVisible') and hasattr(w, 'verticalScrollBar'): + w.verticalScrollBar().setValue(0) + except Exception: + pass + +def _ui_smoke_grab(): + try: + from qtpy.QtWidgets import QApplication + from qtpy.QtCore import QTimer + app = QApplication.instance() + if app is None: + return + target = _ui_smoke_find_main(app) + if target is None: + return + _ui_smoke_normalize(target) + def _save(): + try: + out = os.environ.get('UI_SMOKE_QT_SHOT', 'ui-smoke-qt.png') + target.grab().save(out) + except Exception: + pass + # let the re-fit repaint before grabbing + QTimer.singleShot(250, _save) + except Exception: + pass + +def _ui_smoke_on_sigusr1(signum, frame): + try: + from qtpy.QtCore import QTimer + QTimer.singleShot(0, _ui_smoke_grab) + except Exception: + pass + +try: + signal.signal(signal.SIGUSR1, _ui_smoke_on_sigusr1) +except Exception: + pass PY export PYTHONPATH="$SHIM_DIR${PYTHONPATH:+:$PYTHONPATH}" diff --git a/tests/ui-smoke/_lib/quit-launch.sh b/tests/ui-smoke/_lib/quit-launch.sh index ce6a172f8f7..c374c22e83d 100755 --- a/tests/ui-smoke/_lib/quit-launch.sh +++ b/tests/ui-smoke/_lib/quit-launch.sh @@ -43,10 +43,14 @@ QUIT_GRACE=15 . "$LIB_DIR/crashdump.sh" crashdump_arm +# Absolute path the offscreen-Qt self-grab writes to (the test dir, this +# shell's cwd); see launch.sh for why a relative name would miss. +export UI_SMOKE_QT_SHOT="$PWD/ui-smoke-qt.png" + export CONFIG_INI LIB_DIR DRIVER_TIMEOUT GUI_MATCH QUIT_GRACE # shellcheck disable=SC2016 -xvfb-run -a --server-args="-screen 0 1024x768x24" \ +xvfb-run -a --server-args="-screen 0 $UI_SMOKE_XVFB_SCREEN" \ timeout "$LINUXCNC_TIMEOUT" \ bash -c ' setsid linuxcnc -r "$CONFIG_INI" >linuxcnc.out 2>linuxcnc.err & @@ -95,6 +99,11 @@ xvfb-run -a --server-args="-screen 0 1024x768x24" \ if kill -0 "$GUI_PID" 2>/dev/null; then echo "UI_SMOKE_QUIT_FAIL: GUI (pid $GUI_PID) still alive ${QUIT_GRACE}s after SIGTERM" + # A GUI that absorbs SIGTERM is usually blocked on a modal it + # cannot dismiss headless. Photograph it before teardown so the + # offending dialog is visible. The GUI is still up here. + . "$LIB_DIR/screenshot.sh" + screenshot_grab screenshot.png RC=1 else echo "UI_SMOKE_QUIT_OK: GUI exited ${waited}s after SIGTERM" @@ -126,4 +135,7 @@ echo "=== ui-smoke.err ===" # If the GUI dumped a core, print its native backtrace. crashdump_report +# Note any failure screenshot for the CI artifact step and reviewer. +[ -f screenshot.png ] && echo "=== screenshot: $TEST_DIR/screenshot.png ===" + exit "$RC" diff --git a/tests/ui-smoke/_lib/screenshot.sh b/tests/ui-smoke/_lib/screenshot.sh new file mode 100644 index 00000000000..45cc70b1ba3 --- /dev/null +++ b/tests/ui-smoke/_lib/screenshot.sh @@ -0,0 +1,143 @@ +#!/bin/bash +# Screen capture for the UI smoke launchers. Complements crashdump.sh: +# that one fires only on a segfault, but a GUI can also fail by hanging +# (a modal dialog blocking headless startup, a wedged event loop) with no +# core and no Python traceback. A picture of the root window at failure +# time shows what is actually on screen. Source with no state needed; the +# grab is a no-op (with a logged reason) when there is no display or no +# grabber, so it can never turn a pass into a fail. +# +# Must be called from inside the xvfb-run subshell, where DISPLAY points +# at the Xvfb server and the GUI window still exists. CI uploads the PNG +# as a build artifact (see .github/workflows/ci.yml). + +screenshot_grab() { + out="$1" + # Offscreen Qt (qtdragon) renders in memory, never to the X server, so + # an X root grab gets a black frame. Ask the GUI to grab itself + # instead: the qtdragon shim installs a SIGUSR1 handler that saves its + # top-level window to ui-smoke-qt.png in the test dir. + if [ "${QT_QPA_PLATFORM:-}" = "offscreen" ]; then + screenshot_grab_qt "$out" + return 0 + fi + if [ -z "${DISPLAY:-}" ]; then + echo "screenshot: no DISPLAY set, skipping $out" + return 0 + fi + # ImageMagick's import grabs X11 directly with no xwd dependency and + # is the grabber present in the CI image. Fall back to xwd|convert for + # local dev boxes that have xwd but not import. + if command -v import >/dev/null 2>&1; then + if import -window root "$out" 2>/dev/null; then + echo "screenshot: wrote $out" + else + echo "screenshot: import failed for $out" + fi + elif command -v xwd >/dev/null 2>&1 && command -v convert >/dev/null 2>&1; then + if xwd -root -display "$DISPLAY" 2>/dev/null | convert xwd:- "$out" 2>/dev/null; then + echo "screenshot: wrote $out" + else + echo "screenshot: xwd|convert failed for $out" + fi + else + echo "screenshot: no grabber (import or xwd) available, skipping $out" + fi + return 0 +} + +# Native grab for an offscreen Qt GUI. Find the qtvcp python process, +# SIGUSR1 it, and wait for the shim's handler to drop ui-smoke-qt.png in +# the test dir (the GUI's cwd), then move it to $out. No-op (logged) if +# the GUI or the grab is not found, so it can never fail a test. +screenshot_grab_qt() { + out="$1" + # The GUI shim saves to this absolute path (set by the launcher), + # since the offscreen GUI's cwd is the config mirror, not the test dir. + shot="${UI_SMOKE_QT_SHOT:-ui-smoke-qt.png}" + rm -f "$shot" + pid="" + for p in $(pgrep -f "qtvcp" 2>/dev/null); do + arg0=$(tr '\0' '\n' <"/proc/$p/cmdline" 2>/dev/null | head -1) + case "$(basename "$arg0" 2>/dev/null)" in + python*) pid="$p"; break ;; + esac + done + if [ -z "$pid" ]; then + echo "screenshot: qtvcp process not found, skipping $out" + return 0 + fi + kill -USR1 "$pid" 2>/dev/null || true + waited=0 + while [ "$waited" -lt 10 ]; do + [ -s "$shot" ] && break + sleep 0.5 + waited=$((waited + 1)) + done + if [ -s "$shot" ]; then + mv -f "$shot" "$out" + echo "screenshot: wrote $out (qt native grab)" + else + echo "screenshot: qt native grab produced no file for $out" + fi + return 0 +} + +# Differing-pixel count between two PNGs via ImageMagick; empty if neither +# entry point is present (the settle loop then just keeps the last grab). +_screenshot_ae() { + if command -v magick >/dev/null 2>&1; then + magick compare -metric AE "$1" "$2" null: 2>&1 + elif command -v compare >/dev/null 2>&1; then + compare -metric AE "$1" "$2" null: 2>&1 + fi +} + +# Confirm-shot grab that waits for the UI to stop changing before keeping it. +# A slow CI startup can leave a GUI half-built (missing widgets, program not +# loaded, a slider still syncing to the trajectory limit); a single grab can +# capture that. Re-grab until two frames in a row match within a threshold +# (which absorbs preview anti-aliasing and a ticking clock) or a timeout +# hits, then keep the last frame. Offscreen Qt normalizes itself in the +# grab shim, so it just grabs once. +screenshot_grab_settled() { + # Distinct from screenshot_grab's own "out": it is called in the loop + # below and would otherwise clobber our destination path. + settle_dest="$1" + if [ "${QT_QPA_PLATFORM:-}" = "offscreen" ]; then + screenshot_grab "$settle_dest" + return 0 + fi + settle_dir="$(mktemp -d -t ui-smoke-settle.XXXXXX 2>/dev/null)" || { + screenshot_grab "$settle_dest" + return 0 + } + settle_prev="$settle_dir/prev.png" + settle_cur="$settle_dir/cur.png" + settle_thresh="${UI_SMOKE_SETTLE_AE:-400}" + settle_tries="${UI_SMOKE_SETTLE_TRIES:-25}" + settle_stable=0 + while [ "$settle_tries" -gt 0 ]; do + screenshot_grab "$settle_cur" >/dev/null 2>&1 + if [ -s "$settle_cur" ] && [ -s "$settle_prev" ]; then + settle_ae="$(_screenshot_ae "$settle_prev" "$settle_cur")" + case "$settle_ae" in + ''|*[!0-9]*) settle_stable=0 ;; + *) + if [ "$settle_ae" -le "$settle_thresh" ]; then + settle_stable=$((settle_stable + 1)) + [ "$settle_stable" -ge 2 ] && { echo "screenshot: $settle_dest settled (AE=$settle_ae)"; break; } + else + settle_stable=0 + fi + ;; + esac + fi + cp -f "$settle_cur" "$settle_prev" 2>/dev/null + settle_tries=$((settle_tries - 1)) + sleep 0.4 + done + [ -s "$settle_cur" ] && mv -f "$settle_cur" "$settle_dest" + rm -rf "$settle_dir" + return 0 +} diff --git a/tests/ui-smoke/_lib/window-fit.sh b/tests/ui-smoke/_lib/window-fit.sh new file mode 100644 index 00000000000..46717c762c6 --- /dev/null +++ b/tests/ui-smoke/_lib/window-fit.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Window-fit regression check for the UI smoke tests. A GUI whose window +# is larger than the screen pushes controls off the display with no way +# to reach them. Touchy has done this (no scrolling, so the window grows +# with its content); this guards against it coming back. Run inside the +# xvfb-run subshell, where DISPLAY and the GUI window exist. Prints a +# UI_SMOKE_FAIL line (which checkresult greps) when the window exceeds the +# screen, so a future regression fails the test instead of silently +# producing an unusable window. +# +# window_fit_check +# is matched literally against the xwininfo tree +# line, e.g. '("touchy" "Touchy")'. The first matching window is checked +# against the root window (the screen). Needs xwininfo (x11-utils). + +window_fit_check() { + pattern="$1" + if ! command -v xwininfo >/dev/null 2>&1; then + echo "UI_SMOKE_FAIL: window-fit: xwininfo not available (install x11-utils)" + return 1 + fi + root=$(xwininfo -root 2>/dev/null) + screen_w=$(echo "$root" | awk '/Width:/{print $2; exit}') + screen_h=$(echo "$root" | awk '/Height:/{print $2; exit}') + geom=$(xwininfo -root -tree 2>/dev/null | grep -F "$pattern" \ + | grep -oE '[0-9]+x[0-9]+\+[0-9]+\+[0-9]+' | head -1) + win_w=${geom%%x*} + rest=${geom#*x} + win_h=${rest%%+*} + if [ -z "$win_w" ] || [ -z "$win_h" ]; then + echo "UI_SMOKE_FAIL: window-fit: no window matching $pattern found" + return 1 + fi + if [ "$win_w" -gt "$screen_w" ] || [ "$win_h" -gt "$screen_h" ]; then + echo "UI_SMOKE_FAIL: window ${win_w}x${win_h} exceeds screen ${screen_w}x${screen_h}" + return 1 + fi + echo "window-fit OK: ${win_w}x${win_h} within ${screen_w}x${screen_h}" + return 0 +} diff --git a/tests/ui-smoke/axis/reference.png b/tests/ui-smoke/axis/reference.png new file mode 100644 index 00000000000..9a3d6c5b853 Binary files /dev/null and b/tests/ui-smoke/axis/reference.png differ diff --git a/tests/ui-smoke/gmoccapy-quit/test.sh b/tests/ui-smoke/gmoccapy-quit/test.sh index 116481f3872..58e77903df1 100755 --- a/tests/ui-smoke/gmoccapy-quit/test.sh +++ b/tests/ui-smoke/gmoccapy-quit/test.sh @@ -1,4 +1,4 @@ #!/bin/bash -exec "$(dirname "$0")/../_lib/quit-launch.sh" \ - "$(cd "$(dirname "$0")/../../../configs/sim" && pwd)/gmoccapy/gmoccapy.ini" \ - "bin/gmoccapy" +LIB_DIR="$(cd "$(dirname "$0")/../_lib" && pwd)" +. "$LIB_DIR/gmoccapy-prepare.sh" +exec "$LIB_DIR/quit-launch.sh" "$GMOCCAPY_INI" "bin/gmoccapy" diff --git a/tests/ui-smoke/gmoccapy/reference.png b/tests/ui-smoke/gmoccapy/reference.png new file mode 100644 index 00000000000..280035d0697 Binary files /dev/null and b/tests/ui-smoke/gmoccapy/reference.png differ diff --git a/tests/ui-smoke/gmoccapy/test.sh b/tests/ui-smoke/gmoccapy/test.sh index de93beaed99..63dbfee3abe 100755 --- a/tests/ui-smoke/gmoccapy/test.sh +++ b/tests/ui-smoke/gmoccapy/test.sh @@ -1,4 +1,5 @@ #!/bin/bash LIB_DIR="$(cd "$(dirname "$0")/../_lib" && pwd)" -exec "$LIB_DIR/run-gui.sh" gmoccapy/gmoccapy.ini \ +. "$LIB_DIR/gmoccapy-prepare.sh" +exec "$LIB_DIR/run-gui.sh" "$GMOCCAPY_INI" \ --run-program "$LIB_DIR/smoke.ngc" --expect-delta-mm 1,1,0 diff --git a/tests/ui-smoke/qtdragon/reference.png b/tests/ui-smoke/qtdragon/reference.png new file mode 100644 index 00000000000..eab0febaf62 Binary files /dev/null and b/tests/ui-smoke/qtdragon/reference.png differ diff --git a/tests/ui-smoke/touchy-fit/checkresult b/tests/ui-smoke/touchy-fit/checkresult new file mode 100755 index 00000000000..bbe14f5f99f --- /dev/null +++ b/tests/ui-smoke/touchy-fit/checkresult @@ -0,0 +1,2 @@ +#!/bin/bash +exec "$(dirname "$0")/../_lib/checkresult.sh" "$@" diff --git a/tests/ui-smoke/touchy-fit/skip b/tests/ui-smoke/touchy-fit/skip new file mode 100755 index 00000000000..c1c260edf05 --- /dev/null +++ b/tests/ui-smoke/touchy-fit/skip @@ -0,0 +1,2 @@ +#!/bin/bash +exec "$(dirname "$0")/../_lib/skip-if-missing.sh" diff --git a/tests/ui-smoke/touchy-fit/test.sh b/tests/ui-smoke/touchy-fit/test.sh new file mode 100755 index 00000000000..484c8fc0323 --- /dev/null +++ b/tests/ui-smoke/touchy-fit/test.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# Regression guard: touchy must fit a 1024x600 panel, the common 7" +# touchscreen size it targets. Boots touchy on that screen and fails +# if its window is larger (see window-fit.sh). Phase 1 only; no +# program is run. +LIB_DIR="$(cd "$(dirname "$0")/../_lib" && pwd)" +export UI_SMOKE_XVFB_SCREEN=1024x600x24 +export UI_SMOKE_FIT_CLASS='("touchy" "Touchy")' +exec "$LIB_DIR/run-gui.sh" touchy/touchy.ini diff --git a/tests/ui-smoke/touchy/reference.png b/tests/ui-smoke/touchy/reference.png new file mode 100644 index 00000000000..5a173dee65e Binary files /dev/null and b/tests/ui-smoke/touchy/reference.png differ