ui-smoke phase 3: screenshots, writable config mirrors, homing settle

.github/workflows/ci.yml

@@ -48,10 +48,27 @@ jobs:
     - name: Verify no untracked or modified files after build
       run: |
         .github/scripts/verify-clean-repo.sh
+    - name: Arm core dumps
+      run: |
+        set -x
+        mkdir -p /tmp/linuxcnc-ui-smoke-cores
+        sudo sysctl -w kernel.core_pattern=/tmp/linuxcnc-ui-smoke-cores/core.%e.%p
     - name: Test
       run: |
         set -x
-        scripts/rip-environment runtests -p
+        scripts/rip-environment runtests -p -d
+    - name: Upload UI smoke screenshots
+      if: always()
+      uses: actions/upload-artifact@v7
+      with:
+        name: ui-smoke-screenshots-gcc
+        path: |
+          tests/ui-smoke/**/screenshot.png
+          tests/ui-smoke/**/confirm.png
+          tests/ui-smoke/**/diff.png
+          tests/ui-smoke/**/diff-abs.png
+        if-no-files-found: ignore
+        retention-days: 14
     - name: Verify no untracked or modified files after test
       run: |
         .github/scripts/verify-clean-repo.sh
@@ -163,10 +180,27 @@ jobs:
     - name: Verify no untracked or modified files after build
       run: |
         .github/scripts/verify-clean-repo.sh
+    - name: Arm core dumps
+      run: |
+        set -x
+        mkdir -p /tmp/linuxcnc-ui-smoke-cores
+        sudo sysctl -w kernel.core_pattern=/tmp/linuxcnc-ui-smoke-cores/core.%e.%p
     - name: Test
       run: |
         set -x
-        scripts/rip-environment runtests -p
+        scripts/rip-environment runtests -p -d
+    - name: Upload UI smoke screenshots
+      if: always()
+      uses: actions/upload-artifact@v7
+      with:
+        name: ui-smoke-screenshots-clang
+        path: |
+          tests/ui-smoke/**/screenshot.png
+          tests/ui-smoke/**/confirm.png
+          tests/ui-smoke/**/diff.png
+          tests/ui-smoke/**/diff-abs.png
+        if-no-files-found: ignore
+        retention-days: 14
     - name: Verify no untracked or modified files after test
       run: |
         .github/scripts/verify-clean-repo.sh
@@ -329,7 +363,7 @@ jobs:
         passwd -d testrunner
         adduser testrunner sudo
         chmod 0777 $(find tests/ -type d) # make test dirs world-writable for the testrunner
-        su -c "./scripts/runtests -p ./tests" testrunner
+        su -c "./scripts/runtests -p -d ./tests" testrunner
     - name: Verify no untracked or modified files after test
       run: |
         .github/scripts/verify-clean-repo.sh ':(exclude)VERSION' ':(exclude)debian/changelog'

debian/control.top.in

@@ -49,6 +49,9 @@ Build-Depends:
     tk@TCLTK_VERSION@-dev,
     xvfb <!nocheck>,
     x11-xserver-utils <!nocheck>,
+    x11-utils <!nocheck>,
+    gdb <!nocheck>,
+    imagemagick <!nocheck>,
     python3-opengl <!nocheck>,
     python3-pyqt5 <!nocheck>,
     python3-pyqt5.qsci <!nocheck>,

scripts/runtests.in

@@ -357,6 +357,11 @@ Usage:
 
     $P -v
         Show stdout and stderr (normally it's hidden).
+
+    $P -d
+        Enable crash dumps: on a crashing test, print a gdb backtrace of
+        the core, collected from wherever the system put it (coredumpctl
+        or a plain-file core_pattern).  Off by default; CI passes it.
 EOF
 }
 
@@ -365,14 +370,15 @@ NOCLEAN=0
 NOSUDO=false
 STOP=0
 PRINT=0
-while getopts cnuvsph opt; do
+while getopts cnuvsphd opt; do
     case "$opt" in
     c) CLEAN_ONLY=1 ;;
     n) NOCLEAN=1 ;;
     u) NOSUDO=true ;;
     v) VERBOSE=1 ;;
     s) STOP=1 ;;
     p) PRINT=1 ;;
+    d) export ENABLE_CRASHDUMPS=1 ;;
     h|?) usage; exit 0 ;;
     *) usage; exit 1 ;;
     esac

tests/ui-smoke/.gitignore

@@ -5,3 +5,11 @@ linuxcnc.err
 linuxcnc.pid
 ui-smoke.out
 ui-smoke.err
+# confirm.png is the per-run shot; diff.png and diff-abs.png are its
+# comparison to the committed reference.png (which IS tracked, so it is
+# not listed here).
+screenshot.png
+confirm.png
+ui-smoke-qt.png
+diff.png
+diff-abs.png

tests/ui-smoke/README

@@ -19,6 +19,35 @@ Shared helpers live in _lib/:
   checkresult.sh      shared pass/fail predicate
   skip-if-missing.sh  shared skip predicate
 
+Failure diagnostics (failure path only, no cost on a green run):
+  crashdump.sh   arms a core dump and prints a native backtrace if a GUI
+                 segfaults (the Qt/dbus/GL frames PYTHONFAULTHANDLER misses)
+  screenshot.sh  photographs the Xvfb root window before teardown. On a
+                 failure it captures the cause (a GUI hung on a blocking
+                 modal leaves no core and no traceback); on a clean Phase 2
+                 run it captures confirm.png, the GUI in its post-movement
+                 idle state (final DRO / toolpath) for visual confirmation.
+                 CI uploads both as build artifacts (ui-smoke-screenshots-*
+                 in ci.yml).
+  compare.sh     on a clean run, compares confirm.png to the committed
+                 known-good reference.png (ImageMagick) and writes a
+                 highlighted diff.png, also uploaded as an artifact.
+
+Reference images:
+  reference.png  committed per-GUI known-good shot (in a --run-program test
+                 directory); diff.png is confirm.png compared against it.
+                 The comparison NEVER fails a test: freetype/font versions
+                 differ across distros, so some drift is expected. The diff
+                 is only recorded, not used to gate the test.
+
+  Create or refresh the references on a built run-in-place tree:
+      . scripts/rip-environment
+      tests/ui-smoke/_lib/make-references.sh           # all run-program GUIs
+      tests/ui-smoke/_lib/make-references.sh axis      # just one
+  This sets UI_SMOKE_UPDATE_REFERENCE=1, which makes compare.sh save each
+  clean confirm shot as that test's reference.png. Review the PNGs before
+  committing; they are baselines from the generating machine's fonts.
+
 Skip vs fail policy: the only condition we skip on is xvfb-run absence
 (rare local dev env). Python and gi typelib deps the GUIs need are
 declared in debian/control under !nocheck so apt-get build-dep

tests/ui-smoke/_lib/compare.sh

@@ -0,0 +1,88 @@
+#!/bin/bash
+# Known-good image comparison for the UI smoke confirm shots. Complements
+# screenshot.sh: that grabs confirm.png on a clean run; this compares it to
+# a committed reference.png and writes two visual diffs. Like screenshot.sh
+# it carries no state and is a logged no-op whenever it cannot run, so it can
+# never turn a pass into a fail.
+#
+# Two diff images, both at 0% fuzz so nothing is hidden:
+#   diff.png      red highlight over a faded copy of the reference (where it
+#                 changed, ImageMagick "compare" style)
+#   diff-abs.png  absolute per-channel difference |b - a|, black where equal
+#                 and bright where it changed (unbiased, magnitude preserved)
+#
+# Policy: we never fail a test on the image difference. freetype/font
+# versions differ across distros, so some drift is expected; the diffs are
+# here to record what changed, not to gate. The function always returns 0.
+#
+# Local "make a known-good image" workflow: run a test with
+# UI_SMOKE_UPDATE_REFERENCE=1 (see make-references.sh) and the freshly
+# captured shot is saved as the committed reference instead of compared.
+
+# Set IM_COMPARE and IM_CONVERT to the IM7 or IM6 entry points; return 1
+# if ImageMagick is absent.
+_im_tools() {
+    if command -v magick >/dev/null 2>&1; then
+        IM_COMPARE="magick compare"; IM_CONVERT="magick"
+    elif command -v compare >/dev/null 2>&1; then
+        IM_COMPARE="compare"; IM_CONVERT="convert"
+    else
+        return 1
+    fi
+}
+
+# compare_to_reference <shot> <reference> <diff> <diff_abs>
+# Compare the captured shot to the committed reference, writing a red-highlight
+# diff and an absolute-difference diff. Always returns 0.
+compare_to_reference() {
+    shot="$1"
+    reference="$2"
+    diff="$3"
+    diff_abs="$4"
+
+    if [ ! -s "$shot" ]; then
+        echo "compare: no shot at $shot, skipping"
+        return 0
+    fi
+
+    # Update mode: adopt this shot as the new known-good reference.
+    if [ "${UI_SMOKE_UPDATE_REFERENCE:-}" = "1" ]; then
+        if cp -f "$shot" "$reference"; then
+            echo "compare: saved reference $reference (UI_SMOKE_UPDATE_REFERENCE=1)"
+        else
+            echo "compare: failed to save reference $reference"
+        fi
+        return 0
+    fi
+
+    if [ ! -s "$reference" ]; then
+        echo "compare: no reference at $reference yet, skipping (run with UI_SMOKE_UPDATE_REFERENCE=1 to create one)"
+        return 0
+    fi
+
+    _im_tools || {
+        echo "compare: no ImageMagick available, skipping"
+        return 0
+    }
+
+    # Red-highlight diff at 0% fuzz: -metric AE counts every differing pixel
+    # (interpretable), and the same call writes diff.png. compare exits 0
+    # (identical), 1 (differ) or 2 (error, e.g. the shots are different
+    # sizes). We log the outcome and always succeed.
+    metric=$($IM_COMPARE -metric AE -fuzz 0% "$reference" "$shot" "$diff" 2>&1)
+    rc=$?
+    case "$rc" in
+        0) echo "compare: $shot matches $reference (AE=$metric)" ;;
+        1) echo "compare: $shot differs from $reference (AE=$metric differing pixels); diff at $diff" ;;
+        *) echo "compare: could not compare $shot to $reference (rc=$rc): $metric" ;;
+    esac
+
+    # Absolute-difference diff |b - a|: black where equal, bright where it
+    # changed. Unbiased (direction does not matter) and keeps magnitude.
+    if $IM_CONVERT "$reference" "$shot" -compose difference -composite "$diff_abs" 2>/dev/null; then
+        echo "compare: absolute-difference diff at $diff_abs"
+    else
+        echo "compare: could not write absolute-difference diff $diff_abs"
+    fi
+    return 0
+}

tests/ui-smoke/_lib/crashdump.sh

@@ -1,45 +1,68 @@
 #!/bin/bash
-# Native crash capture for the UI smoke launchers. A GUI segfault is the
-# failure these tests most need to explain, and it lands in C/C++ (Qt,
-# dbus, GL) where PYTHONFAULTHANDLER stops at the event-loop frame. Arm a
-# core dump before launch; after the run, if the GUI left a core, print a
-# native backtrace into the log so CI shows the faulting frame directly.
-# Source with LIB_DIR set; runs only on the failure path, so green runs
-# pay nothing.
+# Native crash capture for the UI smoke launchers. A GUI segfault lands in
+# C/C++ (Qt, dbus, GL); PYTHONFAULTHANDLER (set in launch-env.sh) prints a
+# Python traceback to linuxcnc.err naming the frame that called in, which is
+# the reliable, environment-independent crash signal and is surfaced in every
+# failure log. This helper adds a best-effort native backtrace on top when
+# runtests is given -d (ENABLE_CRASHDUMPS=1): raise the core size limit
+# before launch, and after the run, if a readable core from this run is
+# found, gdb-print its backtrace. Cores are collected from wherever the
+# system puts them: systemd-coredump via coredumpctl, or a plain-file
+# kernel.core_pattern (the CI workflow points it at CORE_DIR with a job-level
+# sudo sysctl on the disposable runner; local runs often just get ./core).
+# No sudo and no global system changes here. Source with LIB_DIR set; the
+# report runs only on the failure path, so green runs pay nothing.
 
 crashdump_arm() {
-    CORE_DIR="$(mktemp -d -t ui-smoke-cores.XXXXXX)"
+    # Off unless runtests was given -d. The Python faulthandler traceback
+    # does not depend on this and is always present.
+    [ "${ENABLE_CRASHDUMPS:-0}" = "1" ] || return 0
+    # CORE_DIR is where the CI workflow's core_pattern writes; it also
+    # receives a core extracted via coredumpctl.
+    CORE_DIR="${UI_SMOKE_CORE_DIR:-/tmp/linuxcnc-ui-smoke-cores}"
+    mkdir -p "$CORE_DIR" 2>/dev/null || true
     export CORE_DIR
     ulimit -c unlimited 2>/dev/null || true
-    # core_pattern is global; only set it if already root. Never sudo:
-    # the suite must run unattended. Non-root falls back to a cwd "core".
-    if [ "$(id -u)" = 0 ]; then
-        sysctl -w "kernel.core_pattern=$CORE_DIR/core.%e.%p" >/dev/null 2>&1 || true
-    fi
+    crashdump_arm_time=$(date +%s)
 }
 
 crashdump_report() {
+    [ "${ENABLE_CRASHDUMPS:-0}" = "1" ] || return 0
     [ -n "${CORE_DIR:-}" ] || return 0
-    local core
-    # shellcheck disable=SC2012  # mktemp dir, no odd filenames
-    core=$(ls -t "$CORE_DIR"/core* ./core* /tmp/core* 2>/dev/null | head -1)
-    if [ -n "$core" ]; then
-        echo "=== crash: native backtrace ($core) ==="
-        # gdb reads the core; pull it in if missing, only when root.
-        if ! command -v gdb >/dev/null 2>&1 && [ "$(id -u)" = 0 ]; then
-            apt-get install -y -q gdb >/dev/null 2>&1 || true
-        fi
-        if command -v gdb >/dev/null 2>&1; then
-            # "bt" first: gdb auto-selects the faulting thread on a SIGSEGV
-            # core. "thread apply all bt" after gives the rest.
-            gdb -batch -nx \
-                -ex "bt" \
-                -ex "echo \n=== all threads ===\n" \
-                -ex "thread apply all bt" \
-                "$(command -v python3)" "$core" 2>&1 | head -400
-        else
-            echo "(gdb unavailable; core left at $core)"
+    local c core=""
+    # Only trust a core we know is from this run and can actually read: one
+    # in CORE_DIR, a relative "core" in the cwd that postdates arming, or
+    # one systemd-coredump logged since arming (coredumpctl needs no root
+    # for our own processes). A broad /tmp glob would pick up a stale or
+    # foreign core, and gdb would just print "Permission denied".
+    for c in "$CORE_DIR"/core*; do
+        [ -e "$c" ] && [ -r "$c" ] && { core="$c"; break; }
+    done
+    if [ -z "$core" ]; then
+        for c in ./core*; do
+            [ -e "$c" ] && [ -r "$c" ] && [ "$c" -nt "$CORE_DIR" ] && { core="$c"; break; }
+        done
+    fi
+    if [ -z "$core" ] && command -v coredumpctl >/dev/null 2>&1; then
+        if coredumpctl list --no-legend --since "@$crashdump_arm_time" python3 2>/dev/null | grep -q .; then
+            coredumpctl dump python3 --output="$CORE_DIR/core.coredumpctl" >/dev/null 2>&1 || true
+            [ -s "$CORE_DIR/core.coredumpctl" ] && core="$CORE_DIR/core.coredumpctl"
         fi
     fi
+    if [ -n "$core" ] && command -v gdb >/dev/null 2>&1; then
+        echo "=== crash: native backtrace ($core) ==="
+        # "bt" first: gdb auto-selects the faulting thread on a SIGSEGV
+        # core. "thread apply all bt" after gives the rest.
+        gdb -batch -nx \
+            -ex "bt" \
+            -ex "echo \n=== all threads ===\n" \
+            -ex "thread apply all bt" \
+            "$(command -v python3)" "$core" 2>&1 | head -400
+    else
+        # No readable core. The Python faulthandler traceback in
+        # linuxcnc.err already names the crash site; the native backtrace
+        # is only a best-effort extra.
+        echo "=== crash: no readable core dump; see the Python traceback in linuxcnc.err above ==="
+    fi
     rm -rf "$CORE_DIR"
 }