From 5abef632c3d87e1c9982e7616e034f0187068ce9 Mon Sep 17 00:00:00 2001 From: Chun-Hung Tseng Date: Tue, 1 Sep 2026 22:05:25 +0200 Subject: [PATCH 1/3] Share the qemu ssh argv between the shell lanes qemu-runner.sh and test-matrix.sh each spelled the ssh option list for the test VM, and the copies had drifted: the runner kept the peer alive for 10 s x 6 and the matrix for 15 s x 4. tests/lib/qemu-ssh.sh now holds the one list. qemu_ssh_opts fills QEMU_SSH_OPTS at call time so a caller can still wrap its ssh in timeout(1), which cannot wrap a shell function. Both lanes keep the 60 s dead-peer budget. --- tests/lib/qemu-ssh.sh | 24 ++++++++++++++++++++++++ tests/qemu-runner.sh | 15 +++++---------- tests/test-matrix.sh | 13 ++++--------- 3 files changed, 33 insertions(+), 19 deletions(-) create mode 100644 tests/lib/qemu-ssh.sh diff --git a/tests/lib/qemu-ssh.sh b/tests/lib/qemu-ssh.sh new file mode 100644 index 00000000..9ea7a40b --- /dev/null +++ b/tests/lib/qemu-ssh.sh @@ -0,0 +1,24 @@ +# Shared ssh argv for the qemu test VM. +# +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 +# shellcheck shell=bash +# timeout(1) cannot wrap a shell function, so what the callers share is the +# argv: qemu_ssh_opts fills QEMU_SSH_OPTS from QEMU_SSH_KEY and QEMU_PORT at +# call time, and each caller builds its own ssh command line around it. + +# shellcheck disable=SC2034 # Consumed by the sourcing script. +qemu_ssh_opts() +{ + QEMU_SSH_OPTS=( + -o StrictHostKeyChecking=no + -o UserKnownHostsFile=/dev/null + -o LogLevel=ERROR + -o BatchMode=yes + -o ConnectTimeout=10 + -o ServerAliveInterval=10 + -o ServerAliveCountMax=6 + -i "$QEMU_SSH_KEY" + -p "$QEMU_PORT" + ) +} diff --git a/tests/qemu-runner.sh b/tests/qemu-runner.sh index e5837fe4..097a7c98 100755 --- a/tests/qemu-runner.sh +++ b/tests/qemu-runner.sh @@ -19,6 +19,9 @@ _QR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")/.." && pwd)" _QR_FIX="${_QR_DIR}/externals/test-fixtures" +# shellcheck source=tests/lib/qemu-ssh.sh +source "${_QR_DIR}/tests/lib/qemu-ssh.sh" + QEMU_BIN="${QEMU_BIN:-qemu-system-aarch64}" QEMU_PORT="${QEMU_PORT:-2222}" QEMU_MEM="${QEMU_MEM:-2048}" @@ -174,16 +177,8 @@ qemu_start() # the suite's tolerance. _qemu_ssh_raw() { - ssh -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o LogLevel=ERROR \ - -o BatchMode=yes \ - -o ConnectTimeout=10 \ - -o ServerAliveInterval=10 \ - -o ServerAliveCountMax=6 \ - -i "$QEMU_SSH_KEY" \ - -p "$QEMU_PORT" \ - root@127.0.0.1 "$@" + qemu_ssh_opts + ssh "${QEMU_SSH_OPTS[@]}" root@127.0.0.1 "$@" } # Run a command in the VM. Any argument that is an absolute path under the host diff --git a/tests/test-matrix.sh b/tests/test-matrix.sh index 6ddea218..dfdfc1b5 100755 --- a/tests/test-matrix.sh +++ b/tests/test-matrix.sh @@ -73,6 +73,8 @@ source "${REPO_ROOT}/tests/test-config.sh" TEST_LABEL_WIDTH=45 # shellcheck source=tests/lib/test-runner.sh source "${REPO_ROOT}/tests/lib/test-runner.sh" +# shellcheck source=tests/lib/qemu-ssh.sh +source "${REPO_ROOT}/tests/lib/qemu-ssh.sh" # Globals (test-runner.sh seeds pass/fail/skip; test-matrix.sh resets them per # mode and tracks no extra counters). @@ -192,15 +194,8 @@ run_qemu() if [ "${#args[@]}" -gt 0 ]; then printf -v quoted '%q ' "${args[@]}" fi - timeout 60 ssh \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o LogLevel=ERROR \ - -o BatchMode=yes \ - -o ConnectTimeout=10 \ - -o ServerAliveInterval=15 \ - -o ServerAliveCountMax=4 \ - -i "$QEMU_SSH_KEY" -p "$QEMU_PORT" \ + qemu_ssh_opts + timeout 60 ssh "${QEMU_SSH_OPTS[@]}" \ root@127.0.0.1 "cd /mnt/host && ${quoted}" 2> /dev/null } From 3dafb7644e90a5f66e190076159538f226ae6790 Mon Sep 17 00:00:00 2001 From: Chun-Hung Tseng Date: Wed, 9 Sep 2026 00:06:51 +0200 Subject: [PATCH 2/3] Keep qemu-runner VM state across start and stop Run as separate commands, start stopped its VM on exit and stop had no record of it. start --state-file now keeps the VM and records its port and key for a later shell and its pidfile for stop --state-file, which signals the pid only while its argv still names that pidfile, since the pid may have been reused. A pid that survives SIGKILL keeps its pidfile and state file so a later stop can retry. The trap 'qemu_stop' EXIT set when sourcing qemu-runner.sh replaced test-matrix.sh's cleanup trap, so only start and exec set it now; test-matrix.sh marks the VM active before qemu_start so its own trap covers the boot. A failed start now prints the tail of qemu's output and serial console before removing the run directory, so the caller can see why the VM did not come up. Add tests/test-qemu-runner.sh to make check. --- mk/tests.mk | 7 ++- tests/qemu-runner.sh | 96 ++++++++++++++++++++++++++++----- tests/test-matrix.sh | 3 +- tests/test-qemu-runner.sh | 109 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 200 insertions(+), 15 deletions(-) create mode 100755 tests/test-qemu-runner.sh diff --git a/mk/tests.mk b/mk/tests.mk index 724bcf03..eb8a24df 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -55,7 +55,7 @@ ELFUSE_HOST_NOFILE_MIN ?= $(shell bash "$(CURDIR)/tests/test-config.sh" --host-n test-sysroot-pathmax test-sysroot-corpus \ test-sysroot-name-soak check-soak \ check-name-caseexact test-sysroot-path-matrix \ - test-usage-synopsis \ + test-usage-synopsis test-qemu-runner \ probe-volume-naming perf ## Build and run the assembly hello world test @@ -344,6 +344,7 @@ check: $(ELFUSE_BIN) $(TEST_DEPS) check-syscall-coverage check-eintr-contract ch $(call run-lane,test-rosetta-cli,rosetta CLI gating) $(call run-lane,test-bench-guardrail,hot-syscall guardrail) $(call run-lane,test-sharun,sharun launcher and probe) + $(call run-lane,test-qemu-runner,qemu-runner start and stop checks) ## Hot-syscall performance guardrail: ensure getpid, libc clock_gettime, ## and 1-byte /dev/urandom reads stay under their TODO ns/op ceilings. @@ -1170,6 +1171,10 @@ test-usage-synopsis: $(ELFUSE_BIN) test-gdbstub-host: $(BUILD_DIR)/test-gdbstub-host $(BUILD_DIR)/test-gdbstub-host +## Check qemu-runner.sh start reporting and stop identity, against stand-ins +test-qemu-runner: + @bash tests/test-qemu-runner.sh + ## Run GDB stub integration tests (LLDB <-> elfuse gdbstub) test-gdbstub: $(ELFUSE_BIN) $(TEST_DIR)/test-hello $(BUILD_DIR)/test-gdbstub-host $(call run-host-unit,test-gdbstub-host,buffered GDB session regression) diff --git a/tests/qemu-runner.sh b/tests/qemu-runner.sh index 097a7c98..bc41b0db 100755 --- a/tests/qemu-runner.sh +++ b/tests/qemu-runner.sh @@ -41,6 +41,7 @@ QEMU_SHARE_PATH="${QEMU_SHARE_PATH:-${_QR_DIR}}" _QR_PIDFILE="" _QR_LOG="" +_QR_ERR="" _QR_CTL="" # Fixture path inside the VM (always /mnt/host/). @@ -101,6 +102,20 @@ qemu_pick_cpu() esac } +# qemu_stop removes the rundir, so what explains a failed start prints first. +qemu_fail_start() +{ + echo "qemu-runner: $1" >&2 + local f + for f in "$_QR_ERR" "$_QR_LOG"; do + if [ -s "$f" ]; then + echo "qemu-runner: tail of ${f##*/}:" >&2 + tail -n 20 "$f" >&2 + fi + done + qemu_stop +} + qemu_start() { qemu_ensure_fixtures || return 1 @@ -122,6 +137,7 @@ qemu_start() rundir="$(mktemp -d -t elfuse-qemu.XXXXXX)" _QR_PIDFILE="${rundir}/qemu.pid" _QR_LOG="${rundir}/qemu-serial.log" + _QR_ERR="${rundir}/qemu.log" _QR_CTL="${rundir}/ssh-ctl" QEMU_PORT="$(qemu_pick_port)" @@ -138,7 +154,7 @@ qemu_start() -nographic -display none -no-reboot -monitor none \ -serial "file:${_QR_LOG}" \ -pidfile "$_QR_PIDFILE" \ - > /dev/null 2>&1 & + > "$_QR_ERR" 2>&1 & disown # Wait for ssh port to come up. @@ -150,8 +166,7 @@ qemu_start() sleep 1 done if ! (echo > /dev/tcp/127.0.0.1/"$QEMU_PORT") 2> /dev/null; then - echo "qemu-runner: VM did not boot within ${QEMU_BOOT_TIMEOUT}s" >&2 - qemu_stop + qemu_fail_start "VM did not boot within ${QEMU_BOOT_TIMEOUT}s" return 1 fi @@ -168,7 +183,10 @@ qemu_start() # a dedicated tmpfs, as any regular system has, so paths under /tmp map to a # resolvable st_dev. Guarded so a repeated qemu_start against a running VM # does not stack mounts. - _qemu_ssh_raw 'grep -q " /tmp tmpfs " /proc/mounts || mount -t tmpfs tmpfs /tmp' + if ! _qemu_ssh_raw 'grep -q " /tmp tmpfs " /proc/mounts || mount -t tmpfs tmpfs /tmp'; then + qemu_fail_start "could not prepare /tmp in the guest" + return 1 + fi } # Each call opens a fresh ssh connection. Avoids ControlMaster pitfalls (master @@ -205,6 +223,14 @@ qemu_stop() if [ -n "$_QR_PIDFILE" ] && [ -s "$_QR_PIDFILE" ]; then local pid pid=$(cat "$_QR_PIDFILE" 2> /dev/null) + + # A state file outlives its VM, so the pid it names may since have been + # recycled. qemu_start gives qemu this pidfile, and mktemp makes the + # path unique, so the argv is what proves the process is ours. + case " $(ps -o command= -p "${pid:-0}" 2> /dev/null) " in + *" -pidfile $_QR_PIDFILE "*) ;; + *) pid="" ;; + esac if [ -n "$pid" ] && kill -0 "$pid" 2> /dev/null; then kill "$pid" 2> /dev/null # give qemu time to exit cleanly; force-kill if it lingers @@ -213,7 +239,15 @@ qemu_stop() kill -0 "$pid" 2> /dev/null || break sleep 1 done - kill -0 "$pid" 2> /dev/null && kill -9 "$pid" 2> /dev/null + if kill -0 "$pid" 2> /dev/null; then + kill -9 "$pid" 2> /dev/null + sleep 1 + # Keep the pidfile and state so a later stop can retry. + if kill -0 "$pid" 2> /dev/null; then + echo "qemu-runner: pid $pid survived SIGKILL" >&2 + return 1 + fi + fi fi fi if [ -n "$_QR_PIDFILE" ]; then @@ -221,33 +255,69 @@ qemu_stop() fi _QR_PIDFILE="" _QR_LOG="" + _QR_ERR="" _QR_CTL="" } -# When sourced, register a cleanup trap that does not clobber the caller's -# existing trap chain. When executed directly, the EXIT trap fires on script -# exit. -trap 'qemu_stop' EXIT +qemu_write_state() +{ + mkdir -p "$(dirname "$1")" || return 1 + printf 'port=%s\nkey=%s\npidfile=%s\n' \ + "$QEMU_PORT" "$QEMU_SSH_KEY" "$_QR_PIDFILE" > "$1.tmp" && mv -f "$1.tmp" "$1" +} + +qemu_read_state() +{ + [ -s "$1" ] || { + echo "qemu-runner: no state file $1" >&2 + return 1 + } + _QR_PIDFILE="$(sed -n 's/^pidfile=//p' "$1")" + + # Restrict cleanup to the directory shape created by mktemp: ^/ rejects a + # relative path and [^/] a ".." run directory, both of which a case + # pattern's * admits. + if [[ ! "$_QR_PIDFILE" =~ ^/.*/elfuse-qemu\.[^/]+/qemu\.pid$ ]]; then + echo "qemu-runner: $1 names no qemu-runner pidfile: $_QR_PIDFILE; remove the file once the VM is gone" >&2 + return 1 + fi +} -# CLI driver: when run directly, support 'qemu-runner.sh start|exec|stop'. if [ "${BASH_SOURCE[0]:-$0}" = "$0" ]; then cmd="${1:-help}" shift || true + state_file="" + if [ "$cmd" != exec ] && [ "${1:-}" = "--state-file" ]; then + state_file="${2:?--state-file needs a path}" + shift 2 + fi case "$cmd" in start) - qemu_start + if [ -n "$state_file" ] && [ -e "$state_file" ]; then + echo "qemu-runner: $state_file names a live VM; run stop first" >&2 + exit 1 + fi + trap 'qemu_stop' EXIT + qemu_start || exit 1 + if [ -n "$state_file" ]; then + qemu_write_state "$state_file" || exit 1 + trap - EXIT + fi echo "PORT=$QEMU_PORT KEY=$QEMU_SSH_KEY" ;; exec) + trap 'qemu_stop' EXIT qemu_start qemu_exec "$@" ;; stop) - qemu_stop + [ -z "$state_file" ] || qemu_read_state "$state_file" || exit 1 + qemu_stop || exit 1 + [ -z "$state_file" ] || rm -f "$state_file" ;; *) cat << EOF -Usage: $0 +Usage: $0 Boots qemu-system-aarch64 with the test fixtures and exposes ssh. The host repo is shared into the VM at /mnt/host (read-only). diff --git a/tests/test-matrix.sh b/tests/test-matrix.sh index dfdfc1b5..3903904d 100755 --- a/tests/test-matrix.sh +++ b/tests/test-matrix.sh @@ -1273,11 +1273,12 @@ run_suite() # shellcheck disable=SC1091 . "${REPO_ROOT}/tests/qemu-runner.sh" printf "Booting qemu-system-aarch64 (Alpine minirootfs)\n" + _qemu_active=1 qemu_start || { + _qemu_active=0 echo "qemu boot failed" return 1 } - _qemu_active=1 runner="run_qemu" dyn_runner="run_qemu" ;; diff --git a/tests/test-qemu-runner.sh b/tests/test-qemu-runner.sh new file mode 100755 index 00000000..7b1da09a --- /dev/null +++ b/tests/test-qemu-runner.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash + +# test-qemu-runner.sh -- Pin qemu-runner.sh start and stop against stand-ins +# +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Usage: tests/test-qemu-runner.sh +# +# No VM boots here: a stub stands in for qemu. stop must leave a recycled pid +# alone yet terminate a process whose argv names the run's pidfile, and a failed +# start must report what the VM said before its run directory goes. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +RUNNER="$SCRIPT_DIR/qemu-runner.sh" +# shellcheck source=tests/lib/report.sh +. "$SCRIPT_DIR/lib/report.sh" + +work="$(mktemp -d)" +victims=() +cleanup() +{ + local p + for p in ${victims[@]+"${victims[@]}"}; do + kill "$p" 2> /dev/null || true + done + rm -rf "$work" +} +trap cleanup EXIT + +# qemu_read_state accepts only the directory shape mktemp gives a run. +rundir="$work/elfuse-qemu.test" +pidfile="$rundir/qemu.pid" +state="$work/qemu.state" + +fake_start() +{ + mkdir -p "$rundir" + printf '%s\n' "$1" > "$pidfile" + printf 'port=1\nkey=/dev/null\npidfile=%s\n' "$pidfile" > "$state" +} + +dead() +{ + ! kill -0 "$1" 2> /dev/null +} + +# A recycled pid: a sleep whose argv never mentions the pidfile. +sleep 300 & +bystander=$! +victims+=("$bystander") +fake_start "$bystander" +check_host "stop returns 0 for a recycled pid" bash "$RUNNER" stop --state-file "$state" +check_host "a recycled pid survives stop" kill -0 "$bystander" +check_host "stop removes the stale state file" test ! -e "$state" +check_host "stop removes the stale run directory" test ! -e "$rundir" + +# A state file that names a directory outside a run through "..". +victim="$work/victim" +mkdir -p "$rundir" "$victim" +printf 'pidfile=%s/../victim/qemu.pid\n' "$rundir" > "$state" +rc=0 +bash "$RUNNER" stop --state-file "$state" 2> /dev/null || rc=$? +check_host "stop rejects a pidfile outside a run directory" test "$rc" -eq 1 +check_host "the directory a rejected state file names survives" test -d "$victim" + +# The run's own process: its argv carries the pidfile, and it exits on TERM. +loop='trap "exit 0" TERM; while :; do sleep 1; done' +bash -c "$loop" bash -pidfile "$pidfile" & +own=$! +victims+=("$own") +fake_start "$own" +check_host "stop returns 0 for the run's own process" bash "$RUNNER" stop --state-file "$state" +check_host "the run's own process is terminated" dead "$own" +check_host "stop removes the state file after a kill" test ! -e "$state" + +# A failed start: a stub qemu that never opens the port. +stub="$work/qemu-stub" +cat > "$stub" << 'STUB' +#!/usr/bin/env bash +serial="" +while [ $# -gt 0 ]; do + [ "$1" = -serial ] && serial="${2#file:}" + [ "$1" = -pidfile ] && echo "$2" > "$(dirname "$0")/pidfile-arg" + shift +done +[ -z "$serial" ] || echo SERIAL-MARKER > "$serial" +echo STDERR-MARKER >&2 +STUB +chmod +x "$stub" +fixture="$work/fixture" +echo fixture > "$fixture" +rm -f "$state" +rc=0 +out="$(QEMU_BIN="$stub" QEMU_ACCEL=tcg QEMU_BOOT_TIMEOUT=1 \ + QEMU_KERNEL="$fixture" QEMU_INITRD="$fixture" QEMU_SSH_KEY="$fixture" \ + bash "$RUNNER" start --state-file "$state" 2>&1)" || rc=$? +check_host "start fails when the VM never boots" test "$rc" -eq 1 +check_host "the failure names the timeout" grep -qF "did not boot" <<< "$out" +check_host "the serial console reaches the caller" grep -qF SERIAL-MARKER <<< "$out" +check_host "qemu's own output reaches the caller" grep -qF STDERR-MARKER <<< "$out" +check_host "a failed start writes no state file" test ! -e "$state" +failed_rundir="$(dirname "$(cat "$work/pidfile-arg")")" +check_host "a failed start removes its run directory" test ! -e "$failed_rundir" + +report_summary +[ "$fail" -eq 0 ] From f920b8b60932009c1f97af077f329b3abcfd6e2b Mon Sep 17 00:00:00 2001 From: Chun-Hung Tseng Date: Wed, 16 Sep 2026 22:18:27 +0800 Subject: [PATCH 3/3] Offer only the VM key to the qemu guest ssh offers every identity its agent holds before a -i key the agent lacks. The fixture guest's Dropbear v2024.86 never counts a rejected public-key query, but dropbear 2026.90 caps them at MAX_PUBKEY_QUERIES (15), so a host agent holding that many keys would fail every qemu_exec and run_qemu call once the Alpine fixture moves past it. IdentitiesOnly=yes limits the offer to QEMU_SSH_KEY. --- tests/lib/qemu-ssh.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/lib/qemu-ssh.sh b/tests/lib/qemu-ssh.sh index 9ea7a40b..75835251 100644 --- a/tests/lib/qemu-ssh.sh +++ b/tests/lib/qemu-ssh.sh @@ -15,6 +15,7 @@ qemu_ssh_opts() -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o BatchMode=yes + -o IdentitiesOnly=yes -o ConnectTimeout=10 -o ServerAliveInterval=10 -o ServerAliveCountMax=6