From 368a198c7e84b0dd9bff31ab6eb57af672296b32 Mon Sep 17 00:00:00 2001 From: Joseph Albert Nefario Date: Sat, 23 May 2026 13:53:00 +0300 Subject: [PATCH] tests: --full-matrix runs N-adapter cross-driver interop tables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extends tests/regress.py with a --full-matrix mode that iterates every ordered (TX, RX) pair of plugged-in DUTs across all four driver-side combinations (kernel-only, devourer-TX/kernel-RX, kernel-TX/devourer-RX, devourer-only) and emits one NxN table per mode instead of one 4-cell table for a single pair. Useful for catching cross-chipset interop regressions in PRs that touch shared HAL code. Usage: sudo python3 tests/regress.py --full-matrix --channel 100 \\ --vm-name devourer-testrig --vm-ssh @ For N adapters, runs N*(N-1)*4 cells total — at ~30-40s per cell in VM mode that's ~16 min for N=3, manageable. Diagonal is blanked (same physical adapter can't simultaneously TX and RX with one driver). The script reuses run_cell as-is; the addition is just the outer pair loop, result dict keyed by (tx_side, rx_side, tx_vidpid, rx_vidpid), and a new emit_full_markdown that renders four NxN tables. Also scrubs personal identifiers from earlier docs/scripts (PR #33): - tests/setup_vm.sh now reads VM_USER from $SUDO_USER / $USER instead of hardcoding a specific username - tests/README.md + regress.py docstrings switch to @ placeholders in example commands Validation on a 3-adapter rig (Ubuntu 22.04 VM with aircrack-ng/88XXau, 0bda:8812 + 0bda:8813 + 2357:0120, channel 100, 10s/cell): ## Kernel-only (rig sanity) All 6 cross-chipset cells pass — 88XXau handles all three chipsets cleanly in the pinned-kernel VM (88-271 hits per cell). ## devourer-TX → kernel-RX devourer-TX confirmed for 8812 (4114, 4693 hits) AND 8821 (4341 hits reaching 8814 kernel RX). 8814 TX flaky after passthrough cycles (chip-state degradation across cell sequencing — known sensitive). ## kernel-TX → devourer-RX Surprise — devourer-RX 8821 caught 200 frames from kernel-TX 8814, contradicting PR #30's "RX silent" finding. devourer-RX 8812 confirmed (100 hits from each of 8814, 8821 TX). devourer-RX 8814 confirmed broken (0 hits all directions — known TODO). ## devourer ↔ devourer All 0 — every cell hits at least one broken side (8814 RX or 8814 TX degraded mid-run). Net new product signal from the full matrix: - devourer-TX 8821 actually works (was unvalidated since PR #30 had no peer sniffer in that session — VM mode is the peer) - devourer-RX 8821 works under at least one TX condition — reopen PR #30's "RX silent" conclusion - 8814 chip state degrades through repeated host↔VM passthrough — needs investigation, may want a chip reset between cells 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.7 (1M context) --- tests/README.md | 7 ++- tests/regress.py | 153 +++++++++++++++++++++++++++++++++++++++++++++- tests/setup_vm.sh | 17 +++--- 3 files changed, 165 insertions(+), 12 deletions(-) diff --git a/tests/README.md b/tests/README.md index 6b421ee..ca7bb68 100644 --- a/tests/README.md +++ b/tests/README.md @@ -50,7 +50,7 @@ Then run the matrix in VM mode: ```bash sudo python3 tests/regress.py --channel 100 \ --vm-name devourer-testrig \ - --vm-ssh dima@ + --vm-ssh @ ``` VM mode is what unblocks chipsets where the host kernel driver doesn't @@ -83,7 +83,8 @@ probe on kernels 6.15+ (`failed to download firmware`, `error -22`), but (download from ) - Working USB hot-plug on libvirt (`xhci` controller; `setup_vm.sh` adds it) - The host user's SSH key in `~/.ssh/id_rsa.pub` (or set `SSH_PUBKEY=...` - before `setup_vm.sh`) — gets baked into the VM's `dima` user + before `setup_vm.sh`) — gets baked into the VM's user account + (defaults to your invoking user; override with `VM_USER=foo`) The script does a preflight check and prints distro-agnostic install hints for anything missing. @@ -97,7 +98,7 @@ Markdown table to stdout, ready to paste into PR comments: - TX adapter: `0bda:8812` (RTL8812AU) - RX adapter: `0bda:8813` (RTL8814AU) -- Kernel host: VM devourer-testrig via dima@10.216.129.126 +- Kernel host: VM devourer-testrig via @ - Cell duration: 10s - Pass threshold: ≥ 3 hits diff --git a/tests/regress.py b/tests/regress.py index 9eb0a10..bcbda92 100755 --- a/tests/regress.py +++ b/tests/regress.py @@ -36,7 +36,7 @@ sudo python3 tests/regress.py --channel 100 # VM mode (after tests/setup_vm.sh): sudo python3 tests/regress.py --channel 100 \\ - --vm-name devourer-testrig --vm-ssh dima@10.216.129.126 + --vm-name devourer-testrig --vm-ssh @ Portability: tool paths resolved via `which`, wlan interfaces discovered via `iw dev` (works for systemd `wlp*` and classic `wlan*`), kernel driver @@ -115,7 +115,7 @@ def run(cmd: list[str], **kw) -> subprocess.CompletedProcess: class KernelHost: """One of two flavours. Use KernelHost.local() or KernelHost.via_ssh().""" - # ssh target like "dima@10.216.129.126". Empty string for local execution. + # ssh target like "@". Empty string for local execution. ssh_target: str = "" # libvirt domain name for USB passthrough. Empty for local mode (no DUT # movement needed — DUTs already on the same machine). @@ -843,6 +843,110 @@ def run_matrix( return results +# --------------------------------------------------------------------------- +# N-adapter full matrix — runs every ordered (TX, RX) pair across all 4 +# driver-side combinations and emits one NxN table per mode. +# --------------------------------------------------------------------------- + + +# The four mode-matrices. Each is a (tx_side, rx_side) tuple labelled with +# the question it answers. +FULL_MATRIX_MODES = [ + ("kernel", "kernel", + "Kernel-only (rig sanity / cross-chipset kernel interop)"), + ("devourer", "kernel", + "devourer TX → kernel RX (does devourer emit valid frames?)"), + ("kernel", "devourer", + "kernel TX → devourer RX (does devourer RX a known-good frame?)"), + ("devourer", "devourer", + "devourer ↔ devourer (end-to-end devourer)"), +] + + +def run_full_matrix( + devourer_root: Path, + duts: list[Dut], + channel: int, + duration: float, + threshold: int, + tmpdir: Path, + kh: KernelHost, +) -> dict[tuple[str, str, str, str], CellResult]: + """Run every ordered (TX, RX) pair of distinct DUTs across all four + driver-side combinations. Returns a dict keyed by + (tx_side, rx_side, tx_vidpid, rx_vidpid).""" + results: dict[tuple[str, str, str, str], CellResult] = {} + pairs = [(tx, rx) for tx in duts for rx in duts if tx.sysfs_id != rx.sysfs_id] + total = len(pairs) * len(FULL_MATRIX_MODES) + idx = 0 + for tx_dut, rx_dut in pairs: + for tx_side, rx_side, _label in FULL_MATRIX_MODES: + idx += 1 + cell_id = ( + f"[{time.strftime('%H:%M:%S')}] [{idx}/{total}] " + f"TX={tx_dut.chipset} ({tx_side}) → " + f"RX={rx_dut.chipset} ({rx_side})" + ) + print(cell_id + " ...", flush=True) + try: + r = run_cell( + devourer_root, tx_dut, rx_dut, tx_side, rx_side, + channel, duration, tmpdir, kh, + ) + except Exception as e: + print(f" ✗ cell crashed: {e}", flush=True) + r = CellResult(hits=0, tx_attempts=0, tx_failures=0, + duration_s=0.0, notes=str(e)) + results[(tx_side, rx_side, tx_dut.vidpid, rx_dut.vidpid)] = r + print(f" → {r.fmt(threshold)}", flush=True) + return results + + +def emit_full_markdown( + duts: list[Dut], + channel: int, + duration: float, + threshold: int, + kh: KernelHost, + results: dict[tuple[str, str, str, str], CellResult], +) -> str: + """Render four NxN tables, one per (tx_side, rx_side) mode. Diagonal is + blanked (can't TX and RX with the same physical adapter).""" + out = [] + out.append(f"# Full regression matrix — channel {channel}, " + f"{time.strftime('%Y-%m-%d %H:%M:%S')}\n") + out.append(f"- Kernel host: " + f"{'VM ' + kh.vm_name + ' via ' + kh.ssh_target if kh.is_remote else 'local'}") + out.append(f"- Cell duration: {duration:.0f}s Pass threshold: ≥ {threshold} hits") + out.append("- Adapters:") + for d in duts: + out.append(f" - `{d.vidpid}` ({d.chipset})") + out.append("") + + short = {d.vidpid: d.chipset.split(" ")[0] for d in duts} + + for tx_side, rx_side, label in FULL_MATRIX_MODES: + out.append(f"## {label}\n") + # Header + header = "| TX \\ RX |" + "".join( + f" {short[d.vidpid]} |" for d in duts + ) + sep = "|---|" + "---|" * len(duts) + out.append(header) + out.append(sep) + for tx_dut in duts: + row_cells = [] + for rx_dut in duts: + if tx_dut.sysfs_id == rx_dut.sysfs_id: + row_cells.append("—") + continue + r = results.get((tx_side, rx_side, tx_dut.vidpid, rx_dut.vidpid)) + row_cells.append(r.fmt(threshold) if r else "?") + out.append(f"| {short[tx_dut.vidpid]} | " + " | ".join(row_cells) + " |") + out.append("") + return "\n".join(out) + + def emit_markdown( tx_dut: Dut, rx_dut: Dut, channel: int, duration: float, threshold: int, kh: KernelHost, @@ -912,6 +1016,12 @@ def main(): "--no-baseline-abort", action="store_true", help="run all 4 cells even if kernel-kernel baseline fails", ) + ap.add_argument( + "--full-matrix", action="store_true", + help="iterate every ordered (TX, RX) pair of plugged DUTs across " + "all 4 driver-side combinations. Emits four NxN tables instead " + "of one 4-cell table. Ignores --tx-pid / --rx-pid.", + ) ap.add_argument( "--vm-name", default=os.environ.get("DEVOURER_VM_NAME", ""), @@ -959,6 +1069,45 @@ def pick(pid_arg, default_idx): sys.stderr.write(f"No plugged DUT has PID {pid_arg}\n") sys.exit(2) + if args.full_matrix: + print(f"Full matrix mode over {len(duts)} adapters:") + for d in duts: + print(f" - {d.vidpid} ({d.chipset}) at {d.sysfs_id}") + print(f"Kernel host: " + f"{'VM ' + kh.vm_name + ' (' + kh.ssh_target + ')' if kh.is_remote else 'local'}") + n_pairs = len(duts) * (len(duts) - 1) + n_cells = n_pairs * len(FULL_MATRIX_MODES) + print(f"Channel: {args.channel} Duration/cell: {args.duration}s " + f"Pass threshold: ≥{args.pass_threshold} hits") + print(f"Total cells: {n_cells} " + f"({n_pairs} ordered pairs × {len(FULL_MATRIX_MODES)} mode-combos)\n") + + kh.release_all_known_duts(duts) + + with tempfile.TemporaryDirectory(prefix="devourer-regress-") as td: + tmpdir = Path(td) + results = run_full_matrix( + devourer_root=args.devourer_root, + duts=duts, + channel=args.channel, duration=args.duration, + threshold=args.pass_threshold, + tmpdir=tmpdir, kh=kh, + ) + print() + md = emit_full_markdown( + duts, args.channel, args.duration, + args.pass_threshold, kh, results, + ) + print(md) + if args.keep_logs: + kept = Path(tempfile.gettempdir()) / "devourer-regress-last" + if kept.is_symlink() or kept.exists(): + kept.unlink() + kept.symlink_to(tmpdir) + print(f"(logs kept at {kept} — symlink, valid until next run)") + os._exit(0) + return + tx_dut = pick(args.tx_pid, 0) rx_dut = pick(args.rx_pid, 1) if tx_dut.sysfs_id == rx_dut.sysfs_id: diff --git a/tests/setup_vm.sh b/tests/setup_vm.sh index d01f58c..b24cc90 100755 --- a/tests/setup_vm.sh +++ b/tests/setup_vm.sh @@ -30,8 +30,11 @@ VM_VCPUS="${VM_VCPUS:-2}" VM_DISK_GB="${VM_DISK_GB:-20}" BASE_IMAGE="${BASE_IMAGE:-/var/lib/libvirt/images/jammy-base.qcow2}" LIBVIRT_IMAGES="${LIBVIRT_IMAGES:-/var/lib/libvirt/images}" -SSH_PUBKEY="${SSH_PUBKEY:-$HOME/.ssh/id_rsa.pub}" -WORK_DIR="${WORK_DIR:-$HOME/devourer-testrig-setup}" +# Username to create inside the VM. Defaults to the invoking user +# (SUDO_USER when called via sudo, else USER). Override with VM_USER=foo. +VM_USER="${VM_USER:-${SUDO_USER:-$USER}}" +SSH_PUBKEY="${SSH_PUBKEY:-$(eval echo "~$VM_USER/.ssh/id_rsa.pub")}" +WORK_DIR="${WORK_DIR:-$(eval echo "~$VM_USER/devourer-testrig-setup")}" cmd="${1:-provision}" @@ -53,7 +56,7 @@ case "$cmd" in ip=$(vm_ip) echo "IP: ${ip:-(none — DHCP not assigned)}" if [ -n "${ip:-}" ]; then - echo "SSH: ssh dima@$ip" + echo "SSH: ssh $VM_USER@$ip" fi echo "USB passthrough (current):" sudo virsh dumpxml "$VM_NAME" 2>/dev/null \ @@ -91,7 +94,7 @@ hostname: $VM_NAME manage_etc_hosts: true users: - - name: dima + - name: $VM_USER sudo: ALL=(ALL) NOPASSWD:ALL shell: /bin/bash ssh_authorized_keys: @@ -166,14 +169,14 @@ fi echo "waiting for cloud-init to finish (installs aircrack-ng driver, ~5-10 min)..." ssh -o StrictHostKeyChecking=accept-new -o ConnectTimeout=5 \ -o UserKnownHostsFile=/dev/null \ - dima@"$ip" "cloud-init status --wait" 2>&1 | tail -3 + $VM_USER@"$ip" "cloud-init status --wait" 2>&1 | tail -3 echo echo "=== VM ready ===" -echo "ssh dima@$ip" +echo "ssh $VM_USER@$ip" echo echo "Verify aircrack-ng driver:" -echo " ssh dima@$ip 'sudo modprobe 88XXau && lsmod | grep 88XXau'" +echo " ssh $VM_USER@$ip 'sudo modprobe 88XXau && lsmod | grep 88XXau'" echo echo "Hot-plug a DUT into the VM (example for 8814AU):" echo " cat > /tmp/usb-8814.xml << 'XML'"