diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e2fd0c4c7..66447a0fb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -561,7 +561,11 @@ jobs: # fields and the distro name, in # `os_release_reads_follow_the_simulated_host_root` and, for the # `/usr/lib/os-release` fallback, in - # `os_release_reads_fall_back_to_usr_lib_when_etc_is_absent`). The other routed + # `os_release_reads_fall_back_to_usr_lib_when_etc_is_absent`), and the + # driver-presence reads `/sys/class/kfd/kfd`, `/dev/kfd` and + # `/sys/bus/pci/drivers/amdgpu` (`kfd_present_on_host`, + # `amdgpu_present_on_host`, `driver_states` and the three + # `probe_modules_reports_*` tests). The other routed # reads (`/dev/dri`, `/sys/class/drm`, `/sys/module/amdgpu`, the other # `/proc` files, the modprobe directories, the `/usr/lib/wsl/lib` and # ROCDXG checks) have no planted-root unit assertion; they are covered diff --git a/crates/rocm-core/src/examine.rs b/crates/rocm-core/src/examine.rs index d5174f975..19686e156 100644 --- a/crates/rocm-core/src/examine.rs +++ b/crates/rocm-core/src/examine.rs @@ -244,8 +244,15 @@ pub struct Examination { pub has_discrete_amd: bool, // driver / runtime (Linux) + /// amdgpu is present: listed as a module, or registered as a PCI driver + /// (`/sys/bus/pci/drivers/amdgpu`), which it is whether amdgpu is loaded + /// as a module or built into the kernel (where it never appears in + /// `lsmod`), or kfd is present on the host (kfd is linked into amdgpu). pub amdgpu_loaded: Option, pub amdgpu_blacklisted_in: Vec, + /// kfd is available: as a separate `amdkfd` module on kernels before 4.20 + /// (where kfd was merged into `amdgpu.ko`), or through `/sys/class/kfd/kfd` + /// or `/dev/kfd` on current ones. The name predates the merge. pub amdkfd_loaded: Option, pub secure_boot: String, pub iommu_kernel_param: String, @@ -1824,22 +1831,88 @@ fn summarise_gpu_categories(e: &mut Examination) { // Kernel module / device probes (Linux) // --------------------------------------------------------------------------- -fn probe_modules(e: &mut Examination) { +/// Whether the kernel's kfd is present on this host, however it got there. +/// +/// On current kernels kfd is not a module of its own: `HSA_AMD` is a `bool` +/// Kconfig option and amdkfd is linked into `amdgpu`, so no `amdkfd` line +/// appears in the module list, built in or not. (Before 4.20 it was a separate +/// `amdkfd` module, which the module-list check still covers.) The kernel's own +/// interfaces are the signal: the `/sys/class/kfd/kfd` class device, or the +/// `/dev/kfd` node. +fn kfd_present_on_host() -> bool { + crate::host_path("/sys/class/kfd/kfd").exists() || crate::host_path("/dev/kfd").exists() +} + +/// Whether the amdgpu driver is present on this host, loaded or built in. +/// +/// A built-in driver is absent from the module list. `/sys/module/amdgpu` is +/// no answer: the kernel creates it for a built-in driver at an early initcall, +/// before `amdgpu_init` can bail out (`nomodeset`, `amdgpu.modeset=0`). The PCI +/// driver directory exists only once `amdgpu_init` registered the driver. +fn amdgpu_present_on_host() -> bool { + crate::host_path("/sys/bus/pci/drivers/amdgpu").exists() +} + +/// What the host says about `amdgpu_loaded`: the amdgpu PCI driver is +/// registered, or kfd is present. kfd is linked into amdgpu, so kfd on the +/// host proves amdgpu too: a container that passes `/dev/kfd` through without +/// `/sys/bus/pci` must not report amdkfd up and amdgpu down. +fn amdgpu_implied_by_host() -> bool { + amdgpu_present_on_host() || kfd_present_on_host() +} + +/// Whether `name` is loaded or built in: listed in the module list, or present +/// on the host. +/// +/// `modules` is `None` when the module list could not be read; the host +/// answering `true` still answers `true` then, and nothing else is known. The +/// host probe runs only when the module list has not already answered. +fn module_or_host_state( + modules: Option<&[&str]>, + name: &str, + on_host: impl FnOnce() -> bool, +) -> Option { + match modules { + Some(modules) => Some(modules.contains(&name) || on_host()), + None => on_host().then_some(true), + } +} + +/// `amdgpu_loaded` and `amdkfd_loaded`, in that order, from the module list +/// and the host. kfd on the host answers both; the amdgpu PCI driver answers +/// only `amdgpu_loaded`. +fn driver_states(modules: Option<&[&str]>) -> (Option, Option) { + ( + module_or_host_state(modules, "amdgpu", amdgpu_implied_by_host), + module_or_host_state(modules, "amdkfd", kfd_present_on_host), + ) +} + +/// The kernel's module list, one module per line with its name first: `lsmod` +/// without its header, else `/proc/modules`. `None` when neither can be read. +fn read_module_list() -> Option { let (rc, out, _) = run("lsmod", &[], SHORT); - let module_text = if rc == 0 { + if rc == 0 { Some(out.lines().skip(1).collect::>().join("\n")) } else { let txt = read_host_text("/proc/modules"); if txt.is_empty() { None } else { Some(txt) } - }; - if let Some(text) = module_text { - let modules: Vec<&str> = text - .lines() - .filter_map(|line| line.split_whitespace().next()) - .collect(); - e.amdgpu_loaded = Some(modules.contains(&"amdgpu")); - e.amdkfd_loaded = Some(modules.contains(&"amdkfd")); } +} + +fn probe_modules(e: &mut Examination) { + probe_modules_from(e, read_module_list().as_deref()); +} + +/// `probe_modules` given the module list, so a test can hand one in instead +/// of reading the machine's own. +fn probe_modules_from(e: &mut Examination, module_text: Option<&str>) { + let modules: Option> = module_text.map(|text| { + text.lines() + .filter_map(|line| line.split_whitespace().next()) + .collect() + }); + (e.amdgpu_loaded, e.amdkfd_loaded) = driver_states(modules.as_deref()); for dir in ["/etc/modprobe.d", "/usr/lib/modprobe.d", "/run/modprobe.d"] { let Ok(entries) = std::fs::read_dir(crate::host_path(dir)) else { @@ -3403,6 +3476,11 @@ mod tests { .expect("write quarantined repo"); std::fs::write(dir.path().join("unrelated.list"), "deb ...\n").expect("write unrelated"); + // The scan reads through `host_path`; hold the lock so another test's + // host root cannot re-root it. + let _guard = crate::hardware_root::HOST_ROOT_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); let mut seen = live_rocm_repo_paths_in_dir(&dir.path().to_string_lossy()); seen.sort(); @@ -3440,6 +3518,11 @@ mod tests { yum_dir.path().to_string_lossy().into_owned(), zypp_dir.path().to_string_lossy().into_owned(), ]; + // The scan reads through `host_path`; hold the lock so another test's + // host root cannot re-root it. + let _guard = crate::hardware_root::HOST_ROOT_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); let dir_refs: Vec<&str> = dirs.iter().map(String::as_str).collect(); merge_live_repo_paths(&mut e, &dir_refs); @@ -6418,6 +6501,353 @@ mod tests { } } + #[test] + fn a_module_is_present_when_listed() { + assert_eq!( + module_or_host_state(Some(&["amdgpu", "amdkfd"]), "amdkfd", || false), + Some(true) + ); + assert_eq!( + module_or_host_state(Some(&["amdgpu", "drm"]), "amdgpu", || false), + Some(true) + ); + } + + /// Built into the kernel, a driver never appears in the module list. + #[test] + fn a_module_is_present_when_built_in_and_absent_from_the_module_list() { + assert_eq!( + module_or_host_state(Some(&["drm"]), "amdkfd", || true), + Some(true) + ); + assert_eq!( + module_or_host_state(Some(&[]), "amdgpu", || true), + Some(true) + ); + } + + #[test] + fn a_module_is_absent_with_neither_a_listing_nor_the_host() { + assert_eq!( + module_or_host_state(Some(&["drm"]), "amdkfd", || false), + Some(false) + ); + assert_eq!( + module_or_host_state(Some(&[]), "amdgpu", || false), + Some(false) + ); + } + + /// With no module list, the host still answers; without it the field + /// stays unknown rather than claiming `false`. + #[test] + fn an_unreadable_module_list_follows_the_host() { + assert_eq!(module_or_host_state(None, "amdgpu", || true), Some(true)); + assert_eq!(module_or_host_state(None, "amdgpu", || false), None); + } + + /// The host probe is skipped once the module list has answered. + #[test] + fn the_host_probe_is_skipped_when_the_module_list_already_answers() { + assert_eq!( + module_or_host_state(Some(&["amdgpu"]), "amdgpu", || unreachable!()), + Some(true) + ); + } + + /// The host side of the check, against a planted root. + #[cfg(all(feature = "e2e-test-hooks", target_os = "linux"))] + #[test] + fn amdgpu_present_on_host_reads_the_pci_driver_dir() { + use crate::hardware_root::TEST_HOST_ROOT_ENV; + use crate::test_env::RestoredEnvVar; + + let root = std::env::temp_dir().join(format!( + "rocm-core-amdgpu-present-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_nanos()) + )); + std::fs::create_dir_all(&root).expect("create root"); + let (neither, module_dir_only, planted) = { + let _guard = crate::hardware_root::HOST_ROOT_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _root = RestoredEnvVar::set(TEST_HOST_ROOT_ENV, &root); + let neither = amdgpu_present_on_host(); + // The module directory a built-in driver keeps even when it never + // bound: not evidence. + std::fs::create_dir_all(root.join("sys/module/amdgpu")).expect("plant sysfs"); + let module_dir_only = amdgpu_present_on_host(); + std::fs::create_dir_all(root.join("sys/bus/pci/drivers/amdgpu")).expect("plant sysfs"); + (neither, module_dir_only, amdgpu_present_on_host()) + }; + std::fs::remove_dir_all(&root).ok(); + assert!(!neither, "an empty root has no amdgpu"); + assert!( + !module_dir_only, + "/sys/module/amdgpu alone is not a bound driver" + ); + assert!(planted, "/sys/bus/pci/drivers/amdgpu counts"); + } + + /// A module list, as `read_module_list` returns it, naming neither + /// `amdgpu` nor `amdkfd`. + #[cfg(all(feature = "e2e-test-hooks", target_os = "linux"))] + const NO_AMD_MODULES: &str = "snd_hda_intel 61440 3 - Live 0x0000000000000000\n\ + nvme 61440 2 - Live 0x0000000000000000"; + + /// The module list alone, against an empty host root: on a machine where + /// the drivers really are modules, each field must come from its own name + /// in the list. Both named answers both fields; `amdgpu` alone answers + /// only `amdgpu_loaded`, and `amdkfd` alone only `amdkfd_loaded`, so a + /// misspelt or swapped name in `driver_states` fails here. + #[cfg(all(feature = "e2e-test-hooks", target_os = "linux"))] + #[test] + fn probe_modules_reports_each_driver_its_module_list_names() { + use crate::hardware_root::TEST_HOST_ROOT_ENV; + use crate::test_env::RestoredEnvVar; + + let root = tempfile::tempdir().expect("create host root"); + let probe = |module_text: &str| { + let mut e = Examination::default(); + probe_modules_from(&mut e, Some(module_text)); + (e.amdgpu_loaded, e.amdkfd_loaded) + }; + let (both, amdgpu_only, amdkfd_only) = { + let _guard = crate::hardware_root::HOST_ROOT_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _root = RestoredEnvVar::set(TEST_HOST_ROOT_ENV, root.path()); + ( + probe( + "amdgpu 19378176 12 - Live 0x0000000000000000\n\ + amdkfd 380928 1 amdgpu, Live 0x0000000000000000\n\ + nvme 61440 2 - Live 0x0000000000000000", + ), + probe( + "snd_hda_intel 61440 3 - Live 0x0000000000000000\n\ + amdgpu 19378176 12 - Live 0x0000000000000000", + ), + probe("amdkfd 380928 0 - Live 0x0000000000000000"), + ) + }; + assert_eq!(both, (Some(true), Some(true)), "both drivers listed"); + assert_eq!(amdgpu_only, (Some(true), Some(false)), "only amdgpu listed"); + assert_eq!(amdkfd_only, (Some(false), Some(true)), "only amdkfd listed"); + } + + /// The wiring, not just the helpers: `probe_modules_from` must consult the + /// host root. The module list it is handed names neither driver, so + /// `amdgpu_loaded` can only come from the planted + /// `/sys/bus/pci/drivers/amdgpu`, and that directory must not make + /// `amdkfd_loaded` true. + #[cfg(all(feature = "e2e-test-hooks", target_os = "linux"))] + #[test] + fn probe_modules_reports_amdgpu_when_it_is_on_the_host() { + use crate::hardware_root::TEST_HOST_ROOT_ENV; + use crate::test_env::RestoredEnvVar; + + let root = std::env::temp_dir().join(format!( + "rocm-core-probe-amdgpu-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_nanos()) + )); + std::fs::create_dir_all(root.join("sys/bus/pci/drivers/amdgpu")).expect("plant sysfs"); + let (amdgpu_loaded, amdkfd_loaded) = { + let _guard = crate::hardware_root::HOST_ROOT_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _root = RestoredEnvVar::set(TEST_HOST_ROOT_ENV, &root); + let mut e = Examination::default(); + probe_modules_from(&mut e, Some(NO_AMD_MODULES)); + (e.amdgpu_loaded, e.amdkfd_loaded) + }; + std::fs::remove_dir_all(&root).ok(); + assert_eq!(amdgpu_loaded, Some(true)); + assert_eq!( + amdkfd_loaded, + Some(false), + "the amdgpu PCI driver does not imply kfd" + ); + } + + /// The host side of the check, against a planted root: the sysfs class + /// device alone, the device node alone, and neither. + #[cfg(all(feature = "e2e-test-hooks", target_os = "linux"))] + #[test] + fn kfd_present_on_host_reads_sysfs_and_the_device_node() { + use crate::hardware_root::TEST_HOST_ROOT_ENV; + use crate::test_env::RestoredEnvVar; + + let root = std::env::temp_dir().join(format!( + "rocm-core-kfd-present-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_nanos()) + )); + std::fs::create_dir_all(&root).expect("create root"); + let (neither, sysfs_only, dev_only) = { + let _guard = crate::hardware_root::HOST_ROOT_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _root = RestoredEnvVar::set(TEST_HOST_ROOT_ENV, &root); + let neither = kfd_present_on_host(); + std::fs::create_dir_all(root.join("sys/class/kfd/kfd")).expect("plant sysfs"); + let sysfs_only = kfd_present_on_host(); + std::fs::remove_dir_all(root.join("sys")).expect("unplant sysfs"); + std::fs::create_dir_all(root.join("dev")).expect("plant dev"); + std::fs::write(root.join("dev/kfd"), "").expect("plant /dev/kfd"); + (neither, sysfs_only, kfd_present_on_host()) + }; + std::fs::remove_dir_all(&root).ok(); + assert!(!neither, "an empty root has no kfd"); + assert!(sysfs_only, "/sys/class/kfd/kfd alone counts"); + assert!(dev_only, "/dev/kfd alone counts"); + } + + /// kfd on the host alone answers `amdgpu_loaded`: no amdgpu PCI driver + /// directory, only a kfd signal, either one. + #[cfg(all(feature = "e2e-test-hooks", target_os = "linux"))] + #[test] + fn kfd_alone_on_the_host_implies_amdgpu() { + use crate::hardware_root::TEST_HOST_ROOT_ENV; + use crate::test_env::RestoredEnvVar; + + let root = std::env::temp_dir().join(format!( + "rocm-core-kfd-implies-amdgpu-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_nanos()) + )); + std::fs::create_dir_all(&root).expect("create root"); + let (neither, sysfs_only, dev_only) = { + let _guard = crate::hardware_root::HOST_ROOT_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _root = RestoredEnvVar::set(TEST_HOST_ROOT_ENV, &root); + let neither = amdgpu_implied_by_host(); + std::fs::create_dir_all(root.join("sys/class/kfd/kfd")).expect("plant sysfs"); + let sysfs_only = amdgpu_implied_by_host(); + std::fs::remove_dir_all(root.join("sys")).expect("unplant sysfs"); + std::fs::create_dir_all(root.join("dev")).expect("plant dev"); + std::fs::write(root.join("dev/kfd"), "").expect("plant /dev/kfd"); + (neither, sysfs_only, amdgpu_implied_by_host()) + }; + std::fs::remove_dir_all(&root).ok(); + assert!(!neither, "an empty root implies no amdgpu"); + assert!(sysfs_only, "/sys/class/kfd/kfd alone implies amdgpu"); + assert!(dev_only, "/dev/kfd alone implies amdgpu"); + } + + /// Which host signal answers which field. Each case plants one signal into + /// an otherwise empty root and hands over the module list directly, so the + /// answer cannot come from the machine the test runs on: kfd alone answers + /// both fields, the amdgpu PCI driver alone answers only `amdgpu_loaded`, + /// and with the module list unreadable the host still answers. + #[cfg(all(feature = "e2e-test-hooks", target_os = "linux"))] + #[test] + fn driver_states_answer_each_field_from_its_own_host_signal() { + use crate::hardware_root::TEST_HOST_ROOT_ENV; + use crate::test_env::RestoredEnvVar; + + let root = std::env::temp_dir().join(format!( + "rocm-core-driver-states-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_nanos()) + )); + std::fs::create_dir_all(&root).expect("create root"); + let (neither, unread_neither, kfd_only, unread_kfd_only, pci_only, unread_pci_only) = { + let _guard = crate::hardware_root::HOST_ROOT_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _root = RestoredEnvVar::set(TEST_HOST_ROOT_ENV, &root); + let neither = driver_states(Some(&[])); + let unread_neither = driver_states(None); + std::fs::create_dir_all(root.join("dev")).expect("plant dev"); + std::fs::write(root.join("dev/kfd"), "").expect("plant /dev/kfd"); + let kfd_only = driver_states(Some(&[])); + let unread_kfd_only = driver_states(None); + std::fs::remove_dir_all(root.join("dev")).expect("unplant dev"); + std::fs::create_dir_all(root.join("sys/bus/pci/drivers/amdgpu")).expect("plant sysfs"); + ( + neither, + unread_neither, + kfd_only, + unread_kfd_only, + driver_states(Some(&[])), + driver_states(None), + ) + }; + std::fs::remove_dir_all(&root).ok(); + assert_eq!( + neither, + (Some(false), Some(false)), + "nothing listed, nothing on the host" + ); + assert_eq!( + unread_neither, + (None, None), + "nothing read, nothing on the host" + ); + assert_eq!(kfd_only, (Some(true), Some(true)), "kfd implies amdgpu"); + assert_eq!( + unread_kfd_only, + (Some(true), Some(true)), + "kfd answers without a module list" + ); + assert_eq!( + pci_only, + (Some(true), Some(false)), + "amdgpu does not imply kfd" + ); + assert_eq!( + unread_pci_only, + (Some(true), None), + "amdgpu does not imply kfd" + ); + } + + /// The wiring, not just the helpers: `probe_modules_from` must consult the + /// host root. The module list it is handed names neither driver, so + /// `Some(true)` can only come from the planted `/dev/kfd`, which answers + /// `amdgpu_loaded` too. + #[cfg(all(feature = "e2e-test-hooks", target_os = "linux"))] + #[test] + fn probe_modules_reports_amdkfd_when_kfd_is_on_the_host() { + use crate::hardware_root::TEST_HOST_ROOT_ENV; + use crate::test_env::RestoredEnvVar; + + let root = std::env::temp_dir().join(format!( + "rocm-core-probe-kfd-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_nanos()) + )); + std::fs::create_dir_all(root.join("dev")).expect("plant dev"); + std::fs::write(root.join("dev/kfd"), "").expect("plant /dev/kfd"); + let (amdgpu_loaded, amdkfd_loaded) = { + let _guard = crate::hardware_root::HOST_ROOT_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _root = RestoredEnvVar::set(TEST_HOST_ROOT_ENV, &root); + let mut e = Examination::default(); + probe_modules_from(&mut e, Some(NO_AMD_MODULES)); + (e.amdgpu_loaded, e.amdkfd_loaded) + }; + std::fs::remove_dir_all(&root).ok(); + assert_eq!(amdkfd_loaded, Some(true)); + assert_eq!(amdgpu_loaded, Some(true), "kfd implies amdgpu"); + } + /// The probes themselves, not just `host_path`, answer from a simulated /// host root. Plants a bare-metal machine with one GPU, inside a Podman /// container, then runs the converted probes and asserts they describe it diff --git a/crates/rocm-core/src/hardware_root.rs b/crates/rocm-core/src/hardware_root.rs index 86a5e3d9e..1f7fd66fe 100644 --- a/crates/rocm-core/src/hardware_root.rs +++ b/crates/rocm-core/src/hardware_root.rs @@ -9,8 +9,10 @@ //! directly. That is every one of these reads, and only these: //! //! - device nodes: `/dev/kfd`, the render nodes under `/dev/dri`, `/dev/dxg`; -//! - `sysfs`: the KFD topology under `/sys/class/kfd/kfd/topology/nodes`, the -//! DRM cards under `/sys/class/drm`, `/sys/module/amdgpu` and +//! - `sysfs`: the `/sys/class/kfd/kfd` class device and the KFD topology under +//! `/sys/class/kfd/kfd/topology/nodes`, the +//! DRM cards under `/sys/class/drm`, `/sys/bus/pci/drivers/amdgpu`, +//! `/sys/module/amdgpu` and //! `/sys/module/amdgpu/version`; //! - `procfs`: `/proc/version`, `/proc/cpuinfo`, `/proc/meminfo`, //! `/proc/cmdline`, `/proc/modules` (the fallback when `lsmod` cannot run) diff --git a/docs/release-trust.md b/docs/release-trust.md index b5d4429b9..b70a77085 100644 --- a/docs/release-trust.md +++ b/docs/release-trust.md @@ -312,10 +312,12 @@ is already trusted. ## Simulated Host Root rocm-cli decides what hardware it is running on by reading fixed host paths: -`/dev/kfd`, the render nodes under `/dev/dri`, `/dev/dxg`, the KFD topology -under `/sys/class/kfd`, the DRM cards under `/sys/class/drm`, -`/sys/module/amdgpu` and its `version` file, `/proc/version`, `/proc/cpuinfo`, -`/proc/meminfo`, `/proc/cmdline`, `/proc/modules`, `/proc/1/cgroup`, the +`/dev/kfd`, the render nodes under `/dev/dri`, `/dev/dxg`, the +`/sys/class/kfd/kfd` class device and the KFD topology under it, the DRM cards +under `/sys/class/drm`, the amdgpu PCI driver directory +`/sys/bus/pci/drivers/amdgpu`, `/sys/module/amdgpu` and its `version` file, +`/proc/version`, `/proc/cpuinfo`, `/proc/meminfo`, `/proc/cmdline`, +`/proc/modules`, `/proc/1/cgroup`, the container markers `/.dockerenv` and `/run/.containerenv`, the `/etc/os-release` (falling back to `/usr/lib/os-release` when absent, per `os-release(5)`) behind the distro fields and name `rocm examine` reports, the modprobe diff --git a/tests/e2e-cucumber/features/examine.feature b/tests/e2e-cucumber/features/examine.feature index 34c732c3c..fe6d47985 100644 --- a/tests/e2e-cucumber/features/examine.feature +++ b/tests/e2e-cucumber/features/examine.feature @@ -464,3 +464,42 @@ Feature: GPU detection and system inspection Given a machine with a ROCm install that was not set up by the CLI When the user inspects the system in machine-readable form Then the inspection reports a current install-method name + + # kfd is linked into `amdgpu` (`HSA_AMD` is a `bool` Kconfig option), so a + # healthy machine never lists an `amdkfd` module. The report used to read + # that as "kfd missing". Each scenario plants a module list naming `amdgpu` + # only, so the answer can come only from the kernel's kfd interfaces. + @id:examine-reports-kfd-built-into-the-kernel @requires-os:linux + Scenario: examine-31 - The inspection reports kfd on a machine whose module list has no amdkfd + Given a machine whose kernel exposes kfd but lists no amdkfd module + When the user inspects that machine in machine-readable form + Then the inspection reports kfd as loaded + + @id:examine-reports-kfd-absent-without-kfd-interfaces @requires-os:linux + Scenario: examine-32 - The inspection reports kfd as not loaded on a machine without it + Given a machine whose kernel has no kfd and lists no amdkfd module + When the user inspects that machine in machine-readable form + Then the inspection reports kfd as not loaded + + # amdgpu built into the kernel is absent from `lsmod` too, so it was read as + # "module not loaded" and `rocm diagnose` scored that, plus a DKMS failure + # and Secure Boot, as evidence. `/sys/bus/pci/drivers/amdgpu` exists for a loaded + # module and a built-in driver alike. The machine also has a ROCm install + # and Secure Boot enabled, so the evidence lines are reachable if the flag + # regresses. + @id:examine-reports-amdgpu-built-into-the-kernel @requires-os:linux + Scenario: examine-33 - A built-in amdgpu is reported as loaded and diagnose finds no driver fault + Given a machine whose kernel has amdgpu built in and lists no amdgpu module + When the user inspects and diagnoses that machine + Then the inspection reports amdgpu as loaded + And the diagnosis cites no missing amdgpu module, DKMS failure or Secure Boot evidence + + # With no module list at all (`lsmod` fails and `/proc/modules` is empty), + # the kernel's kfd interfaces still answer: kfd is linked into amdgpu, so + # `/dev/kfd` reports both drivers as loaded rather than leaving them unknown. + @id:examine-reports-drivers-from-kfd-without-a-module-list @requires-os:linux + Scenario: examine-34 - The inspection reports kfd and amdgpu from the kernel's kfd node when no module list can be read + Given a machine whose kernel exposes kfd but whose module list cannot be read + When the user inspects that machine in machine-readable form + Then the inspection reports kfd as loaded + And the inspection reports amdgpu as loaded diff --git a/tests/e2e-cucumber/tests/e2e/examine_steps.rs b/tests/e2e-cucumber/tests/e2e/examine_steps.rs index 67898e407..928794863 100644 --- a/tests/e2e-cucumber/tests/e2e/examine_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/examine_steps.rs @@ -1432,6 +1432,27 @@ async fn os_release_with_a_command(world: &mut E2eWorld) { const RUNS_A_COMMAND_ON_LINE_3: &str = "ID=ubuntu\nVERSION_ID=\"24.04\"\nexport ID=debian\nPRETTY_NAME=\"Ubuntu 24.04 LTS\"\n"; +/// Writes each `(name, script)` as an executable in the isolated root's +/// `fake-bin` and puts that directory first on the scenario's `PATH`. +fn put_fake_executables_on_path(world: &mut E2eWorld, executables: &[(&str, &str)]) { + let bin = world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .join("fake-bin"); + std::fs::create_dir_all(&bin).expect("failed to create the fake PATH dir"); + for (name, script) in executables { + rocm_core::fake_executable::write_fake_executable(&bin.join(name), script) + .unwrap_or_else(|e| panic!("failed to write the fake {name}: {e}")); + } + let path = std::env::join_paths(std::iter::once(bin).chain(std::env::split_paths( + &std::env::var_os("PATH").unwrap_or_default(), + ))) + .expect("PATH entries cannot hold a separator"); + world.command_env.push(("PATH", path)); +} + /// One GPU node in the kernel topology: a gfx1101 with no PCI address, so it /// matches no `lspci` entry and is listed from the topology alone. Plus an /// `lspci` that lists an empty bus, ahead of any real one, so the PCI probe @@ -1453,15 +1474,7 @@ fn plant_gpu_fixtures(world: &mut E2eWorld) { "cpu_cores_count 0\nsimd_count 120\ngfx_target_version 110001\n", ) .expect("failed to write planted KFD node properties"); - let bin = root.join("fake-bin"); - std::fs::create_dir_all(&bin).expect("failed to create the fake PATH dir"); - rocm_core::fake_executable::write_fake_executable(&bin.join("lspci"), "#!/bin/sh\nexit 0\n") - .expect("failed to write the fake lspci"); - let path = std::env::join_paths(std::iter::once(bin).chain(std::env::split_paths( - &std::env::var_os("PATH").unwrap_or_default(), - ))) - .expect("PATH entries cannot hold a separator"); - world.command_env.push(("PATH", path)); + put_fake_executables_on_path(world, &[("lspci", "#!/bin/sh\nexit 0\n")]); } #[given("a GPU machine whose release file runs a command on line 3")] @@ -1490,6 +1503,161 @@ async fn gpu_machine_with_unreadable_release_file(world: &mut E2eWorld) { plant_gpu_fixtures(world); } +/// A simulated host root, plus a `lsmod` on `PATH` that lists `amdgpu` and no +/// `amdkfd`, so the module list never answers for kfd. `with_kfd` plants the +/// `/dev/kfd` node. +fn plant_machine_without_amdkfd_module(world: &mut E2eWorld, with_kfd: bool) { + let root = world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .to_owned(); + let host = root.join("host-root"); + std::fs::create_dir_all(host.join("dev")).expect("failed to create planted /dev"); + if with_kfd { + std::fs::write(host.join("dev/kfd"), "").expect("failed to plant /dev/kfd"); + } + put_fake_executables_on_path( + world, + &[( + "lsmod", + "#!/bin/sh\nprintf 'Module Size Used by\\namdgpu 1 0\\ndrm 1 1 amdgpu\\n'\n", + )], + ); + world + .command_env + .push(("ROCM_CLI_TEST_HOST_ROOT", host.into_os_string())); +} + +#[given("a machine whose kernel exposes kfd but lists no amdkfd module")] +async fn machine_with_built_in_kfd(world: &mut E2eWorld) { + plant_machine_without_amdkfd_module(world, true); +} + +#[given("a machine whose kernel has no kfd and lists no amdkfd module")] +async fn machine_without_kfd(world: &mut E2eWorld) { + plant_machine_without_amdkfd_module(world, false); +} + +/// A simulated host root with `/dev/kfd`, no `/proc/modules`, and a `lsmod` +/// on `PATH` that fails: no module list can be read at all. +#[given("a machine whose kernel exposes kfd but whose module list cannot be read")] +async fn machine_with_kfd_and_no_module_list(world: &mut E2eWorld) { + let root = world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .to_owned(); + let host = root.join("host-root"); + std::fs::create_dir_all(host.join("dev")).expect("failed to create planted /dev"); + std::fs::write(host.join("dev/kfd"), "").expect("failed to plant /dev/kfd"); + put_fake_executables_on_path(world, &[("lsmod", "#!/bin/sh\nexit 1\n")]); + world + .command_env + .push(("ROCM_CLI_TEST_HOST_ROOT", host.into_os_string())); +} + +#[then("the inspection reports kfd as loaded")] +async fn assert_kfd_loaded(world: &mut E2eWorld) { + let json = parsed_json(world); + assert_eq!(json["amdkfd_loaded"], serde_json::json!(true), "{json:#}"); +} + +#[then("the inspection reports kfd as not loaded")] +async fn assert_kfd_not_loaded(world: &mut E2eWorld) { + let json = parsed_json(world); + assert_eq!(json["amdkfd_loaded"], serde_json::json!(false), "{json:#}"); +} + +/// A simulated host root with `/sys/bus/pci/drivers/amdgpu` and a `lsmod` that lists +/// neither `amdgpu` nor `amdkfd`: amdgpu built into the kernel. A ROCm install +/// (`ROCM_PATH`) and a `mokutil` reporting Secure Boot enabled are planted too, +/// so the three `rocm diagnose` evidence lines that read `amdgpu_loaded` have +/// everything else they need to fire. +#[given("a machine whose kernel has amdgpu built in and lists no amdgpu module")] +async fn machine_with_built_in_amdgpu(world: &mut E2eWorld) { + let root = world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .to_owned(); + let host = root.join("host-root"); + std::fs::create_dir_all(host.join("sys/bus/pci/drivers/amdgpu")) + .expect("failed to plant /sys/bus/pci/drivers/amdgpu"); + let rocm = root.join("rocm"); + std::fs::create_dir_all(rocm.join(".info")).expect("failed to plant a ROCm install"); + std::fs::write(rocm.join(".info/version"), "6.4.1\n").expect("failed to plant its version"); + put_fake_executables_on_path( + world, + &[ + ( + "lsmod", + "#!/bin/sh\nprintf 'Module Size Used by\\ndrm 1 0\\n'\n", + ), + ("mokutil", "#!/bin/sh\necho 'SecureBoot enabled'\n"), + ], + ); + world.command_env.push(("ROCM_PATH", rocm.into_os_string())); + world + .command_env + .push(("ROCM_CLI_TEST_HOST_ROOT", host.into_os_string())); +} + +/// Both commands read the same planted machine, so the scenario environment is +/// restored between them: `run_rocm_with_scenario_env` consumes it. +#[when("the user inspects and diagnoses that machine")] +async fn user_inspects_and_diagnoses_planted_machine(world: &mut E2eWorld) { + let env = world.command_env.clone(); + let (stdout, stderr, rc) = + crate::run_rocm_with_scenario_env(world, &["examine", "--framework", "skip", "--json"]); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); + world.command_env = env; + let (diagnosis, _, diagnose_rc) = + crate::run_rocm_with_scenario_env(world, &["diagnose", "--json"]); + assert_eq!(diagnose_rc, 0, "diagnose is a query and exits 0"); + world.cli_outputs = Some(vec![diagnosis]); +} + +#[then("the inspection reports amdgpu as loaded")] +async fn assert_amdgpu_loaded(world: &mut E2eWorld) { + let json = parsed_json(world); + assert_eq!(json["amdgpu_loaded"], serde_json::json!(true), "{json:#}"); +} + +#[then("the diagnosis cites no missing amdgpu module, DKMS failure or Secure Boot evidence")] +async fn assert_no_amdgpu_fault_evidence(world: &mut E2eWorld) { + let diagnosis = &world.cli_outputs.as_ref().expect("no diagnosis was run")[0]; + let report: serde_json::Value = serde_json::from_str(diagnosis) + .unwrap_or_else(|e| panic!("diagnose --json is not JSON ({e}):\n{diagnosis}")); + let evidence: Vec<&str> = report["matched"] + .as_array() + .unwrap_or_else(|| panic!("diagnose --json has no matched array:\n{report:#}")) + .iter() + .flat_map(|m| m["evidence"].as_array().into_iter().flatten()) + .filter_map(serde_json::Value::as_str) + .collect(); + // One phrase per evidence line that `amdgpu_loaded == Some(false)` produces in + // diagnose.rs, each copied verbatim from that check so it keeps guarding it. + let amdgpu_missing_evidence = [ + // check_3_rocm_kernel_unsupported + "the amdgpu kernel module is not loaded", + // check_5_amdgpu_blacklisted + "amdgpu module is not loaded", + "Secure Boot is enabled and amdgpu didn't load", + ]; + for line in evidence { + assert!( + !amdgpu_missing_evidence.iter().any(|p| line.contains(p)), + "amdgpu was reported missing though the amdgpu PCI driver is registered: {line}" + ); + } +} + #[when("the user inspects that machine in machine-readable form")] async fn user_inspects_planted_machine(world: &mut E2eWorld) { let (stdout, stderr, rc) =