From 7c445471941bd26c38396df806dcc771e808763d Mon Sep 17 00:00:00 2001 From: Pramod Kumbhar Date: Sun, 28 Jun 2026 17:27:06 +0000 Subject: [PATCH 1/3] Resolve Container Native Libraries - Add a libdwfl fallback that opens absolute module paths through /proc//root so remote Pyxis/Enroot targets can resolve libraries that are only visible inside the container filesystem. - Associate the analyzed PID with each DWFL module before attach so the ELF lookup callback can find the target process root during native unwinding. - Preserve the existing build-id and linux-proc lookup paths first; the process-root fallback is used only when normal host lookup fails. - This fixes the behavior seen with the ImageNet Pyxis NCCL run where PyStack 1.6 reported insufficient native information or lost libtorch/libtorch_cuda/NCCL frames from container targets. Signed-off-by: Pramod Kumbhar --- src/pystack/_pystack/elf_common.cpp | 65 ++++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 2 deletions(-) diff --git a/src/pystack/_pystack/elf_common.cpp b/src/pystack/_pystack/elf_common.cpp index 96e43f25..fb1af722 100644 --- a/src/pystack/_pystack/elf_common.cpp +++ b/src/pystack/_pystack/elf_common.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include "compat.h" @@ -14,6 +15,58 @@ namespace pystack { using file_unique_ptr = std::unique_ptr>; +namespace { + +int +set_process_pid( + Dwfl_Module* mod __attribute__((unused)), + void** userdata, + const char* name __attribute__((unused)), + Dwarf_Addr start __attribute__((unused)), + void* arg) +{ + *userdata = arg; + return DWARF_CB_OK; +} + +int +find_elf_through_process_root(void** userdata, const char* modname, char** file_name) +{ + if (userdata == nullptr || *userdata == nullptr || modname == nullptr || modname[0] != '/') { + return -1; + } + + const auto pid = *static_cast(*userdata); + if (pid <= 0) { + return -1; + } + + const std::string rooted_path = + "/proc/" + std::to_string(pid) + "/root" + std::string(modname); + int fd = open(rooted_path.c_str(), O_RDONLY); + if (fd < 0) { + return -1; + } + + struct stat file_stat; + if (fstat(fd, &file_stat) != 0 || !S_ISREG(file_stat.st_mode)) { + close(fd); + return -1; + } + + if (file_name != nullptr) { + *file_name = strdup(rooted_path.c_str()); + if (*file_name == nullptr) { + close(fd); + return -1; + } + } + + return fd; +} + +} // namespace + int pystack_find_elf( Dwfl_Module* mod, @@ -31,10 +84,14 @@ pystack_find_elf( return ret; } ret = dwfl_linux_proc_find_elf(mod, userdata, modname, base, file_name, elfp); - if (file_name == nullptr) { + if (ret < 0) { + ret = find_elf_through_process_root(userdata, modname, file_name); + } + if (ret < 0) { LOG(DEBUG) << "Could not locate debug info for " << the_modname; } else { - LOG(DEBUG) << "Located debug info for " << the_modname << " by path in " << *file_name; + const char* the_filename = (file_name == nullptr || *file_name == nullptr) ? "???" : *file_name; + LOG(DEBUG) << "Located debug info for " << the_modname << " by path in " << the_filename; } return ret; } @@ -285,6 +342,10 @@ ProcessAnalyzer::ProcessAnalyzer(pid_t pid) throw ElfAnalyzerError("Failed to analyze DWARF information for the remote process"); } + if (dwfl_getmodules(d_dwfl.get(), set_process_pid, &d_pid, 0) == -1) { + throw ElfAnalyzerError("Failed to associate DWARF modules with the remote process"); + } + if (dwfl_linux_proc_attach(d_dwfl.get(), pid, true) != 0) { throw ElfAnalyzerError("Could not attach the DWARF process analyzer"); } From 6da866bc981739b9e2deff4258118e117413a5da Mon Sep 17 00:00:00 2001 From: Pramod Kumbhar Date: Wed, 15 Jul 2026 11:50:16 +0000 Subject: [PATCH 2/3] Prefer process-root native library lookup - Resolve mapped paths through /proc//root before libdwfl's fallback to avoid opening a different host file. - Preserve plain module paths for identical files so same-namespace module and debuginfo behavior stays unchanged. - Handle deleted and missing module names safely, and match load points against reported, main, and debug paths. Signed-off-by: Pramod Kumbhar --- src/pystack/_pystack/elf_common.cpp | 65 ++++++++++++++++++----------- 1 file changed, 41 insertions(+), 24 deletions(-) diff --git a/src/pystack/_pystack/elf_common.cpp b/src/pystack/_pystack/elf_common.cpp index fb1af722..16dafbda 100644 --- a/src/pystack/_pystack/elf_common.cpp +++ b/src/pystack/_pystack/elf_common.cpp @@ -18,7 +18,7 @@ using file_unique_ptr = std::unique_ptr>; namespace { int -set_process_pid( +set_module_process_pid_userdata( Dwfl_Module* mod __attribute__((unused)), void** userdata, const char* name __attribute__((unused)), @@ -29,21 +29,31 @@ set_process_pid( return DWARF_CB_OK; } -int -find_elf_through_process_root(void** userdata, const char* modname, char** file_name) +bool +is_deleted_mapping(const char* modname) { - if (userdata == nullptr || *userdata == nullptr || modname == nullptr || modname[0] != '/') { - return -1; + if (modname == nullptr) { + return false; } - const auto pid = *static_cast(*userdata); - if (pid <= 0) { + // proc_pid_maps(5) documents this suffix for deleted file-backed mappings. + const char* last_space = strrchr(modname, ' '); + return last_space != nullptr && strcmp(last_space, " (deleted)") == 0; +} + +// Open a mapped path relative to the target process root. +int +find_elf_through_proc_pid_root(void** userdata, const char* modname, char** file_name) +{ + if (userdata == nullptr || *userdata == nullptr || modname == nullptr || modname[0] != '/' + || is_deleted_mapping(modname)) + { return -1; } - const std::string rooted_path = - "/proc/" + std::to_string(pid) + "/root" + std::string(modname); - int fd = open(rooted_path.c_str(), O_RDONLY); + const auto pid = *static_cast(*userdata); + const std::string rooted_path = "/proc/" + std::to_string(pid) + "/root" + modname; + int fd = open(rooted_path.c_str(), O_RDONLY | O_CLOEXEC); if (fd < 0) { return -1; } @@ -55,7 +65,13 @@ find_elf_through_process_root(void** userdata, const char* modname, char** file_ } if (file_name != nullptr) { - *file_name = strdup(rooted_path.c_str()); + // Record the plain path when it is the same file on the host so that + // debuginfo and module-name lookups behave exactly as they always have. + struct stat host_file_stat; + const bool same_file = stat(modname, &host_file_stat) == 0 + && host_file_stat.st_dev == file_stat.st_dev + && host_file_stat.st_ino == file_stat.st_ino; + *file_name = strdup(same_file ? modname : rooted_path.c_str()); if (*file_name == nullptr) { close(fd); return -1; @@ -83,9 +99,12 @@ pystack_find_elf( LOG(DEBUG) << "Located debug info for " << the_modname << " using BUILD ID in " << the_filename; return ret; } - ret = dwfl_linux_proc_find_elf(mod, userdata, modname, base, file_name, elfp); + + // A path from /proc//maps belongs to the target's mount namespace. + // Prefer its process root, then retain libdwfl's existing fallback behavior. + ret = find_elf_through_proc_pid_root(userdata, modname, file_name); if (ret < 0) { - ret = find_elf_through_process_root(userdata, modname, file_name); + ret = dwfl_linux_proc_find_elf(mod, userdata, modname, base, file_name, elfp); } if (ret < 0) { LOG(DEBUG) << "Could not locate debug info for " << the_modname; @@ -342,7 +361,8 @@ ProcessAnalyzer::ProcessAnalyzer(pid_t pid) throw ElfAnalyzerError("Failed to analyze DWARF information for the remote process"); } - if (dwfl_getmodules(d_dwfl.get(), set_process_pid, &d_pid, 0) == -1) { + // The find_elf callback needs the PID to retry module paths through /proc//root. + if (dwfl_getmodules(d_dwfl.get(), set_module_process_pid_userdata, &d_pid, 0) == -1) { throw ElfAnalyzerError("Failed to associate DWARF modules with the remote process"); } @@ -552,7 +572,7 @@ static int module_callback( Dwfl_Module* mod, void** userdata __attribute__((unused)), - const char* name __attribute__((unused)), + const char* name, Dwarf_Addr starty __attribute__((unused)), void* arg) { @@ -565,15 +585,12 @@ module_callback( Dwarf_Addr end; const char* mainfile; const char* debugfile; - const char* modname = - dwfl_module_info(mod, nullptr, &start, &end, nullptr, nullptr, &mainfile, &debugfile); - if (mainfile != nullptr) { - modname = mainfile; - } else if (debugfile != nullptr) { - modname = debugfile; - } - - if (args->second == modname) { + dwfl_module_info(mod, nullptr, &start, &end, nullptr, nullptr, &mainfile, &debugfile); + // Match the reported mapping path as well as the located main/debug files, + // which may carry a /proc//root prefix. + if ((name != nullptr && args->second == name) || (mainfile != nullptr && args->second == mainfile) + || (debugfile != nullptr && args->second == debugfile)) + { args->first = start; return DWARF_CB_ABORT; } From bfeccb010f72bff1d4d95d60ed132c053c6e3e39 Mon Sep 17 00:00:00 2001 From: Pramod Kumbhar Date: Wed, 15 Jul 2026 11:51:22 +0000 Subject: [PATCH 3/3] Add process-root native-symbol integration test - Build target and host-decoy libraries at one mapped path to verify native symbols use the target filesystem view. - Require the mount-namespace test in coverage CI while allowing unsupported local environments to skip it. Signed-off-by: Pramod Kumbhar --- .github/workflows/coverage.yml | 12 +- .../native_process_root_program.py | 11 + tests/integration/test_native_process_root.py | 192 ++++++++++++++++++ 3 files changed, 213 insertions(+), 2 deletions(-) create mode 100644 tests/integration/native_process_root_program.py create mode 100644 tests/integration/test_native_process_root.py diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index fb00353e..fd7b9ca4 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -40,10 +40,12 @@ jobs: sudo apt-get install -qy \ gdb \ lcov \ + gcc \ cmake \ ninja-build \ libdw-dev \ libelf-dev \ + util-linux \ python3.10-dev \ python3.10-dbg - name: Install Python dependencies @@ -53,9 +55,15 @@ jobs: - name: Disable ptrace security restrictions run: | echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope + - name: Prepare process-root native-symbol test support + run: | + sudo sysctl -w kernel.unprivileged_userns_clone=1 || true + sudo sysctl -w user.max_user_namespaces=15000 || true + sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || true - name: Compute Python coverage run: | - python3 -m pytest -vvv --log-cli-level=info -s --color=yes \ + PYSTACK_REQUIRE_PROCESS_ROOT_TEST=1 \ + python3 -m pytest -vvv --log-cli-level=info -s --color=yes \ --cov=pystack --cov=tests --cov-config=pyproject.toml --cov-report=term \ --cov-append tests --cov-fail-under=85 python3 -m coverage lcov -i -o pycoverage.lcov @@ -64,7 +72,7 @@ jobs: run: | rm -rf build CFLAGS="-O0 -pg --coverage" CXXFLAGS="-O0 -pg --coverage" SKBUILD_BUILD_DIR=build pip install -e . --no-build-isolation - python3 -m pytest tests -v + PYSTACK_REQUIRE_PROCESS_ROOT_TEST=1 python3 -m pytest tests -v find build -name "*.gcda" -o -name "*.gcno" | head -5 lcov --capture --directory build --output-file cppcoverage.lcov lcov --extract cppcoverage.lcov '*/src/pystack/_pystack/*' --output-file cppcoverage.lcov diff --git a/tests/integration/native_process_root_program.py b/tests/integration/native_process_root_program.py new file mode 100644 index 00000000..fe3d1dc2 --- /dev/null +++ b/tests/integration/native_process_root_program.py @@ -0,0 +1,11 @@ +"""Load the target library and keep its native frame active for PyStack to sample.""" + +import ctypes +import os +import subprocess +import sys + +target_dir, mapped_dir, library_name, target_symbol = sys.argv[1:5] +subprocess.run(["mount", "--bind", target_dir, mapped_dir], check=True) +library = ctypes.CDLL(os.path.join(mapped_dir, library_name)) +getattr(library, target_symbol)() diff --git a/tests/integration/test_native_process_root.py b/tests/integration/test_native_process_root.py new file mode 100644 index 00000000..da86cc8a --- /dev/null +++ b/tests/integration/test_native_process_root.py @@ -0,0 +1,192 @@ +from __future__ import annotations + +import contextlib +import os +import select +import shutil +import subprocess +import sys +import textwrap +from pathlib import Path +from typing import Generator +from typing import NoReturn + +import pytest + +from pystack.engine import NativeReportingMode +from pystack.engine import get_process_threads +from tests.utils import TIMEOUT + +# Same filename, different symbols, so a host-path lookup is visible in frames. +LIBRARY_NAME = "libpystack_process_root_test.so" +TARGET_SYMBOL = "pystack_target_process_root_symbol" +HOST_DECOY_SYMBOL = "pystack_host_decoy_symbol" +READY_MESSAGE = b"ready" + +# Focused local/CI verification can turn namespace setup skips into failures. +REQUIRE_PROCESS_ROOT_TEST = os.environ.get("PYSTACK_REQUIRE_PROCESS_ROOT_TEST") == "1" + +TEST_PROCESS_ROOT_PROGRAM = Path(__file__).parent / "native_process_root_program.py" + + +def skip_or_fail(reason: str) -> NoReturn: + """Skip unless this test was explicitly requested as required.""" + if REQUIRE_PROCESS_ROOT_TEST: + pytest.fail(reason) + raise pytest.skip.Exception(reason) + + +def compile_native_sleeper(compiler: str, output: Path, symbol: str) -> None: + """Build a shared library that signals readiness from inside symbol.""" + source = output.with_suffix(".c") + source.write_text(textwrap.dedent(f""" + #include + + __attribute__((noinline)) void + {symbol}(void) + {{ + write(STDOUT_FILENO, "{READY_MESSAGE.decode()}", {len(READY_MESSAGE)}); + sleep(1000); + }} + """)) + subprocess.run( + [ + compiler, + "-g", + "-O0", + "-fno-omit-frame-pointer", + "-fPIC", + "-shared", + "-o", + str(output), + str(source), + ], + check=True, + ) + + +def mount_namespace_command() -> list[str] | None: + """Return an unshare command that can create the test namespace.""" + unshare = shutil.which("unshare") + if unshare is None: + return None + + if os.geteuid() == 0: + command = [unshare, "--mount", "--propagation", "private"] + else: + command = [ + unshare, + "--user", + "--map-root-user", + "--mount", + "--propagation", + "private", + ] + + result = subprocess.run( + [*command, "true"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + return command if result.returncode == 0 else None + + +def wait_for_target_ready(process: subprocess.Popen[str]) -> None: + """Wait for the readiness message written from inside the native symbol.""" + assert process.stdout is not None + stdout_fd = process.stdout.fileno() + readable, _, _ = select.select([stdout_fd], [], [], TIMEOUT) + if not readable: + process.kill() + pytest.fail("timed out waiting for target process") + if os.read(stdout_fd, len(READY_MESSAGE)) == READY_MESSAGE: + return + + # EOF on stdout: the target died before it was ready. + _, stderr = process.communicate() + message = stderr.strip() + if "Operation not permitted" in message or "permission denied" in message.lower(): + skip_or_fail(f"mount namespace setup is not permitted: {message}") + pytest.fail(f"target process exited before it was ready: {message}") + + +@contextlib.contextmanager +def spawn_namespaced_target( + unshare_command: list[str], + target_dir: Path, + mapped_dir: Path, +) -> Generator[subprocess.Popen[str], None, None]: + """Run the target after bind-mounting target_dir over mapped_dir.""" + with subprocess.Popen( + [ + *unshare_command, + sys.executable, + "-S", + str(TEST_PROCESS_ROOT_PROGRAM), + str(target_dir), + str(mapped_dir), + LIBRARY_NAME, + TARGET_SYMBOL, + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) as process: + wait_for_target_ready(process) + try: + yield process + finally: + process.kill() + process.wait(timeout=TIMEOUT) + + +@pytest.mark.skipif(sys.platform != "linux", reason="requires Linux procfs") +def test_native_symbols_use_target_process_root(tmp_path: Path) -> None: + """Verify native symbols use the target process's filesystem view. + + The target bind-mounts its library over a host decoy at the same path. + PyStack must report the target symbol and not the host symbol. + """ + + # GIVEN: the host and target process see different libraries at mapped_library. + compiler = shutil.which("cc") or shutil.which("gcc") + if compiler is None: + skip_or_fail("a C compiler is required to build the test shared libraries") + if shutil.which("mount") is None: + skip_or_fail("mount is required to set up the private mount namespace") + + unshare_command = mount_namespace_command() + if unshare_command is None: + skip_or_fail("user and mount namespaces are not available") + + target_dir = tmp_path / "target" + mapped_dir = tmp_path / "mapped" + target_dir.mkdir() + mapped_dir.mkdir() + + target_library = target_dir / LIBRARY_NAME + mapped_library = mapped_dir / LIBRARY_NAME + compile_native_sleeper(compiler, target_library, TARGET_SYMBOL) + compile_native_sleeper(compiler, mapped_library, HOST_DECOY_SYMBOL) + + with spawn_namespaced_target(unshare_command, target_dir, mapped_dir) as process: + process_root_library = Path(f"/proc/{process.pid}/root") / str( + mapped_library + ).lstrip(os.sep) + + # Verify the host path and target-root path resolve to different files. + assert not os.path.samefile(mapped_library, process_root_library) + + # WHEN: PyStack collects native frames from the target process. + threads = list( + get_process_threads( + process.pid, + native_mode=NativeReportingMode.PYTHON, + stop_process=True, + ) + ) + + # THEN: native symbols come from the target library, not the host decoy. + symbols = {frame.symbol for thread in threads for frame in thread.native_frames} + assert TARGET_SYMBOL in symbols + assert HOST_DECOY_SYMBOL not in symbols