Resolve native symbols for containerized python applications

.github/workflows/coverage.yml

@@ -40,10 +40,12 @@ jobs:
           sudo apt-get install -qy \
             gdb \
             lcov \
+            gcc \
             cmake \
             ninja-build \
             libdw-dev \
             libelf-dev \
+            util-linux \
             python3.10-dev \
             python3.10-dbg
       - name: Install Python dependencies
@@ -53,9 +55,15 @@ jobs:
       - name: Disable ptrace security restrictions
         run: |
           echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope
+      - name: Prepare process-root native-symbol test support
+        run: |
+          sudo sysctl -w kernel.unprivileged_userns_clone=1 || true
+          sudo sysctl -w user.max_user_namespaces=15000 || true
+          sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || true
       - name: Compute Python coverage
         run: |
-          python3 -m pytest -vvv --log-cli-level=info -s --color=yes \
+          PYSTACK_REQUIRE_PROCESS_ROOT_TEST=1 \
+            python3 -m pytest -vvv --log-cli-level=info -s --color=yes \
                       --cov=pystack --cov=tests --cov-config=pyproject.toml --cov-report=term \
                       --cov-append tests --cov-fail-under=85
           python3 -m coverage lcov -i -o pycoverage.lcov
@@ -64,7 +72,7 @@ jobs:
         run: |
           rm -rf build
           CFLAGS="-O0 -pg --coverage" CXXFLAGS="-O0 -pg --coverage" SKBUILD_BUILD_DIR=build pip install -e . --no-build-isolation
-          python3 -m pytest tests -v
+          PYSTACK_REQUIRE_PROCESS_ROOT_TEST=1 python3 -m pytest tests -v
           find build -name "*.gcda" -o -name "*.gcno" | head -5
           lcov --capture --directory build --output-file cppcoverage.lcov
           lcov --extract cppcoverage.lcov '*/src/pystack/_pystack/*' --output-file cppcoverage.lcov

src/pystack/_pystack/elf_common.cpp

@@ -5,6 +5,7 @@
 #include <iomanip>
 #include <iostream>
 #include <string>
+#include <sys/stat.h>
 #include <utility>
 
 #include "compat.h"
@@ -14,6 +15,74 @@ namespace pystack {
 
 using file_unique_ptr = std::unique_ptr<FILE, std::function<int(FILE*)>>;
 
+namespace {
+
+int
+set_module_process_pid_userdata(
+        Dwfl_Module* mod __attribute__((unused)),
+        void** userdata,
+        const char* name __attribute__((unused)),
+        Dwarf_Addr start __attribute__((unused)),
+        void* arg)
+{
+    *userdata = arg;
+    return DWARF_CB_OK;
+}
+
+bool
+is_deleted_mapping(const char* modname)
+{
+    if (modname == nullptr) {
+        return false;
+    }
+
+    // proc_pid_maps(5) documents this suffix for deleted file-backed mappings.
+    const char* last_space = strrchr(modname, ' ');
+    return last_space != nullptr && strcmp(last_space, " (deleted)") == 0;
+}
+
+// Open a mapped path relative to the target process root.
+int
+find_elf_through_proc_pid_root(void** userdata, const char* modname, char** file_name)
+{
+    if (userdata == nullptr || *userdata == nullptr || modname == nullptr || modname[0] != '/'
+        || is_deleted_mapping(modname))
+    {
+        return -1;
+    }
+
+    const auto pid = *static_cast<const int*>(*userdata);
+    const std::string rooted_path = "/proc/" + std::to_string(pid) + "/root" + modname;
+    int fd = open(rooted_path.c_str(), O_RDONLY | O_CLOEXEC);
+    if (fd < 0) {
+        return -1;
+    }
+
+    struct stat file_stat;
+    if (fstat(fd, &file_stat) != 0 || !S_ISREG(file_stat.st_mode)) {
+        close(fd);
+        return -1;
+    }
+
+    if (file_name != nullptr) {
+        // Record the plain path when it is the same file on the host so that
+        // debuginfo and module-name lookups behave exactly as they always have.
+        struct stat host_file_stat;
+        const bool same_file = stat(modname, &host_file_stat) == 0
+                               && host_file_stat.st_dev == file_stat.st_dev
+                               && host_file_stat.st_ino == file_stat.st_ino;
+        *file_name = strdup(same_file ? modname : rooted_path.c_str());
+        if (*file_name == nullptr) {
+            close(fd);
+            return -1;
+        }
+    }
+
+    return fd;
+}
+
+}  // namespace
+
 int
 pystack_find_elf(
         Dwfl_Module* mod,
@@ -30,11 +99,18 @@ pystack_find_elf(
         LOG(DEBUG) << "Located debug info for " << the_modname << " using BUILD ID in " << the_filename;
         return ret;
     }
-    ret = dwfl_linux_proc_find_elf(mod, userdata, modname, base, file_name, elfp);
-    if (file_name == nullptr) {
+
+    // A path from /proc/<pid>/maps belongs to the target's mount namespace.
+    // Prefer its process root, then retain libdwfl's existing fallback behavior.
+    ret = find_elf_through_proc_pid_root(userdata, modname, file_name);
+    if (ret < 0) {
+        ret = dwfl_linux_proc_find_elf(mod, userdata, modname, base, file_name, elfp);
+    }
+    if (ret < 0) {
         LOG(DEBUG) << "Could not locate debug info for " << the_modname;
     } else {
-        LOG(DEBUG) << "Located debug info for " << the_modname << " by path in " << *file_name;
+        const char* the_filename = (file_name == nullptr || *file_name == nullptr) ? "???" : *file_name;
+        LOG(DEBUG) << "Located debug info for " << the_modname << " by path in " << the_filename;
     }
     return ret;
 }
@@ -285,6 +361,11 @@ ProcessAnalyzer::ProcessAnalyzer(pid_t pid)
         throw ElfAnalyzerError("Failed to analyze DWARF information for the remote process");
     }
 
+    // The find_elf callback needs the PID to retry module paths through /proc/<pid>/root.
+    if (dwfl_getmodules(d_dwfl.get(), set_module_process_pid_userdata, &d_pid, 0) == -1) {
+        throw ElfAnalyzerError("Failed to associate DWARF modules with the remote process");
+    }
+
     if (dwfl_linux_proc_attach(d_dwfl.get(), pid, true) != 0) {
         throw ElfAnalyzerError("Could not attach the DWARF process analyzer");
     }
@@ -491,7 +572,7 @@ static int
 module_callback(
         Dwfl_Module* mod,
         void** userdata __attribute__((unused)),
-        const char* name __attribute__((unused)),
+        const char* name,
         Dwarf_Addr starty __attribute__((unused)),
         void* arg)
 {
@@ -504,15 +585,12 @@ module_callback(
     Dwarf_Addr end;
     const char* mainfile;
     const char* debugfile;
-    const char* modname =
-            dwfl_module_info(mod, nullptr, &start, &end, nullptr, nullptr, &mainfile, &debugfile);
-    if (mainfile != nullptr) {
-        modname = mainfile;
-    } else if (debugfile != nullptr) {
-        modname = debugfile;
-    }
-
-    if (args->second == modname) {
+    dwfl_module_info(mod, nullptr, &start, &end, nullptr, nullptr, &mainfile, &debugfile);
+    // Match the reported mapping path as well as the located main/debug files,
+    // which may carry a /proc/<pid>/root prefix.
+    if ((name != nullptr && args->second == name) || (mainfile != nullptr && args->second == mainfile)
+        || (debugfile != nullptr && args->second == debugfile))
+    {
         args->first = start;
         return DWARF_CB_ABORT;
     }

tests/integration/native_process_root_program.py

@@ -0,0 +1,11 @@
+"""Load the target library and keep its native frame active for PyStack to sample."""
+
+import ctypes
+import os
+import subprocess
+import sys
+
+target_dir, mapped_dir, library_name, target_symbol = sys.argv[1:5]
+subprocess.run(["mount", "--bind", target_dir, mapped_dir], check=True)
+library = ctypes.CDLL(os.path.join(mapped_dir, library_name))
+getattr(library, target_symbol)()

tests/integration/test_native_process_root.py

@@ -0,0 +1,192 @@
+from __future__ import annotations
+
+import contextlib
+import os
+import select
+import shutil
+import subprocess
+import sys
+import textwrap
+from pathlib import Path
+from typing import Generator
+from typing import NoReturn
+
+import pytest
+
+from pystack.engine import NativeReportingMode
+from pystack.engine import get_process_threads
+from tests.utils import TIMEOUT
+
+# Same filename, different symbols, so a host-path lookup is visible in frames.
+LIBRARY_NAME = "libpystack_process_root_test.so"
+TARGET_SYMBOL = "pystack_target_process_root_symbol"
+HOST_DECOY_SYMBOL = "pystack_host_decoy_symbol"
+READY_MESSAGE = b"ready"
+
+# Focused local/CI verification can turn namespace setup skips into failures.
+REQUIRE_PROCESS_ROOT_TEST = os.environ.get("PYSTACK_REQUIRE_PROCESS_ROOT_TEST") == "1"
+
+TEST_PROCESS_ROOT_PROGRAM = Path(__file__).parent / "native_process_root_program.py"
+
+
+def skip_or_fail(reason: str) -> NoReturn:
+    """Skip unless this test was explicitly requested as required."""
+    if REQUIRE_PROCESS_ROOT_TEST:
+        pytest.fail(reason)
+    raise pytest.skip.Exception(reason)
+
+
+def compile_native_sleeper(compiler: str, output: Path, symbol: str) -> None:
+    """Build a shared library that signals readiness from inside symbol."""
+    source = output.with_suffix(".c")
+    source.write_text(textwrap.dedent(f"""
+            #include <unistd.h>
+
+            __attribute__((noinline)) void
+            {symbol}(void)
+            {{
+                write(STDOUT_FILENO, "{READY_MESSAGE.decode()}", {len(READY_MESSAGE)});
+                sleep(1000);
+            }}
+            """))
+    subprocess.run(
+        [
+            compiler,
+            "-g",
+            "-O0",
+            "-fno-omit-frame-pointer",
+            "-fPIC",
+            "-shared",
+            "-o",
+            str(output),
+            str(source),
+        ],
+        check=True,
+    )
+
+
+def mount_namespace_command() -> list[str] | None:
+    """Return an unshare command that can create the test namespace."""
+    unshare = shutil.which("unshare")
+    if unshare is None:
+        return None
+
+    if os.geteuid() == 0:
+        command = [unshare, "--mount", "--propagation", "private"]
+    else:
+        command = [
+            unshare,
+            "--user",
+            "--map-root-user",
+            "--mount",
+            "--propagation",
+            "private",
+        ]
+
+    result = subprocess.run(
+        [*command, "true"],
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+    )
+    return command if result.returncode == 0 else None
+
+
+def wait_for_target_ready(process: subprocess.Popen[str]) -> None:
+    """Wait for the readiness message written from inside the native symbol."""
+    assert process.stdout is not None
+    stdout_fd = process.stdout.fileno()
+    readable, _, _ = select.select([stdout_fd], [], [], TIMEOUT)
+    if not readable:
+        process.kill()
+        pytest.fail("timed out waiting for target process")
+    if os.read(stdout_fd, len(READY_MESSAGE)) == READY_MESSAGE:
+        return
+
+    # EOF on stdout: the target died before it was ready.
+    _, stderr = process.communicate()
+    message = stderr.strip()
+    if "Operation not permitted" in message or "permission denied" in message.lower():
+        skip_or_fail(f"mount namespace setup is not permitted: {message}")
+    pytest.fail(f"target process exited before it was ready: {message}")
+
+
+@contextlib.contextmanager
+def spawn_namespaced_target(
+    unshare_command: list[str],
+    target_dir: Path,
+    mapped_dir: Path,
+) -> Generator[subprocess.Popen[str], None, None]:
+    """Run the target after bind-mounting target_dir over mapped_dir."""
+    with subprocess.Popen(
+        [
+            *unshare_command,
+            sys.executable,
+            "-S",
+            str(TEST_PROCESS_ROOT_PROGRAM),
+            str(target_dir),
+            str(mapped_dir),
+            LIBRARY_NAME,
+            TARGET_SYMBOL,
+        ],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True,
+    ) as process:
+        wait_for_target_ready(process)
+        try:
+            yield process
+        finally:
+            process.kill()
+            process.wait(timeout=TIMEOUT)
+
+
+@pytest.mark.skipif(sys.platform != "linux", reason="requires Linux procfs")
+def test_native_symbols_use_target_process_root(tmp_path: Path) -> None:
+    """Verify native symbols use the target process's filesystem view.
+
+    The target bind-mounts its library over a host decoy at the same path.
+    PyStack must report the target symbol and not the host symbol.
+    """
+
+    # GIVEN: the host and target process see different libraries at mapped_library.
+    compiler = shutil.which("cc") or shutil.which("gcc")
+    if compiler is None:
+        skip_or_fail("a C compiler is required to build the test shared libraries")
+    if shutil.which("mount") is None:
+        skip_or_fail("mount is required to set up the private mount namespace")
+
+    unshare_command = mount_namespace_command()
+    if unshare_command is None:
+        skip_or_fail("user and mount namespaces are not available")
+
+    target_dir = tmp_path / "target"
+    mapped_dir = tmp_path / "mapped"
+    target_dir.mkdir()
+    mapped_dir.mkdir()
+
+    target_library = target_dir / LIBRARY_NAME
+    mapped_library = mapped_dir / LIBRARY_NAME
+    compile_native_sleeper(compiler, target_library, TARGET_SYMBOL)
+    compile_native_sleeper(compiler, mapped_library, HOST_DECOY_SYMBOL)
+
+    with spawn_namespaced_target(unshare_command, target_dir, mapped_dir) as process:
+        process_root_library = Path(f"/proc/{process.pid}/root") / str(
+            mapped_library
+        ).lstrip(os.sep)
+
+        # Verify the host path and target-root path resolve to different files.
+        assert not os.path.samefile(mapped_library, process_root_library)
+
+        # WHEN: PyStack collects native frames from the target process.
+        threads = list(
+            get_process_threads(
+                process.pid,
+                native_mode=NativeReportingMode.PYTHON,
+                stop_process=True,
+            )
+        )
+
+        # THEN: native symbols come from the target library, not the host decoy.
+        symbols = {frame.symbol for thread in threads for frame in thread.native_frames}
+        assert TARGET_SYMBOL in symbols
+        assert HOST_DECOY_SYMBOL not in symbols