diff --git a/.ci/scripts/test-cuda-build.sh b/.ci/scripts/test-cuda-build.sh index e717718be66..4326aba8f33 100755 --- a/.ci/scripts/test-cuda-build.sh +++ b/.ci/scripts/test-cuda-build.sh @@ -80,6 +80,53 @@ except Exception as e: exit(1) " + # The CUDA delegate ships as its own shared library. Nothing else here would + # notice if it were built into more than one place, and a process with two + # copies of the delegate has two copies of its state, so check that the + # installed tree defines it exactly once. + python -c " +import shutil +import subprocess +import sys +from pathlib import Path + +if shutil.which('nm') is None: + print('INFO: nm unavailable, skipping the delegate duplication check') + sys.exit(0) + +import executorch + +# A namespace package has no __file__, so derive the directory from the loader's +# search path instead. +locations = list(getattr(executorch, '__path__', []) or []) +if not locations: + print('INFO: cannot locate the installed package, skipping the check') + sys.exit(0) +package = Path(locations[0]) +symbol = 'executorch::backends::cuda::clearCurrentCUDAStream' +libraries = [p for p in package.rglob('*.so*') if p.is_file() and not p.is_symlink()] +definers = [] +for library in libraries: + result = subprocess.run( + ['nm', '-DC', str(library)], capture_output=True, text=True, check=False + ) + if result.returncode != 0: + continue + for line in result.stdout.splitlines(): + parts = line.split(maxsplit=2) + if len(parts) == 3 and parts[1] in 'TtWVu' and parts[2].startswith(symbol): + definers.append(str(library.relative_to(package))) + break + +if not definers: + print('INFO: no CUDA delegate in this install, nothing to check') + sys.exit(0) +if len(definers) != 1: + print(f'ERROR: expected one library to define the CUDA delegate, found {definers}') + sys.exit(1) +print(f'SUCCESS: exactly one CUDA delegate across {len(libraries)} shipped libraries') +" || exit $? + echo "SUCCESS: ExecuTorch CUDA ${cuda_version} build and verification completed successfully" } diff --git a/.ci/scripts/wheel/test_cpp_sdk.py b/.ci/scripts/wheel/test_cpp_sdk.py index 7f72167b044..8004bb0dccc 100644 --- a/.ci/scripts/wheel/test_cpp_sdk.py +++ b/.ci/scripts/wheel/test_cpp_sdk.py @@ -26,6 +26,7 @@ import re import shutil import subprocess +import tempfile from pathlib import Path # Registry entry points. A second definer of any of these means a second @@ -59,6 +60,10 @@ "executorch::backends::xnnpack::XnnpackBackendOptions::workspace_manager", ) +# A representative symbol from the CUDA delegate's shim layer. The delegate's own +# methods are weak symbols, so this checks a strong one instead. +_CUDA_SYMBOLS = ("executorch::backends::cuda::clearCurrentCUDAStream",) + # `nm -DC` prints " " for a definition and # " U " for an undefined reference. _DEFINED = re.compile(r"^[0-9a-fA-F]+\s+(?P[A-Za-z])\s+(?P.+)$") @@ -149,16 +154,210 @@ def _report_definers(symbols, what: str) -> None: print(f" {definer}") -def _assert_single_definer(symbols, what: str) -> None: - """Exactly one shipped library may define each of `symbols`.""" +def report_wheel_composition() -> None: + """Print what the wheel ships and what each library needs. + + Not an assertion. A size jump or an unexpected external dependency is the + first visible sign that a component got statically duplicated again, so the + numbers are worth having in the log of every run. + """ + package_dir = _installed_package_dir() + libraries = _shipped_shared_objects(package_dir) + + print("shipped libraries:") + total = 0 + for library in sorted(libraries, key=lambda path: path.name): + size = library.stat().st_size + total += size + print(f" {size / 1024:9.1f} KiB {library.relative_to(package_dir)}") + print(f" {total / 1024:9.1f} KiB total") + + if shutil.which("readelf") is None: + return + # Anything the libraries need that the wheel does not itself ship has to be + # present on the user's machine, so it belongs in the report. Compare against + # the shipped file names rather than guessing from name prefixes. + shipped = {library.name for library in libraries} + external = set() + for library in libraries: + dynamic = subprocess.run( + ["readelf", "-d", str(library)], + capture_output=True, + text=True, + check=False, + ).stdout + for line in dynamic.splitlines(): + if "(NEEDED)" not in line or "[" not in line: + continue + name = line.split("[", 1)[1].rstrip("]").strip() + if name not in shipped: + external.add(name) + if external: + print("external dependencies expected from the environment:") + for name in sorted(external): + print(f" {name}") + + +def test_shipped_libraries_load() -> None: + """Every shipped library must depend only on things that exist. + + The symbol checks prove each component is defined exactly once, but a library + can still be unloadable if it needs something nothing provides, which is a + packaging bug rather than a duplication bug. + + A dependency the wheel ships elsewhere is fine even when `ldd` cannot resolve + it: some extensions are loaded after `import torch` has already brought their + dependencies into the process, so they intentionally carry no path to them. + Only a name nothing in the wheel provides is a real problem. + """ + if shutil.which("ldd") is None: + print("- ldd not available, skipping the load check") + return + + package_dir = _installed_package_dir() + libraries = _shipped_shared_objects(package_dir) + shipped = {library.name for library in libraries} + + # A dependency is only excusable when the wheel ships it AND the loader can + # actually reach it from the library that needs it. Loaded-later extensions + # such as the Torch libraries are the real exception: they resolve once the + # Python package that owns them is imported. Anything the wheel itself ships + # must resolve here, because a RUNPATH applies to the library carrying it and + # is not inherited on behalf of a dependency's own dependencies. + broken = {} + unreachable = {} + for library in libraries: + resolved = subprocess.run( + ["ldd", str(library)], + capture_output=True, + text=True, + check=False, + # Any LD_LIBRARY_PATH in the build environment would paper over a + # RUNPATH the shipped library is actually missing. + env={ + key: value + for key, value in os.environ.items() + if key != "LD_LIBRARY_PATH" + }, + ).stdout + missing = [ + line.split("=>")[0].strip() + for line in resolved.splitlines() + if "not found" in line + ] + absent = [name for name in missing if name not in shipped] + present_but_unreachable = [name for name in missing if name in shipped] + if absent: + broken[str(library.relative_to(package_dir))] = absent + if present_but_unreachable: + unreachable[str(library.relative_to(package_dir))] = present_but_unreachable + + assert not broken, ( + "shipped libraries need dependencies that nothing provides, so they will " + f"fail to load: {broken}" + ) + assert not unreachable, ( + "shipped libraries need dependencies the wheel ships but the loader " + "cannot reach from them, which usually means a missing RUNPATH entry: " + f"{unreachable}" + ) + print("✓ every shipped library resolves every dependency it needs") + + +def test_shipped_libraries_resolve_without_build_tree() -> None: + """A shipped library must resolve using only its relative runtime paths. + + Packaging copies binaries out of the build directory, so they still carry the + absolute paths they were linked with. On the machine that produced the wheel + those paths exist, which means a library whose relative path is wrong can still + resolve and look correct. Anywhere else it would fail. + + Copy each library and its wheel-provided dependencies into a fresh tree that + mirrors the wheel layout, drop every absolute runtime path, and check what is + left is enough. + """ + if shutil.which("ldd") is None or shutil.which("patchelf") is None: + print("- ldd or patchelf unavailable, skipping the relocated load check") + return + + package_dir = _installed_package_dir() + libraries = _shipped_shared_objects(package_dir) + environment = { + key: value for key, value in os.environ.items() if key != "LD_LIBRARY_PATH" + } + + with tempfile.TemporaryDirectory() as work_dir: + root = Path(work_dir) / package_dir.name + # Mirror the layout so a relative path such as $ORIGIN/../../lib still + # points where it would in a real install. + for library in libraries: + target = root / library.relative_to(package_dir) + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(library, target) + + broken = {} + for library in libraries: + target = root / library.relative_to(package_dir) + current = subprocess.run( + ["patchelf", "--print-rpath", str(target)], + capture_output=True, + text=True, + check=False, + ).stdout.strip() + relative = [ + entry for entry in current.split(":") if entry.startswith("$ORIGIN") + ] + subprocess.run( + ["patchelf", "--set-rpath", ":".join(relative), str(target)], + check=False, + ) + resolved = subprocess.run( + ["ldd", str(target)], + capture_output=True, + text=True, + check=False, + env=environment, + ).stdout + shipped = {item.name for item in libraries} + missing = [ + line.split("=>")[0].strip() + for line in resolved.splitlines() + if "not found" in line and line.split("=>")[0].strip() in shipped + ] + if missing: + broken[str(library.relative_to(package_dir))] = missing + + assert not broken, ( + "shipped libraries only resolve their wheel-provided dependencies " + "through absolute build paths, so they would fail on any other " + f"machine: {broken}" + ) + print("✓ every shipped library resolves without the build tree") + + +def _assert_single_definer(symbols, what: str, optional: bool = False) -> None: + """Exactly one shipped library may define each of `symbols`. + + `optional` allows a component that is only present in some wheel flavors, + such as an accelerator delegate, to be absent without failing. + """ assert shutil.which("nm") is not None, "nm is required to inspect the wheel" package_dir = _installed_package_dir() libraries = _shipped_shared_objects(package_dir) assert libraries, f"no shared libraries found under {package_dir}" - for symbol in symbols: - definers = [lib for lib in libraries if _defines_symbol(lib, symbol)] + # Resolve every symbol first, so a component that is only half present is + # reported rather than being mistaken for one that is absent entirely. + found = { + symbol: [lib for lib in libraries if _defines_symbol(lib, symbol)] + for symbol in symbols + } + if optional and not any(found.values()): + print(f"- no {what} in this wheel, skipping") + return + + for symbol, definers in found.items(): pretty = [str(lib.relative_to(package_dir)) for lib in definers] assert len(definers) == 1, ( f"expected exactly one library to define {symbol}, found " @@ -194,6 +393,11 @@ def test_single_xnnpack_delegate() -> None: _assert_single_definer(_XNNPACK_SYMBOLS, "XNNPACK delegate") +def test_single_cuda_delegate() -> None: + """Exactly one shipped library may define the CUDA delegate, if present.""" + _assert_single_definer(_CUDA_SYMBOLS, "CUDA delegate", optional=True) + + def test_cpp_consumer(work_dir: Path) -> None: """A standalone C++ app builds and runs against the installed wheel.""" assert shutil.which("cmake") is not None, "cmake is required to build a consumer" @@ -292,8 +496,12 @@ def _assert_runs_relocated(consumer, package_dir, work_dir, environment) -> None def run_tests(work_dir: Path) -> None: + report_wheel_composition() + test_shipped_libraries_load() + test_shipped_libraries_resolve_without_build_tree() test_single_backend_registry() test_single_threadpool() test_single_kernel_registration() test_single_xnnpack_delegate() + test_single_cuda_delegate() test_cpp_consumer(work_dir) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1816ba41d9e..98d70292092 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1273,7 +1273,9 @@ if(EXECUTORCH_BUILD_PYBIND) # initializer, so nothing here references a symbol from them and some linkers # drop them from DT_NEEDED. That surfaces at runtime as a missing kernel or an # unregistered backend rather than as a link error. - foreach(_retained_component optimized_native_cpu_ops_lib xnnpack_backend) + foreach(_retained_component optimized_native_cpu_ops_lib xnnpack_backend + aoti_cuda_backend + ) if(TARGET ${_retained_component}) executorch_target_retain_shared_library( portable_lib ${_retained_component} diff --git a/backends/cuda/CMakeLists.txt b/backends/cuda/CMakeLists.txt index 06990692428..b46c7b34ef1 100644 --- a/backends/cuda/CMakeLists.txt +++ b/backends/cuda/CMakeLists.txt @@ -93,8 +93,17 @@ target_compile_options( PUBLIC "$<$:${_cuda_cxx_compile_options}>" ) -# Link against ExecuTorch core libraries -target_link_libraries(cuda_platform PRIVATE executorch_core ${CMAKE_DL_LIBS}) +# Link against ExecuTorch core libraries. Resolve them from the shared runtime +# when there is one, so this does not carry a second copy of the backend +# registry. +if(EXECUTORCH_BUILD_SHARED) + target_link_libraries( + cuda_platform PRIVATE executorch_shared ${CMAKE_DL_LIBS} + ) + executorch_target_link_shared_runtime(cuda_platform) +else() + target_link_libraries(cuda_platform PRIVATE executorch_core ${CMAKE_DL_LIBS}) +endif() install( TARGETS cuda_platform @@ -169,14 +178,9 @@ if(_cuda_is_msvc_toolchain) else() target_link_libraries( aoti_cuda_shims - PRIVATE cuda_platform - PUBLIC -Wl,--whole-archive - aoti_common_shims_slim - -Wl,--no-whole-archive - CUDA::cudart - CUDA::curand - extension_cuda - ${CMAKE_DL_LIBS} + PRIVATE cuda_platform -Wl,--whole-archive aoti_common_shims_slim + -Wl,--no-whole-archive + PUBLIC CUDA::cudart CUDA::curand extension_cuda ${CMAKE_DL_LIBS} ) endif() @@ -200,7 +204,47 @@ if(_cuda_is_msvc_toolchain) list(APPEND _aoti_cuda_backend_sources runtime/cuda_allocator.cpp) endif() -add_library(aoti_cuda_backend STATIC ${_aoti_cuda_backend_sources}) +# Build the delegate as a shared library for the wheel so a process has one copy +# of it, and keep it static everywhere else so no other build changes. +if(EXECUTORCH_BUILD_SHARED) + set(_aoti_cuda_backend_library_type SHARED) +else() + set(_aoti_cuda_backend_library_type STATIC) +endif() +add_library( + aoti_cuda_backend ${_aoti_cuda_backend_library_type} + ${_aoti_cuda_backend_sources} +) +if(EXECUTORCH_BUILD_SHARED) + set_target_properties( + aoti_cuda_backend + PROPERTIES OUTPUT_NAME executorch_cuda_backend + VERSION "${PROJECT_VERSION}" + SOVERSION "${PROJECT_VERSION_MAJOR}" + ) + if(NOT APPLE) + # Ships in the wheel's lib/ directory, but the CUDA shim library it links + # lives under backends/cuda, so both locations have to be searchable. The + # CUDA runtime itself comes from the environment and is not bundled. + set(_cuda_backend_rpath "$ORIGIN:$ORIGIN/../backends/cuda") + set_target_properties( + aoti_cuda_backend PROPERTIES BUILD_RPATH "${_cuda_backend_rpath}" + INSTALL_RPATH "${_cuda_backend_rpath}" + ) + # The shim needs its own entry rather than relying on the backend's. A + # RUNPATH applies to the library that carries it, not to what its own + # dependencies need, so loading the shim first, or on its own, would fail to + # find the extension library it links. The shim ships under backends/cuda + # while that library ships in the wheel's lib/ directory. + if(TARGET aoti_cuda_shims) + set(_cuda_shims_rpath "$ORIGIN:$ORIGIN/../../lib") + set_target_properties( + aoti_cuda_shims PROPERTIES BUILD_RPATH "${_cuda_shims_rpath}" + INSTALL_RPATH "${_cuda_shims_rpath}" + ) + endif() + endif() +endif() target_include_directories( aoti_cuda_backend diff --git a/setup.py b/setup.py index cf08272eb7c..ec35a205c82 100644 --- a/setup.py +++ b/setup.py @@ -1174,6 +1174,36 @@ def run(self): # noqa C901 "EXECUTORCH_BUILD_XNNPACK", ], ), + # Install the CUDA delegate beside them when it is built. The CUDA + # runtime itself is not bundled; it comes from the environment. + BuiltFile( + src_dir="%CMAKE_CACHE_DIR%/backends/cuda/", + src_name=( + "libexecutorch_cuda_backend.so." + f"{get_runtime_soname_major()}.*" + ), + dst=( + "executorch/lib/libexecutorch_cuda_backend.so." + f"{get_runtime_soname_major()}" + ), + dependent_cmake_flags=[ + "EXECUTORCH_BUILD_SHARED", + "EXECUTORCH_BUILD_CUDA", + ], + ), + # The CUDA delegate calls into this for stream handling, so an + # application that links the delegate from the wheel cannot + # resolve it unless this ships too. It carries no SONAME version, + # so the name is used as built. + BuiltFile( + src_dir="%CMAKE_CACHE_DIR%/extension/cuda/", + src_name="libextension_cuda.so", + dst="executorch/lib/libextension_cuda.so", + dependent_cmake_flags=[ + "EXECUTORCH_BUILD_SHARED", + "EXECUTORCH_BUILD_CUDA", + ], + ), # Install the prebuilt pybindings extension wrapper for the runtime, # portable kernels, and a selection of backends. This lets users # load and execute .pte files from python.