From fbb117d8aba486b7b80e3fa7c61e4ad629bbc64e Mon Sep 17 00:00:00 2001 From: chaofengw Date: Mon, 7 Sep 2026 09:45:45 +0000 Subject: [PATCH 01/13] feat(cli): discover runtime roots automatically Let execution commands resolve a complete runtime cohort from the current directory, the active CLI installation, or TRTMC_RUNTIME_PATH when --runtime-root is omitted. Mark native artifacts with a configure-scoped cohort identity so discovery cannot combine core, runtime, backend, family, or BYOK DSOs from different builds. Keep explicit roots and the public loader contract unchanged. Signed-off-by: chaofengw --- CMakeLists.txt | 24 +- README.md | 1 - apps/cli/cli.cpp | 310 +++++++++++++++++- apps/cli/cli.h | 11 + apps/cli/tests/test_cli.cpp | 250 +++++++++++++- tools/ci/package.py | 14 +- .../ai-native-horizontal-scaling.md | 10 + website/docs/getting-started/quick-start.md | 36 +- 8 files changed, 625 insertions(+), 31 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1098879680..17098f4fc9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,6 +12,20 @@ set(CMAKE_CUDA_STANDARD_REQUIRED ON) set(CMAKE_POSITION_INDEPENDENT_CODE ON) set(CMAKE_BUILD_RPATH_USE_ORIGIN TRUE) +if(NOT TRTMC_BUILD_COHORT_ID) + string(RANDOM LENGTH 32 ALPHABET 0123456789abcdef TRTMC_BUILD_COHORT_ID) +endif() +string(LENGTH "${TRTMC_BUILD_COHORT_ID}" _trtmc_build_cohort_id_length) +if(NOT _trtmc_build_cohort_id_length EQUAL 32 OR + NOT TRTMC_BUILD_COHORT_ID MATCHES "^[0-9a-f]+$") + message(FATAL_ERROR "TRTMC_BUILD_COHORT_ID must contain exactly 32 lowercase hex characters") +endif() +set(_trtmc_build_cohort_symbol "trtmc_build_cohort_${TRTMC_BUILD_COHORT_ID}") +add_link_options( + "LINKER:--defsym=${_trtmc_build_cohort_symbol}=0" + "LINKER:--export-dynamic-symbol=${_trtmc_build_cohort_symbol}" +) + include(GNUInstallDirs) find_package(CUDAToolkit REQUIRED) find_package(nlohmann_json 3.11 REQUIRED) @@ -244,6 +258,8 @@ endif() option(TRTMC_BUILD_TESTS "Build tests" ON) option(TRTMC_BUILD_EXAMPLES "Build examples" ON) if(TRTMC_BUILD_TESTS) + set(_trtmc_test_runtime_root "${CMAKE_BINARY_DIR}/tests/runtime") + enable_testing() endif() @@ -346,9 +362,12 @@ if(TRTMC_BUILD_TESTS) ) target_link_libraries(test_cli PRIVATE trtmc_cli nlohmann_json::nlohmann_json) target_compile_options(test_cli PRIVATE -Wall -Wextra -Wpedantic) - add_test(NAME cli COMMAND test_cli) + add_test(NAME cli COMMAND test_cli + "${_trtmc_test_runtime_root}" + $ + $ + ) - set(_trtmc_test_runtime_root "${CMAKE_BINARY_DIR}/tests/runtime") add_library(trtmc_test_backend_fake SHARED core/runtime/tests/fake_backend.cpp) target_include_directories(trtmc_test_backend_fake PRIVATE ${PROJECT_SOURCE_DIR}/core/runtime/include) target_link_libraries(trtmc_test_backend_fake PRIVATE CUDA::cudart) @@ -378,6 +397,7 @@ if(TRTMC_BUILD_TESTS) LIBRARY_OUTPUT_DIRECTORY "${_trtmc_test_runtime_root}" BUILD_RPATH "\$ORIGIN/../.." ) + add_dependencies(test_cli trtmc_test_backend_fake trtmc_test_family_fake) add_executable(test_family_loader core/runtime/tests/test_family_loader.cpp) target_include_directories(test_family_loader PRIVATE diff --git a/README.md b/README.md index a68e9a7afb..ed0200c1fb 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,6 @@ python -m tensorrt_model_connect build Qwen/Qwen3-0.6B \ --max-sequence-length 16384 \ --output qwen3-0.6b.bundle trtmc run ./qwen3-0.6b.bundle \ - --runtime-root /opt/trtmc/lib \ --prompt "What is the capital of France? Answer in one word." \ --use-chat-template true \ --enable-thinking false diff --git a/apps/cli/cli.cpp b/apps/cli/cli.cpp index 6aee9dd1df..a8fabbba03 100644 --- a/apps/cli/cli.cpp +++ b/apps/cli/cli.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -22,9 +23,11 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -141,6 +144,183 @@ bool is_byok_option(const std::string& option) { return option == "--byok-library" || option == "--byok-function" || option == "--byok-name"; } +void append_candidate(std::vector& candidates, std::set& seen, + const fs::path& candidate) { + if (candidate.empty()) + return; + std::error_code error; + fs::path absolute = fs::absolute(candidate, error); + if (error) + return; + fs::path normalized = fs::weakly_canonical(absolute, error); + if (error) + normalized = absolute.lexically_normal(); + const std::string key = normalized.string(); + if (seen.insert(key).second) + candidates.push_back(std::move(normalized)); +} + +void append_path_list(std::vector& candidates, std::set& seen, + const std::string& paths) { + std::size_t begin = 0; + while (begin <= paths.size()) { + const std::size_t end = paths.find(':', begin); + const std::string path = + paths.substr(begin, end == std::string::npos ? std::string::npos : end - begin); + if (!path.empty()) + append_candidate(candidates, seen, path); + if (end == std::string::npos) + break; + begin = end + 1; + } +} + +void append_python_package_runtime_roots(std::vector& candidates, + std::set& seen, const fs::path& prefix) { + std::vector python_roots; + for (const auto& library_root : {prefix / "lib", prefix / "lib64"}) { + std::error_code error; + if (!fs::is_directory(library_root, error)) + continue; + for (fs::directory_iterator iterator(library_root, error), end; !error && iterator != end; + iterator.increment(error)) { + std::error_code entry_error; + if (!iterator->is_directory(entry_error)) + continue; + const std::string name = iterator->path().filename().string(); + if (name.rfind("python", 0) != 0) + continue; + for (const auto& packages : {"site-packages", "dist-packages"}) { + python_roots.push_back(iterator->path() / packages / "tensorrt_model_connect" / + "bin"); + } + } + } + std::sort(python_roots.begin(), python_roots.end()); + for (const auto& root : python_roots) + append_candidate(candidates, seen, root); +} + +std::string read_build_cohort(const fs::path& library) { + static constexpr char prefix[] = "trtmc_build_cohort_"; + static constexpr std::size_t id_size = 32; + static constexpr std::uint64_t max_string_table_size = 16ULL * 1024ULL * 1024ULL; + std::ifstream input(library, std::ios::binary); + if (!input) + return {}; + + std::error_code error; + const std::uintmax_t file_size = fs::file_size(library, error); + if (error || file_size < sizeof(Elf64_Ehdr)) + return {}; + const auto read_at = [&](std::uint64_t offset, void* destination, std::size_t size) { + if (offset > file_size || size > file_size - offset) + return false; + input.clear(); + input.seekg(static_cast(offset)); + input.read(static_cast(destination), static_cast(size)); + return input.good(); + }; + + Elf64_Ehdr header{}; + if (!read_at(0, &header, sizeof(header)) || header.e_ident[EI_MAG0] != ELFMAG0 || + header.e_ident[EI_MAG1] != ELFMAG1 || header.e_ident[EI_MAG2] != ELFMAG2 || + header.e_ident[EI_MAG3] != ELFMAG3 || header.e_ident[EI_CLASS] != ELFCLASS64 || + header.e_ident[EI_DATA] != ELFDATA2LSB || header.e_shentsize != sizeof(Elf64_Shdr) || + header.e_shnum == 0 || header.e_shstrndx >= header.e_shnum || header.e_shoff > file_size || + header.e_shnum > (file_size - header.e_shoff) / sizeof(Elf64_Shdr)) { + return {}; + } + + const auto read_section_header = [&](std::size_t index, Elf64_Shdr& section) { + return read_at(header.e_shoff + index * sizeof(Elf64_Shdr), §ion, sizeof(section)); + }; + Elf64_Shdr names_header{}; + if (!read_section_header(header.e_shstrndx, names_header) || + names_header.sh_size > max_string_table_size || names_header.sh_offset > file_size || + names_header.sh_size > file_size - names_header.sh_offset) { + return {}; + } + std::string names(static_cast(names_header.sh_size), '\0'); + if (!read_at(names_header.sh_offset, names.data(), names.size())) + return {}; + + for (std::size_t index = 0; index < header.e_shnum; ++index) { + Elf64_Shdr section{}; + if (!read_section_header(index, section) || section.sh_name >= names.size()) + return {}; + const auto name_end = names.find('\0', section.sh_name); + if (name_end == std::string::npos || + names.compare(section.sh_name, name_end - section.sh_name, ".dynstr") != 0) { + continue; + } + if (section.sh_size > max_string_table_size || section.sh_offset > file_size || + section.sh_size > file_size - section.sh_offset) { + return {}; + } + std::string strings(static_cast(section.sh_size), '\0'); + if (!read_at(section.sh_offset, strings.data(), strings.size())) + return {}; + std::size_t position = strings.find(prefix); + while (position != std::string::npos) { + const std::size_t id_begin = position + sizeof(prefix) - 1; + const std::size_t marker_end = id_begin + id_size; + if (marker_end < strings.size() && strings[marker_end] == '\0' && + std::all_of(strings.begin() + static_cast(id_begin), + strings.begin() + static_cast(marker_end), + [](const unsigned char character) { + return (character >= '0' && character <= '9') || + (character >= 'a' && character <= 'f'); + })) { + return strings.substr(id_begin, id_size); + } + position = strings.find(prefix, position + 1); + } + return {}; + } + return {}; +} + +RuntimeRootSearchContext runtime_root_search_context() { + RuntimeRootSearchContext context; + std::error_code error; + context.current_directory = fs::current_path(error); + + fs::path core_library; + Dl_info core_info{}; + using InspectBundleFn = BundleInfo (*)(const std::string&); + const auto inspect_bundle_function = static_cast(&InspectBundle); + if (dladdr(reinterpret_cast(inspect_bundle_function), &core_info) != 0 && + core_info.dli_fname != nullptr) { + core_library = core_info.dli_fname; + } + + Dl_info runtime_info{}; + using LoadTaskFn = std::unique_ptr (*)(const std::string&, const std::string&, + std::uint64_t, const std::string&, bool); + const auto load_task_function = static_cast(&load_task); + if (dladdr(reinterpret_cast(load_task_function), &runtime_info) != 0 && + runtime_info.dli_fname != nullptr) { + context.runtime_library = runtime_info.dli_fname; + } + + const std::string core_cohort = read_build_cohort(core_library); + const std::string runtime_cohort = read_build_cohort(context.runtime_library); + if (!core_cohort.empty() && core_cohort == runtime_cohort) + context.cohort_id = core_cohort; + + std::vector executable(4096, '\0'); + const ssize_t length = readlink("/proc/self/exe", executable.data(), executable.size() - 1); + if (length > 0) { + executable[static_cast(length)] = '\0'; + context.executable = executable.data(); + } + + if (const char* value = std::getenv("TRTMC_RUNTIME_PATH")) + context.runtime_path = value; + return context; +} + void load_byok_extension(const Command& command) { using LoadKernelFn = const char* (*)(const char*, const char*, const char*) noexcept; const fs::path extension = fs::path(command.runtime_root) / "libtrtmc_byok_tvm_ffi.so"; @@ -661,6 +841,117 @@ int dispatch_run(const Command& command, ITask& task, std::ostream& output) { } // namespace +std::string resolve_runtime_root(const BundleInfo& bundle, const std::string& explicit_root, + bool require_byok, const RuntimeRootSearchContext& context) { + if (!explicit_root.empty()) + return explicit_root; + + const auto is_safe_id = [](const std::string& value) { + if (value.empty() || value.front() < 'a' || value.front() > 'z') + return false; + return std::all_of(value.begin(), value.end(), [](const unsigned char character) { + return (character >= 'a' && character <= 'z') || + (character >= '0' && character <= '9') || character == '_'; + }); + }; + if (!is_safe_id(bundle.family) || !is_safe_id(bundle.backend)) + throw std::runtime_error("Cannot discover a runtime for unsafe bundle family/backend IDs"); + if (context.runtime_library.empty() || context.cohort_id.empty()) { + throw std::runtime_error( + "Unable to identify one build cohort for the loaded TRTMC core/runtime libraries; " + "pass --runtime-root DIR"); + } + + std::vector current_candidates; + std::vector installed_candidates; + std::vector wheel_candidates; + std::vector configured_candidates; + std::set seen; + append_candidate(current_candidates, seen, context.current_directory); + append_candidate(installed_candidates, seen, context.runtime_library.parent_path()); + + if (!context.executable.empty()) { + const fs::path executable_directory = context.executable.parent_path(); + append_candidate(installed_candidates, seen, executable_directory); + if (executable_directory.filename() == "bin") { + const fs::path prefix = executable_directory.parent_path(); + append_candidate(installed_candidates, seen, prefix / "lib"); + append_candidate(installed_candidates, seen, prefix / "lib64"); + append_python_package_runtime_roots(wheel_candidates, seen, prefix); + } + } + + append_path_list(configured_candidates, seen, context.runtime_path); + + std::vector required{ + "libtrtmc_core.so", + "libtrtmc_runtime.so", + "libtrtmc_backend_" + bundle.backend + ".so", + "libtrtmc_model_" + bundle.family + ".so", + }; + if (require_byok) + required.emplace_back("libtrtmc_byok_tvm_ffi.so"); + + const auto is_matching_runtime = [&](const fs::path& candidate) { + return std::all_of(required.begin(), required.end(), [&](const std::string& library) { + std::error_code error; + const fs::path path = candidate / library; + return fs::is_regular_file(path, error) && read_build_cohort(path) == context.cohort_id; + }); + }; + + std::vector searched; + if (!current_candidates.empty()) { + const auto& current = current_candidates.front(); + searched.push_back(current); + if (is_matching_runtime(current)) { + return current.string(); + } + } + for (const auto& candidate : installed_candidates) { + searched.push_back(candidate); + if (is_matching_runtime(candidate)) + return candidate.string(); + } + + std::vector matching_wheels; + for (const auto& candidate : wheel_candidates) { + searched.push_back(candidate); + if (is_matching_runtime(candidate)) + matching_wheels.push_back(candidate); + } + if (matching_wheels.size() == 1) + return matching_wheels.front().string(); + if (matching_wheels.size() > 1) { + std::ostringstream message; + message << "Multiple installed TRTMC runtimes match the running CLI:"; + for (const auto& candidate : matching_wheels) + message << " " << candidate.string(); + message << ". Pass --runtime-root DIR to select one."; + throw std::runtime_error(message.str()); + } + + for (const auto& candidate : configured_candidates) { + searched.push_back(candidate); + if (is_matching_runtime(candidate)) + return candidate.string(); + } + + std::ostringstream message; + message << "Unable to discover a complete TRTMC runtime for family '" << bundle.family + << "' and backend '" << bundle.backend << "'. Expected in one directory: "; + for (std::size_t index = 0; index < required.size(); ++index) { + if (index != 0) + message << ", "; + message << required[index]; + } + message << ". Searched:"; + for (const auto& candidate : searched) + message << " " << candidate.string(); + message << ". Pass --runtime-root DIR or add a directory to TRTMC_RUNTIME_PATH."; + throw std::runtime_error(message.str()); +} + Command parse_args(int argc, char** argv) { if (argc < 2) throw std::invalid_argument("a command is required"); @@ -728,8 +1019,6 @@ Command parse_args(int argc, char** argv) { throw std::invalid_argument(option + " may be specified only once"); command.options.emplace(option, take_value(argc, argv, index, option)); } - if (command.runtime_root.empty()) - throw std::invalid_argument("--runtime-root is required for " + name); const int byok_option_count = static_cast(command.options.count("--byok-library") + command.options.count("--byok-function") + command.options.count("--byok-name")); @@ -1280,7 +1569,7 @@ void print_usage(std::ostream& output) { output << "Usage:\n" " trtmc version\n" " trtmc inspect BUNDLE\n" - " trtmc COMMAND BUNDLE --runtime-root DIR [OPTIONS]\n\n" + " trtmc COMMAND BUNDLE [--runtime-root DIR] [OPTIONS]\n\n" "Execution commands:\n" " run, encode, embed, rerank, classify, detect, extract-features,\n" " predict-structure, disparity, geometry,\n" @@ -1308,12 +1597,14 @@ void print_usage(std::ostream& output) { " [--kv-cache-size BYTES|GB|GiB]\n\n" "TensorRT-RTX runtime options:\n" " [--runtime-cache PATH] [--cuda-graphs]\n\n" - "Execution never searches for runtimes; --runtime-root is always required.\n"; + "Runtime discovery: current directory, the active trtmc installation, then\n" + "TRTMC_RUNTIME_PATH. LD_LIBRARY_PATH can select the active cohort before startup.\n" + "--runtime-root overrides discovery.\n"; } int run(int argc, char** argv, std::ostream& output, std::ostream& error) { try { - const Command command = parse_args(argc, argv); + Command command = parse_args(argc, argv); if (command.kind == CommandKind::kHelp) { print_usage(output); return EXIT_SUCCESS; @@ -1342,8 +1633,15 @@ int run(int argc, char** argv, std::ostream& output, std::ostream& error) { throw std::invalid_argument( "--byok-library, --byok-function, and --byok-name must be used together"); } - load_byok_extension(command); } + const bool discover_runtime = command.runtime_root.empty(); + const BundleInfo bundle = InspectBundle(command.bundle); + command.runtime_root = resolve_runtime_root(bundle, command.runtime_root, has_byok_library, + runtime_root_search_context()); + if (discover_runtime) + error << "Using TRTMC runtime: " << command.runtime_root << '\n'; + if (has_byok_library) + load_byok_extension(command); std::unique_ptr task = load_task(command.bundle, command.runtime_root, command.kv_cache_size_bytes, command.runtime_cache_path, command.cuda_graphs); diff --git a/apps/cli/cli.h b/apps/cli/cli.h index 9fe114557f..d74958bef3 100644 --- a/apps/cli/cli.h +++ b/apps/cli/cli.h @@ -9,6 +9,7 @@ #include "trtmc/task.h" #include +#include #include #include #include @@ -61,7 +62,17 @@ struct Command { bool cuda_graphs{false}; }; +struct RuntimeRootSearchContext { + std::filesystem::path current_directory; + std::filesystem::path runtime_library; + std::filesystem::path executable; + std::string cohort_id; + std::string runtime_path; +}; + Command parse_args(int argc, char** argv); +std::string resolve_runtime_root(const BundleInfo& bundle, const std::string& explicit_root, + bool require_byok, const RuntimeRootSearchContext& context); int dispatch(const Command& command, ITask& task, std::ostream& output); void print_usage(std::ostream& output); int run(int argc, char** argv, std::ostream& output, std::ostream& error); diff --git a/apps/cli/tests/test_cli.cpp b/apps/cli/tests/test_cli.cpp index 2023b7ce0d..c26eeb3bbf 100644 --- a/apps/cli/tests/test_cli.cpp +++ b/apps/cli/tests/test_cli.cpp @@ -7,6 +7,8 @@ #include "cli/io.h" #include +#include +#include #include #include #include @@ -21,6 +23,8 @@ namespace { int failures = 0; +constexpr char kRuntimeCohort[] = "0123456789abcdef0123456789abcdef"; +constexpr char kForeignCohort[] = "fedcba9876543210fedcba9876543210"; void check(bool condition, const char* name) { if (!condition) { @@ -46,6 +50,137 @@ bool parse_throws(std::vector arguments) { } } +void touch(const std::filesystem::path& path, const char* contents = "test") { + std::filesystem::create_directories(path.parent_path()); + std::ofstream file(path); + file << contents; +} + +void make_runtime_library(const std::filesystem::path& path, const char* cohort = kRuntimeCohort) { + std::filesystem::create_directories(path.parent_path()); + const std::string section_names{"\0.shstrtab\0.dynstr\0", 19}; + std::string dynamic_strings(1, '\0'); + dynamic_strings += "trtmc_build_cohort_"; + dynamic_strings += cohort; + dynamic_strings.push_back('\0'); + + const std::size_t section_names_offset = sizeof(Elf64_Ehdr); + const std::size_t dynamic_strings_offset = section_names_offset + section_names.size(); + const std::size_t section_headers_offset = + (dynamic_strings_offset + dynamic_strings.size() + 7U) & ~std::size_t{7U}; + std::vector image(section_headers_offset + 3 * sizeof(Elf64_Shdr), '\0'); + + Elf64_Ehdr header{}; + std::memcpy(header.e_ident, ELFMAG, SELFMAG); + header.e_ident[EI_CLASS] = ELFCLASS64; + header.e_ident[EI_DATA] = ELFDATA2LSB; + header.e_ident[EI_VERSION] = EV_CURRENT; + header.e_type = ET_DYN; + header.e_machine = EM_X86_64; + header.e_version = EV_CURRENT; + header.e_ehsize = sizeof(Elf64_Ehdr); + header.e_shoff = section_headers_offset; + header.e_shentsize = sizeof(Elf64_Shdr); + header.e_shnum = 3; + header.e_shstrndx = 1; + std::memcpy(image.data(), &header, sizeof(header)); + std::memcpy(image.data() + section_names_offset, section_names.data(), section_names.size()); + std::memcpy(image.data() + dynamic_strings_offset, dynamic_strings.data(), + dynamic_strings.size()); + + Elf64_Shdr section_names_header{}; + section_names_header.sh_name = 1; + section_names_header.sh_type = SHT_STRTAB; + section_names_header.sh_offset = section_names_offset; + section_names_header.sh_size = section_names.size(); + std::memcpy(image.data() + section_headers_offset + sizeof(Elf64_Shdr), §ion_names_header, + sizeof(section_names_header)); + Elf64_Shdr dynamic_strings_header{}; + dynamic_strings_header.sh_name = 11; + dynamic_strings_header.sh_type = SHT_STRTAB; + dynamic_strings_header.sh_offset = dynamic_strings_offset; + dynamic_strings_header.sh_size = dynamic_strings.size(); + std::memcpy(image.data() + section_headers_offset + 2 * sizeof(Elf64_Shdr), + &dynamic_strings_header, sizeof(dynamic_strings_header)); + + std::ofstream file(path, std::ios::binary); + file.write(image.data(), static_cast(image.size())); +} + +void make_runtime_root(const std::filesystem::path& root, bool with_byok = false, + const char* cohort = kRuntimeCohort) { + make_runtime_library(root / "libtrtmc_core.so", cohort); + make_runtime_library(root / "libtrtmc_runtime.so", cohort); + make_runtime_library(root / "libtrtmc_backend_trt.so", cohort); + make_runtime_library(root / "libtrtmc_model_gpt2.so", cohort); + if (with_byok) + make_runtime_library(root / "libtrtmc_byok_tvm_ffi.so", cohort); +} + +void write_fake_bundle(const std::filesystem::path& path) { + static constexpr unsigned char magic[8] = {'B', 'U', 'N', 'D', 'L', 'E', '\x01', '\0'}; + const std::string header = + "{\"format\":1,\"family\":\"fake\",\"task\":\"time_series_forecast\"," + "\"backend\":\"fake\",\"sections\":{\"runtime.json\":{\"offset\":0,\"length\":2}," + "\"engine.plan\":{\"offset\":2,\"length\":4}}}"; + std::ofstream output(path, std::ios::binary); + output.write(reinterpret_cast(magic), 8); + const std::uint64_t length = header.size(); + for (int shift = 0; shift < 64; shift += 8) + output.put(static_cast((length >> shift) & 0xffU)); + output.write(header.data(), static_cast(header.size())); + output.write("{}PLAN", 6); +} + +void check_cli_runtime_discovery(const std::filesystem::path& runtime_root, + const std::filesystem::path& core_library, + const std::filesystem::path& runtime_library) { + std::filesystem::copy_file(core_library, runtime_root / "libtrtmc_core.so", + std::filesystem::copy_options::overwrite_existing); + std::filesystem::copy_file(runtime_library, runtime_root / "libtrtmc_runtime.so", + std::filesystem::copy_options::overwrite_existing); + const auto bundle = runtime_root / "cli-fake.bundle"; + const auto input = runtime_root / "cli-fake-input.f32"; + write_fake_bundle(bundle); + { + const float values[] = {1.0F, 2.0F, 3.0F}; + std::ofstream output(input, std::ios::binary); + output.write(reinterpret_cast(values), sizeof(values)); + } + + const auto previous_directory = std::filesystem::current_path(); + std::filesystem::current_path(runtime_root); + std::vector arguments{"trtmc", "forecast", bundle.string(), "--input", + input.string()}; + std::vector argv; + for (auto& argument : arguments) + argv.push_back(argument.data()); + std::ostringstream output; + std::ostringstream error; + const int result = trtmc::cli::run(static_cast(argv.size()), argv.data(), output, error); + std::filesystem::current_path(previous_directory); + check(result == 0 && error.str() == "Using TRTMC runtime: " + runtime_root.string() + "\n", + "CLI executes with a runtime discovered from the current directory"); + check(output.str().find("\"shape\":[1,3]") != std::string::npos, + "automatically discovered runtime dispatches the bundle task"); + + std::filesystem::remove(bundle); + std::filesystem::remove(input); + std::filesystem::remove(runtime_root / "libtrtmc_core.so"); + std::filesystem::remove(runtime_root / "libtrtmc_runtime.so"); +} + +bool resolve_throws(const trtmc::BundleInfo& bundle, + const trtmc::cli::RuntimeRootSearchContext& context, std::string& message) { + try { + (void)trtmc::cli::resolve_runtime_root(bundle, {}, false, context); + return false; + } catch (const std::runtime_error& error) { + message = error.what(); + return true; + } +} + class FakeText final : public trtmc::ITextGeneration, public trtmc::IEmbedding, public trtmc::ILoraAdapterManager { @@ -269,7 +404,7 @@ bool dispatch_throws(const trtmc::cli::Command& command, trtmc::ITask& task) { } // namespace -int main() { +int main(int argc, char** argv) { const std::vector execution_commands{ "run", "encode", @@ -303,11 +438,120 @@ int main() { check(command.runtime_root == "lib", "runtime root is retained"); } - check(parse_throws({"trtmc", "run", "model.bundle"}), - "execution command requires runtime root"); + check(parse({"trtmc", "run", "model.bundle"}).runtime_root.empty(), + "execution command permits automatic runtime discovery"); check(parse_throws( {"trtmc", "run", "model.bundle", "--runtime-root", "a", "--runtime-root", "b"}), "duplicate runtime root rejected"); + + const std::filesystem::path runtime_test_root = + argc == 4 ? std::filesystem::path(argv[1]).parent_path() / "cli-runtime-root-test" + : std::filesystem::temp_directory_path() / "trtmc-cli-runtime-root-test"; + std::filesystem::remove_all(runtime_test_root); + const trtmc::BundleInfo runtime_bundle{1, "gpt2", "text_generation", "trt", {}}; + const auto current_root = runtime_test_root / "current"; + const auto loaded_runtime_root = runtime_test_root / "loaded-runtime"; + const auto optional_root = runtime_test_root / "optional"; + make_runtime_root(current_root); + make_runtime_root(loaded_runtime_root); + make_runtime_root(optional_root); + + trtmc::cli::RuntimeRootSearchContext runtime_context; + runtime_context.current_directory = current_root; + runtime_context.runtime_library = loaded_runtime_root / "libtrtmc_runtime.so"; + runtime_context.cohort_id = kRuntimeCohort; + runtime_context.runtime_path = optional_root.string(); + check(trtmc::cli::resolve_runtime_root(runtime_bundle, {}, false, runtime_context) == + current_root.string(), + "current directory wins automatic runtime discovery"); + + std::filesystem::remove(current_root / "libtrtmc_model_gpt2.so"); + check(trtmc::cli::resolve_runtime_root(runtime_bundle, {}, false, runtime_context) == + loaded_runtime_root.string(), + "incomplete current directory is skipped as a whole runtime cohort"); + + const auto malformed_root = runtime_test_root / "malformed"; + touch(malformed_root / "libtrtmc_core.so", "not an ELF file"); + std::filesystem::resize_file(malformed_root / "libtrtmc_core.so", 64ULL * 1024ULL * 1024ULL); + runtime_context.current_directory = malformed_root; + check(trtmc::cli::resolve_runtime_root(runtime_bundle, {}, false, runtime_context) == + loaded_runtime_root.string(), + "large malformed candidates are rejected without an unbounded scan"); + + const auto wheel_prefix = runtime_test_root / "wheel"; + const auto wheel_bin = wheel_prefix / "bin"; + const auto wheel_runtime = + wheel_prefix / "lib" / "python3.12" / "dist-packages" / "tensorrt_model_connect" / "bin"; + touch(wheel_bin / "trtmc"); + make_runtime_library(wheel_bin / "libtrtmc_core.so"); + make_runtime_library(wheel_bin / "libtrtmc_runtime.so"); + make_runtime_root(wheel_runtime); + std::filesystem::create_directory_symlink("lib", wheel_prefix / "lib64"); + runtime_context = {}; + runtime_context.executable = wheel_bin / "trtmc"; + runtime_context.runtime_library = wheel_bin / "libtrtmc_runtime.so"; + runtime_context.cohort_id = kRuntimeCohort; + check(trtmc::cli::resolve_runtime_root(runtime_bundle, {}, false, runtime_context) == + wheel_runtime.string(), + "wheel runtime is discovered once when lib64 aliases lib"); + + const auto second_wheel_runtime = + wheel_prefix / "lib" / "python3.13" / "site-packages" / "tensorrt_model_connect" / "bin"; + make_runtime_root(second_wheel_runtime); + std::string discovery_error; + check(resolve_throws(runtime_bundle, runtime_context, discovery_error) && + discovery_error.find("Multiple installed TRTMC runtimes") != std::string::npos, + "ambiguous matching wheel runtimes require an explicit selection"); + std::filesystem::remove_all(second_wheel_runtime); + + const auto first_optional = runtime_test_root / "first-optional"; + const auto second_optional = runtime_test_root / "second-optional"; + const auto loaded_libraries = runtime_test_root / "loaded-libraries"; + touch(first_optional / "libtrtmc_backend_trt.so"); + make_runtime_root(second_optional); + make_runtime_library(loaded_libraries / "libtrtmc_runtime.so"); + runtime_context = {}; + runtime_context.runtime_library = loaded_libraries / "libtrtmc_runtime.so"; + runtime_context.cohort_id = kRuntimeCohort; + runtime_context.runtime_path = first_optional.string() + ":" + second_optional.string(); + check(trtmc::cli::resolve_runtime_root(runtime_bundle, {}, false, runtime_context) == + second_optional.string(), + "dedicated runtime path preserves directory order"); + + const auto byok_root = runtime_test_root / "byok"; + make_runtime_root(byok_root, true); + runtime_context.runtime_path = second_optional.string() + ":" + byok_root.string(); + check(trtmc::cli::resolve_runtime_root(runtime_bundle, {}, true, runtime_context) == + byok_root.string(), + "BYOK discovery requires the extension in the selected runtime cohort"); + + const auto explicit_root = runtime_test_root / "explicit-missing"; + check(trtmc::cli::resolve_runtime_root(runtime_bundle, explicit_root.string(), false, + runtime_context) == explicit_root.string(), + "explicit runtime root bypasses automatic fallback"); + + const auto foreign_root = runtime_test_root / "foreign-build"; + make_runtime_root(foreign_root); + make_runtime_library(foreign_root / "libtrtmc_model_gpt2.so", kForeignCohort); + runtime_context.runtime_path = foreign_root.string(); + check(resolve_throws(runtime_bundle, runtime_context, discovery_error), + "automatic discovery rejects a foreign family with matching core/runtime files"); + + runtime_context = {}; + runtime_context.current_directory = first_optional; + runtime_context.runtime_library = loaded_libraries / "libtrtmc_runtime.so"; + runtime_context.cohort_id = kRuntimeCohort; + runtime_context.runtime_path = (runtime_test_root / "family-only").string(); + touch(runtime_test_root / "family-only" / "libtrtmc_model_gpt2.so"); + check(resolve_throws(runtime_bundle, runtime_context, discovery_error), + "runtime discovery never combines partial roots"); + check(discovery_error.find("libtrtmc_backend_trt.so") != std::string::npos && + discovery_error.find("libtrtmc_model_gpt2.so") != std::string::npos && + discovery_error.find("--runtime-root") != std::string::npos, + "runtime discovery failure identifies required files and explicit override"); + std::filesystem::remove_all(runtime_test_root); + if (argc == 4) + check_cli_runtime_discovery(argv[1], argv[2], argv[3]); const auto dynamic_kv = parse({"trtmc", "run", "model.bundle", "--runtime-root", "lib", "--kv-cache-size", "1GiB"}); check(dynamic_kv.kv_cache_size_bytes == 1024ULL * 1024ULL * 1024ULL, diff --git a/tools/ci/package.py b/tools/ci/package.py index d1b097d187..dc44f3c43e 100644 --- a/tools/ci/package.py +++ b/tools/ci/package.py @@ -378,7 +378,7 @@ def validate(self, wheel: Path) -> None: "from pathlib import Path; " "from tensorrt_model_connect.bundle_writer import BundleWriter; " "writer = BundleWriter(Path(__import__('sys').argv[1])); " - "writer.set_header(family='inspect', task='text_generation', backend='trt'); " + "writer.set_header(family='gpt2', task='text_generation', backend='trt'); " "writer.finish()", bundle, ], @@ -395,8 +395,18 @@ def validate(self, wheel: Path) -> None: env=environment, ) metadata = json.loads(inspected.stdout) - if metadata.get("family") != "inspect" or metadata.get("backend") != "trt": + if metadata.get("family") != "gpt2" or metadata.get("backend") != "trt": raise CiError("installed trtmc CLI failed bundle inspection") + executed = subprocess.run( + [executable, "run", bundle], + capture_output=True, + text=True, + cwd=Path("/tmp"), + env=environment, + ) + selected = f"Using TRTMC runtime: {bin_dir}\n" + if executed.returncode == 0 or selected not in executed.stderr: + raise CiError("installed trtmc CLI failed automatic wheel runtime discovery") print(f"installed wheel={wheel} trtmc={executable} families={len(packaged)}") diff --git a/website/docs/architecture/ai-native-horizontal-scaling.md b/website/docs/architecture/ai-native-horizontal-scaling.md index f975ff1c79..1d13406df2 100644 --- a/website/docs/architecture/ai-native-horizontal-scaling.md +++ b/website/docs/architecture/ai-native-horizontal-scaling.md @@ -517,6 +517,16 @@ Runtime dispatch occurs once: Core, family, and backend DSOs are produced by one product build. There is no ABI negotiation, version translation, old-symbol alias, or compatibility shim. +The human-facing `trtmc` CLI may discover a complete runtime cohort before this +control transfer. It prefers the current directory, then the runtime belonging +to the running CLI installation, followed by explicitly configured runtime +library paths. The public C++ load API still receives one explicit root, and +the loader never combines or falls back across roots. Every native artifact +carries the build-cohort identity generated when CMake configures the build. +Automatic +candidates must contain that same identity in core, runtime, backend, family, +and optional BYOK DSOs, preventing another build cohort from being selected +implicitly. ### Task API diff --git a/website/docs/getting-started/quick-start.md b/website/docs/getting-started/quick-start.md index 26f29757ae..10583595a0 100644 --- a/website/docs/getting-started/quick-start.md +++ b/website/docs/getting-started/quick-start.md @@ -41,29 +41,31 @@ selecting another task supported by the same family. The build then imports only the selected `families.gpt2.model` and calls `build(request, writer)` once. A prepared local snapshot can be passed in place of the model ID. -For a wheel install, resolve its native runtime directory directly from the -installed package: +Run the bundle directly: ```bash -TRTMC_RUNTIME_ROOT="$(python -c 'import pathlib, tensorrt_model_connect as m; print(pathlib.Path(m.__file__).parent / "bin")')" trtmc run gpt2.bundle \ - --runtime-root "$TRTMC_RUNTIME_ROOT" \ --prompt "Hello" \ --max-new-tokens 32 ``` -For a native CMake install, point the loader at the directory containing the matching -`libtrtmc_core.so`, `libtrtmc_runtime.so`, `libtrtmc_backend_trt.so`, and -`libtrtmc_model_gpt2.so`. The loader reads the bundle header, loads exactly -those DSOs, and returns the abstract task interface declared by the bundle. +The CLI reads the bundle family and backend, then selects the first complete, +single-directory runtime in this order: -```bash -trtmc run gpt2.bundle \ - --runtime-root /opt/trtmc/lib \ - --prompt "Hello" \ - --max-new-tokens 32 -``` +1. the current directory, when all required libraries match the active build + cohort; +2. the runtime belonging to the active `trtmc` selected through `PATH`, + including native CMake and wheel install layouts; +3. colon-separated directories in `TRTMC_RUNTIME_PATH`. + +A complete GPT-2 TensorRT runtime contains matching `libtrtmc_core.so`, +`libtrtmc_runtime.so`, `libtrtmc_backend_trt.so`, and +`libtrtmc_model_gpt2.so` files. Candidates are never combined across +directories, and the CLI prints the automatically selected directory. If more +than one installed wheel runtime matches, select one with `--runtime-root DIR`. +An explicit root bypasses discovery. -The shell variable above is only a convenient explicit argument. The CLI never -searches environment variables, the current directory, or an installed -fallback runtime. +Every native artifact carries a build-cohort identity, and automatic discovery +accepts a directory only when the identity matches the core and runtime already +loaded by `trtmc`. The platform loader evaluates `LD_LIBRARY_PATH` before the +CLI starts, so it can determine that active cohort before the search above. From 29df18d0b6e30118a4365ca14862e17229418a49 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Mon, 7 Sep 2026 09:56:31 +0000 Subject: [PATCH 02/13] style(cli): apply repository clang format Format the runtime cohort predicate with clang-format 22.1.8, matching the version enforced by Community CPU source quality. Signed-off-by: chaofengw --- apps/cli/cli.cpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/apps/cli/cli.cpp b/apps/cli/cli.cpp index a8fabbba03..32ad37c0c2 100644 --- a/apps/cli/cli.cpp +++ b/apps/cli/cli.cpp @@ -266,12 +266,13 @@ std::string read_build_cohort(const fs::path& library) { const std::size_t id_begin = position + sizeof(prefix) - 1; const std::size_t marker_end = id_begin + id_size; if (marker_end < strings.size() && strings[marker_end] == '\0' && - std::all_of(strings.begin() + static_cast(id_begin), - strings.begin() + static_cast(marker_end), - [](const unsigned char character) { - return (character >= '0' && character <= '9') || - (character >= 'a' && character <= 'f'); - })) { + std::all_of( + strings.begin() + static_cast(id_begin), + strings.begin() + static_cast(marker_end), + [](const unsigned char character) { + return (character >= '0' && character <= '9') || + (character >= 'a' && character <= 'f'); + })) { return strings.substr(id_begin, id_size); } position = strings.find(prefix, position + 1); From a1706b73ac0b14df1c8cbcb23cee11952b53055d Mon Sep 17 00:00:00 2001 From: chaofengw Date: Mon, 7 Sep 2026 11:26:41 +0000 Subject: [PATCH 03/13] refactor(runtime): restore loader ownership Move DSO naming, safe-ID, and build-cohort validation behind a model-agnostic runtime-root contract. Keep the CLI responsible only for candidate enumeration and search order. Signed-off-by: chaofengw --- CMakeLists.txt | 14 ++ apps/cli/cli.cpp | 179 ++------------- apps/cli/cli.h | 10 +- apps/cli/tests/test_cli.cpp | 201 ++++++----------- .../include/trtmc/runtime/runtime_root.h | 26 +++ core/runtime/loader/family_loader.cpp | 206 +++++++++++++++++- core/runtime/tests/test_runtime_root.cpp | 114 ++++++++++ tools/tests/test_architecture.py | 2 + .../ai-native-horizontal-scaling.md | 26 ++- website/docs/getting-started/quick-start.md | 7 +- 10 files changed, 478 insertions(+), 307 deletions(-) create mode 100644 core/runtime/include/trtmc/runtime/runtime_root.h create mode 100644 core/runtime/tests/test_runtime_root.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 17098f4fc9..47ff1ab0b9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -399,6 +399,19 @@ if(TRTMC_BUILD_TESTS) ) add_dependencies(test_cli trtmc_test_backend_fake trtmc_test_family_fake) + add_executable(test_runtime_root core/runtime/tests/test_runtime_root.cpp) + target_include_directories(test_runtime_root PRIVATE ${PROJECT_SOURCE_DIR}/core/runtime/include) + target_link_libraries(test_runtime_root PRIVATE trtmc_runtime) + target_compile_options(test_runtime_root PRIVATE -Wall -Wextra -Wpedantic) + add_dependencies(test_runtime_root trtmc_test_backend_fake trtmc_test_family_fake) + add_test(NAME runtime_root COMMAND test_runtime_root + "${CMAKE_BINARY_DIR}/tests/runtime-root-validator" + $ + $ + $ + $ + ) + add_executable(test_family_loader core/runtime/tests/test_family_loader.cpp) target_include_directories(test_family_loader PRIVATE ${PROJECT_SOURCE_DIR}/core/runtime/include @@ -564,6 +577,7 @@ install(FILES core/runtime/include/trtmc/runtime/device_tensor.h core/runtime/include/trtmc/runtime/family_factory.h core/runtime/include/trtmc/runtime/family_loader.h + core/runtime/include/trtmc/runtime/runtime_root.h core/runtime/include/trtmc/runtime/tensor.h core/runtime/include/trtmc/runtime/trt_backend.h core/runtime/include/trtmc/runtime/trt_module.h diff --git a/apps/cli/cli.cpp b/apps/cli/cli.cpp index 32ad37c0c2..18e0b027da 100644 --- a/apps/cli/cli.cpp +++ b/apps/cli/cli.cpp @@ -7,6 +7,7 @@ #include "cli/io.h" #include "trtmc/runtime/family_loader.h" +#include "trtmc/runtime/runtime_root.h" #include #include @@ -15,7 +16,6 @@ #include #include #include -#include #include #include #include @@ -201,114 +201,11 @@ void append_python_package_runtime_roots(std::vector& candidates, append_candidate(candidates, seen, root); } -std::string read_build_cohort(const fs::path& library) { - static constexpr char prefix[] = "trtmc_build_cohort_"; - static constexpr std::size_t id_size = 32; - static constexpr std::uint64_t max_string_table_size = 16ULL * 1024ULL * 1024ULL; - std::ifstream input(library, std::ios::binary); - if (!input) - return {}; - - std::error_code error; - const std::uintmax_t file_size = fs::file_size(library, error); - if (error || file_size < sizeof(Elf64_Ehdr)) - return {}; - const auto read_at = [&](std::uint64_t offset, void* destination, std::size_t size) { - if (offset > file_size || size > file_size - offset) - return false; - input.clear(); - input.seekg(static_cast(offset)); - input.read(static_cast(destination), static_cast(size)); - return input.good(); - }; - - Elf64_Ehdr header{}; - if (!read_at(0, &header, sizeof(header)) || header.e_ident[EI_MAG0] != ELFMAG0 || - header.e_ident[EI_MAG1] != ELFMAG1 || header.e_ident[EI_MAG2] != ELFMAG2 || - header.e_ident[EI_MAG3] != ELFMAG3 || header.e_ident[EI_CLASS] != ELFCLASS64 || - header.e_ident[EI_DATA] != ELFDATA2LSB || header.e_shentsize != sizeof(Elf64_Shdr) || - header.e_shnum == 0 || header.e_shstrndx >= header.e_shnum || header.e_shoff > file_size || - header.e_shnum > (file_size - header.e_shoff) / sizeof(Elf64_Shdr)) { - return {}; - } - - const auto read_section_header = [&](std::size_t index, Elf64_Shdr& section) { - return read_at(header.e_shoff + index * sizeof(Elf64_Shdr), §ion, sizeof(section)); - }; - Elf64_Shdr names_header{}; - if (!read_section_header(header.e_shstrndx, names_header) || - names_header.sh_size > max_string_table_size || names_header.sh_offset > file_size || - names_header.sh_size > file_size - names_header.sh_offset) { - return {}; - } - std::string names(static_cast(names_header.sh_size), '\0'); - if (!read_at(names_header.sh_offset, names.data(), names.size())) - return {}; - - for (std::size_t index = 0; index < header.e_shnum; ++index) { - Elf64_Shdr section{}; - if (!read_section_header(index, section) || section.sh_name >= names.size()) - return {}; - const auto name_end = names.find('\0', section.sh_name); - if (name_end == std::string::npos || - names.compare(section.sh_name, name_end - section.sh_name, ".dynstr") != 0) { - continue; - } - if (section.sh_size > max_string_table_size || section.sh_offset > file_size || - section.sh_size > file_size - section.sh_offset) { - return {}; - } - std::string strings(static_cast(section.sh_size), '\0'); - if (!read_at(section.sh_offset, strings.data(), strings.size())) - return {}; - std::size_t position = strings.find(prefix); - while (position != std::string::npos) { - const std::size_t id_begin = position + sizeof(prefix) - 1; - const std::size_t marker_end = id_begin + id_size; - if (marker_end < strings.size() && strings[marker_end] == '\0' && - std::all_of( - strings.begin() + static_cast(id_begin), - strings.begin() + static_cast(marker_end), - [](const unsigned char character) { - return (character >= '0' && character <= '9') || - (character >= 'a' && character <= 'f'); - })) { - return strings.substr(id_begin, id_size); - } - position = strings.find(prefix, position + 1); - } - return {}; - } - return {}; -} - RuntimeRootSearchContext runtime_root_search_context() { RuntimeRootSearchContext context; std::error_code error; context.current_directory = fs::current_path(error); - - fs::path core_library; - Dl_info core_info{}; - using InspectBundleFn = BundleInfo (*)(const std::string&); - const auto inspect_bundle_function = static_cast(&InspectBundle); - if (dladdr(reinterpret_cast(inspect_bundle_function), &core_info) != 0 && - core_info.dli_fname != nullptr) { - core_library = core_info.dli_fname; - } - - Dl_info runtime_info{}; - using LoadTaskFn = std::unique_ptr (*)(const std::string&, const std::string&, - std::uint64_t, const std::string&, bool); - const auto load_task_function = static_cast(&load_task); - if (dladdr(reinterpret_cast(load_task_function), &runtime_info) != 0 && - runtime_info.dli_fname != nullptr) { - context.runtime_library = runtime_info.dli_fname; - } - - const std::string core_cohort = read_build_cohort(core_library); - const std::string runtime_cohort = read_build_cohort(context.runtime_library); - if (!core_cohort.empty() && core_cohort == runtime_cohort) - context.cohort_id = core_cohort; + context.loaded_runtime_root = loaded_runtime_root(); std::vector executable(4096, '\0'); const ssize_t length = readlink("/proc/self/exe", executable.data(), executable.size() - 1); @@ -843,25 +740,12 @@ int dispatch_run(const Command& command, ITask& task, std::ostream& output) { } // namespace std::string resolve_runtime_root(const BundleInfo& bundle, const std::string& explicit_root, - bool require_byok, const RuntimeRootSearchContext& context) { + bool require_byok, const RuntimeRootSearchContext& context, + const RuntimeRootMatcher& matches) { if (!explicit_root.empty()) return explicit_root; - - const auto is_safe_id = [](const std::string& value) { - if (value.empty() || value.front() < 'a' || value.front() > 'z') - return false; - return std::all_of(value.begin(), value.end(), [](const unsigned char character) { - return (character >= 'a' && character <= 'z') || - (character >= '0' && character <= '9') || character == '_'; - }); - }; - if (!is_safe_id(bundle.family) || !is_safe_id(bundle.backend)) - throw std::runtime_error("Cannot discover a runtime for unsafe bundle family/backend IDs"); - if (context.runtime_library.empty() || context.cohort_id.empty()) { - throw std::runtime_error( - "Unable to identify one build cohort for the loaded TRTMC core/runtime libraries; " - "pass --runtime-root DIR"); - } + if (!matches) + throw std::logic_error("runtime-root discovery requires a candidate matcher"); std::vector current_candidates; std::vector installed_candidates; @@ -869,7 +753,7 @@ std::string resolve_runtime_root(const BundleInfo& bundle, const std::string& ex std::vector configured_candidates; std::set seen; append_candidate(current_candidates, seen, context.current_directory); - append_candidate(installed_candidates, seen, context.runtime_library.parent_path()); + append_candidate(installed_candidates, seen, context.loaded_runtime_root); if (!context.executable.empty()) { const fs::path executable_directory = context.executable.parent_path(); @@ -884,41 +768,24 @@ std::string resolve_runtime_root(const BundleInfo& bundle, const std::string& ex append_path_list(configured_candidates, seen, context.runtime_path); - std::vector required{ - "libtrtmc_core.so", - "libtrtmc_runtime.so", - "libtrtmc_backend_" + bundle.backend + ".so", - "libtrtmc_model_" + bundle.family + ".so", - }; - if (require_byok) - required.emplace_back("libtrtmc_byok_tvm_ffi.so"); - - const auto is_matching_runtime = [&](const fs::path& candidate) { - return std::all_of(required.begin(), required.end(), [&](const std::string& library) { - std::error_code error; - const fs::path path = candidate / library; - return fs::is_regular_file(path, error) && read_build_cohort(path) == context.cohort_id; - }); - }; - std::vector searched; if (!current_candidates.empty()) { const auto& current = current_candidates.front(); searched.push_back(current); - if (is_matching_runtime(current)) { + if (matches(bundle, current, require_byok)) { return current.string(); } } for (const auto& candidate : installed_candidates) { searched.push_back(candidate); - if (is_matching_runtime(candidate)) + if (matches(bundle, candidate, require_byok)) return candidate.string(); } std::vector matching_wheels; for (const auto& candidate : wheel_candidates) { searched.push_back(candidate); - if (is_matching_runtime(candidate)) + if (matches(bundle, candidate, require_byok)) matching_wheels.push_back(candidate); } if (matching_wheels.size() == 1) @@ -934,19 +801,13 @@ std::string resolve_runtime_root(const BundleInfo& bundle, const std::string& ex for (const auto& candidate : configured_candidates) { searched.push_back(candidate); - if (is_matching_runtime(candidate)) + if (matches(bundle, candidate, require_byok)) return candidate.string(); } std::ostringstream message; - message << "Unable to discover a complete TRTMC runtime for family '" << bundle.family - << "' and backend '" << bundle.backend << "'. Expected in one directory: "; - for (std::size_t index = 0; index < required.size(); ++index) { - if (index != 0) - message << ", "; - message << required[index]; - } - message << ". Searched:"; + message << "Unable to discover a complete TRTMC runtime for bundle '" << bundle.family << "/" + << bundle.backend << "'. Searched:"; for (const auto& candidate : searched) message << " " << candidate.string(); message << ". Pass --runtime-root DIR or add a directory to TRTMC_RUNTIME_PATH."; @@ -1636,11 +1497,17 @@ int run(int argc, char** argv, std::ostream& output, std::ostream& error) { } } const bool discover_runtime = command.runtime_root.empty(); - const BundleInfo bundle = InspectBundle(command.bundle); - command.runtime_root = resolve_runtime_root(bundle, command.runtime_root, has_byok_library, - runtime_root_search_context()); - if (discover_runtime) + if (discover_runtime) { + const BundleInfo bundle = InspectBundle(command.bundle); + command.runtime_root = + resolve_runtime_root(bundle, {}, has_byok_library, runtime_root_search_context(), + [](const BundleInfo& candidate_bundle, + const fs::path& candidate, bool require_byok) { + return runtime_root_matches_loaded_build( + candidate_bundle, candidate.string(), require_byok); + }); error << "Using TRTMC runtime: " << command.runtime_root << '\n'; + } if (has_byok_library) load_byok_extension(command); std::unique_ptr task = diff --git a/apps/cli/cli.h b/apps/cli/cli.h index d74958bef3..46e882e416 100644 --- a/apps/cli/cli.h +++ b/apps/cli/cli.h @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -64,15 +65,18 @@ struct Command { struct RuntimeRootSearchContext { std::filesystem::path current_directory; - std::filesystem::path runtime_library; + std::filesystem::path loaded_runtime_root; std::filesystem::path executable; - std::string cohort_id; std::string runtime_path; }; +using RuntimeRootMatcher = + std::function; + Command parse_args(int argc, char** argv); std::string resolve_runtime_root(const BundleInfo& bundle, const std::string& explicit_root, - bool require_byok, const RuntimeRootSearchContext& context); + bool require_byok, const RuntimeRootSearchContext& context, + const RuntimeRootMatcher& matches); int dispatch(const Command& command, ITask& task, std::ostream& output); void print_usage(std::ostream& output); int run(int argc, char** argv, std::ostream& output, std::ostream& error); diff --git a/apps/cli/tests/test_cli.cpp b/apps/cli/tests/test_cli.cpp index c26eeb3bbf..eafa6409bb 100644 --- a/apps/cli/tests/test_cli.cpp +++ b/apps/cli/tests/test_cli.cpp @@ -7,13 +7,12 @@ #include "cli/io.h" #include -#include -#include #include #include #include #include #include +#include #include #include #include @@ -23,8 +22,6 @@ namespace { int failures = 0; -constexpr char kRuntimeCohort[] = "0123456789abcdef0123456789abcdef"; -constexpr char kForeignCohort[] = "fedcba9876543210fedcba9876543210"; void check(bool condition, const char* name) { if (!condition) { @@ -50,71 +47,13 @@ bool parse_throws(std::vector arguments) { } } -void touch(const std::filesystem::path& path, const char* contents = "test") { - std::filesystem::create_directories(path.parent_path()); - std::ofstream file(path); - file << contents; -} - -void make_runtime_library(const std::filesystem::path& path, const char* cohort = kRuntimeCohort) { - std::filesystem::create_directories(path.parent_path()); - const std::string section_names{"\0.shstrtab\0.dynstr\0", 19}; - std::string dynamic_strings(1, '\0'); - dynamic_strings += "trtmc_build_cohort_"; - dynamic_strings += cohort; - dynamic_strings.push_back('\0'); - - const std::size_t section_names_offset = sizeof(Elf64_Ehdr); - const std::size_t dynamic_strings_offset = section_names_offset + section_names.size(); - const std::size_t section_headers_offset = - (dynamic_strings_offset + dynamic_strings.size() + 7U) & ~std::size_t{7U}; - std::vector image(section_headers_offset + 3 * sizeof(Elf64_Shdr), '\0'); - - Elf64_Ehdr header{}; - std::memcpy(header.e_ident, ELFMAG, SELFMAG); - header.e_ident[EI_CLASS] = ELFCLASS64; - header.e_ident[EI_DATA] = ELFDATA2LSB; - header.e_ident[EI_VERSION] = EV_CURRENT; - header.e_type = ET_DYN; - header.e_machine = EM_X86_64; - header.e_version = EV_CURRENT; - header.e_ehsize = sizeof(Elf64_Ehdr); - header.e_shoff = section_headers_offset; - header.e_shentsize = sizeof(Elf64_Shdr); - header.e_shnum = 3; - header.e_shstrndx = 1; - std::memcpy(image.data(), &header, sizeof(header)); - std::memcpy(image.data() + section_names_offset, section_names.data(), section_names.size()); - std::memcpy(image.data() + dynamic_strings_offset, dynamic_strings.data(), - dynamic_strings.size()); - - Elf64_Shdr section_names_header{}; - section_names_header.sh_name = 1; - section_names_header.sh_type = SHT_STRTAB; - section_names_header.sh_offset = section_names_offset; - section_names_header.sh_size = section_names.size(); - std::memcpy(image.data() + section_headers_offset + sizeof(Elf64_Shdr), §ion_names_header, - sizeof(section_names_header)); - Elf64_Shdr dynamic_strings_header{}; - dynamic_strings_header.sh_name = 11; - dynamic_strings_header.sh_type = SHT_STRTAB; - dynamic_strings_header.sh_offset = dynamic_strings_offset; - dynamic_strings_header.sh_size = dynamic_strings.size(); - std::memcpy(image.data() + section_headers_offset + 2 * sizeof(Elf64_Shdr), - &dynamic_strings_header, sizeof(dynamic_strings_header)); - - std::ofstream file(path, std::ios::binary); - file.write(image.data(), static_cast(image.size())); -} - -void make_runtime_root(const std::filesystem::path& root, bool with_byok = false, - const char* cohort = kRuntimeCohort) { - make_runtime_library(root / "libtrtmc_core.so", cohort); - make_runtime_library(root / "libtrtmc_runtime.so", cohort); - make_runtime_library(root / "libtrtmc_backend_trt.so", cohort); - make_runtime_library(root / "libtrtmc_model_gpt2.so", cohort); - if (with_byok) - make_runtime_library(root / "libtrtmc_byok_tvm_ffi.so", cohort); +std::string path_key(const std::filesystem::path& path) { + std::error_code error; + const auto absolute = std::filesystem::absolute(path, error); + if (error) + return path.lexically_normal().string(); + const auto canonical = std::filesystem::weakly_canonical(absolute, error); + return (error ? absolute.lexically_normal() : canonical).string(); } void write_fake_bundle(const std::filesystem::path& path) { @@ -135,12 +74,19 @@ void write_fake_bundle(const std::filesystem::path& path) { void check_cli_runtime_discovery(const std::filesystem::path& runtime_root, const std::filesystem::path& core_library, const std::filesystem::path& runtime_library) { - std::filesystem::copy_file(core_library, runtime_root / "libtrtmc_core.so", + const auto discovered_root = runtime_root.parent_path() / "cli-e2e-runtime"; + std::filesystem::remove_all(discovered_root); + std::filesystem::create_directories(discovered_root); + std::filesystem::copy_file(core_library, discovered_root / "libtrtmc_core.so", std::filesystem::copy_options::overwrite_existing); - std::filesystem::copy_file(runtime_library, runtime_root / "libtrtmc_runtime.so", + std::filesystem::copy_file(runtime_library, discovered_root / "libtrtmc_runtime.so", std::filesystem::copy_options::overwrite_existing); - const auto bundle = runtime_root / "cli-fake.bundle"; - const auto input = runtime_root / "cli-fake-input.f32"; + std::filesystem::copy_file(runtime_root / "libtrtmc_backend_fake.so", + discovered_root / "libtrtmc_backend_fake.so"); + std::filesystem::copy_file(runtime_root / "libtrtmc_model_fake.so", + discovered_root / "libtrtmc_model_fake.so"); + const auto bundle = discovered_root / "cli-fake.bundle"; + const auto input = discovered_root / "cli-fake-input.f32"; write_fake_bundle(bundle); { const float values[] = {1.0F, 2.0F, 3.0F}; @@ -149,7 +95,7 @@ void check_cli_runtime_discovery(const std::filesystem::path& runtime_root, } const auto previous_directory = std::filesystem::current_path(); - std::filesystem::current_path(runtime_root); + std::filesystem::current_path(discovered_root); std::vector arguments{"trtmc", "forecast", bundle.string(), "--input", input.string()}; std::vector argv; @@ -159,21 +105,19 @@ void check_cli_runtime_discovery(const std::filesystem::path& runtime_root, std::ostringstream error; const int result = trtmc::cli::run(static_cast(argv.size()), argv.data(), output, error); std::filesystem::current_path(previous_directory); - check(result == 0 && error.str() == "Using TRTMC runtime: " + runtime_root.string() + "\n", + check(result == 0 && error.str() == "Using TRTMC runtime: " + discovered_root.string() + "\n", "CLI executes with a runtime discovered from the current directory"); check(output.str().find("\"shape\":[1,3]") != std::string::npos, "automatically discovered runtime dispatches the bundle task"); - std::filesystem::remove(bundle); - std::filesystem::remove(input); - std::filesystem::remove(runtime_root / "libtrtmc_core.so"); - std::filesystem::remove(runtime_root / "libtrtmc_runtime.so"); + std::filesystem::remove_all(discovered_root); } bool resolve_throws(const trtmc::BundleInfo& bundle, - const trtmc::cli::RuntimeRootSearchContext& context, std::string& message) { + const trtmc::cli::RuntimeRootSearchContext& context, + const trtmc::cli::RuntimeRootMatcher& matches, std::string& message) { try { - (void)trtmc::cli::resolve_runtime_root(bundle, {}, false, context); + (void)trtmc::cli::resolve_runtime_root(bundle, {}, false, context, matches); return false; } catch (const std::runtime_error& error) { message = error.what(); @@ -452,103 +396,98 @@ int main(int argc, char** argv) { const auto current_root = runtime_test_root / "current"; const auto loaded_runtime_root = runtime_test_root / "loaded-runtime"; const auto optional_root = runtime_test_root / "optional"; - make_runtime_root(current_root); - make_runtime_root(loaded_runtime_root); - make_runtime_root(optional_root); + std::filesystem::create_directories(current_root); + std::filesystem::create_directories(loaded_runtime_root); + std::filesystem::create_directories(optional_root); + std::set complete_roots{ + path_key(current_root), + path_key(loaded_runtime_root), + path_key(optional_root), + }; + std::set byok_roots; + const trtmc::cli::RuntimeRootMatcher matches = + [&](const trtmc::BundleInfo&, const std::filesystem::path& candidate, bool require_byok) { + const std::string key = path_key(candidate); + return complete_roots.count(key) != 0 && (!require_byok || byok_roots.count(key) != 0); + }; trtmc::cli::RuntimeRootSearchContext runtime_context; runtime_context.current_directory = current_root; - runtime_context.runtime_library = loaded_runtime_root / "libtrtmc_runtime.so"; - runtime_context.cohort_id = kRuntimeCohort; + runtime_context.loaded_runtime_root = loaded_runtime_root; runtime_context.runtime_path = optional_root.string(); - check(trtmc::cli::resolve_runtime_root(runtime_bundle, {}, false, runtime_context) == + check(trtmc::cli::resolve_runtime_root(runtime_bundle, {}, false, runtime_context, matches) == current_root.string(), "current directory wins automatic runtime discovery"); - std::filesystem::remove(current_root / "libtrtmc_model_gpt2.so"); - check(trtmc::cli::resolve_runtime_root(runtime_bundle, {}, false, runtime_context) == - loaded_runtime_root.string(), - "incomplete current directory is skipped as a whole runtime cohort"); - - const auto malformed_root = runtime_test_root / "malformed"; - touch(malformed_root / "libtrtmc_core.so", "not an ELF file"); - std::filesystem::resize_file(malformed_root / "libtrtmc_core.so", 64ULL * 1024ULL * 1024ULL); - runtime_context.current_directory = malformed_root; - check(trtmc::cli::resolve_runtime_root(runtime_bundle, {}, false, runtime_context) == + complete_roots.erase(path_key(current_root)); + check(trtmc::cli::resolve_runtime_root(runtime_bundle, {}, false, runtime_context, matches) == loaded_runtime_root.string(), - "large malformed candidates are rejected without an unbounded scan"); + "an invalid current-directory candidate is skipped as a whole"); const auto wheel_prefix = runtime_test_root / "wheel"; const auto wheel_bin = wheel_prefix / "bin"; const auto wheel_runtime = wheel_prefix / "lib" / "python3.12" / "dist-packages" / "tensorrt_model_connect" / "bin"; - touch(wheel_bin / "trtmc"); - make_runtime_library(wheel_bin / "libtrtmc_core.so"); - make_runtime_library(wheel_bin / "libtrtmc_runtime.so"); - make_runtime_root(wheel_runtime); + std::filesystem::create_directories(wheel_bin); + std::filesystem::create_directories(wheel_runtime); + complete_roots.insert(path_key(wheel_runtime)); std::filesystem::create_directory_symlink("lib", wheel_prefix / "lib64"); runtime_context = {}; runtime_context.executable = wheel_bin / "trtmc"; - runtime_context.runtime_library = wheel_bin / "libtrtmc_runtime.so"; - runtime_context.cohort_id = kRuntimeCohort; - check(trtmc::cli::resolve_runtime_root(runtime_bundle, {}, false, runtime_context) == + runtime_context.loaded_runtime_root = wheel_bin; + check(trtmc::cli::resolve_runtime_root(runtime_bundle, {}, false, runtime_context, matches) == wheel_runtime.string(), "wheel runtime is discovered once when lib64 aliases lib"); const auto second_wheel_runtime = wheel_prefix / "lib" / "python3.13" / "site-packages" / "tensorrt_model_connect" / "bin"; - make_runtime_root(second_wheel_runtime); + std::filesystem::create_directories(second_wheel_runtime); + complete_roots.insert(path_key(second_wheel_runtime)); std::string discovery_error; - check(resolve_throws(runtime_bundle, runtime_context, discovery_error) && + check(resolve_throws(runtime_bundle, runtime_context, matches, discovery_error) && discovery_error.find("Multiple installed TRTMC runtimes") != std::string::npos, "ambiguous matching wheel runtimes require an explicit selection"); + complete_roots.erase(path_key(second_wheel_runtime)); std::filesystem::remove_all(second_wheel_runtime); const auto first_optional = runtime_test_root / "first-optional"; const auto second_optional = runtime_test_root / "second-optional"; const auto loaded_libraries = runtime_test_root / "loaded-libraries"; - touch(first_optional / "libtrtmc_backend_trt.so"); - make_runtime_root(second_optional); - make_runtime_library(loaded_libraries / "libtrtmc_runtime.so"); + std::filesystem::create_directories(first_optional); + std::filesystem::create_directories(second_optional); + std::filesystem::create_directories(loaded_libraries); + complete_roots.insert(path_key(second_optional)); runtime_context = {}; - runtime_context.runtime_library = loaded_libraries / "libtrtmc_runtime.so"; - runtime_context.cohort_id = kRuntimeCohort; + runtime_context.loaded_runtime_root = loaded_libraries; runtime_context.runtime_path = first_optional.string() + ":" + second_optional.string(); - check(trtmc::cli::resolve_runtime_root(runtime_bundle, {}, false, runtime_context) == + check(trtmc::cli::resolve_runtime_root(runtime_bundle, {}, false, runtime_context, matches) == second_optional.string(), "dedicated runtime path preserves directory order"); const auto byok_root = runtime_test_root / "byok"; - make_runtime_root(byok_root, true); + std::filesystem::create_directories(byok_root); + complete_roots.insert(path_key(byok_root)); + byok_roots.insert(path_key(byok_root)); runtime_context.runtime_path = second_optional.string() + ":" + byok_root.string(); - check(trtmc::cli::resolve_runtime_root(runtime_bundle, {}, true, runtime_context) == + check(trtmc::cli::resolve_runtime_root(runtime_bundle, {}, true, runtime_context, matches) == byok_root.string(), - "BYOK discovery requires the extension in the selected runtime cohort"); + "BYOK discovery delegates its extra requirement to the runtime matcher"); const auto explicit_root = runtime_test_root / "explicit-missing"; check(trtmc::cli::resolve_runtime_root(runtime_bundle, explicit_root.string(), false, - runtime_context) == explicit_root.string(), + runtime_context, {}) == explicit_root.string(), "explicit runtime root bypasses automatic fallback"); - const auto foreign_root = runtime_test_root / "foreign-build"; - make_runtime_root(foreign_root); - make_runtime_library(foreign_root / "libtrtmc_model_gpt2.so", kForeignCohort); - runtime_context.runtime_path = foreign_root.string(); - check(resolve_throws(runtime_bundle, runtime_context, discovery_error), - "automatic discovery rejects a foreign family with matching core/runtime files"); - runtime_context = {}; runtime_context.current_directory = first_optional; - runtime_context.runtime_library = loaded_libraries / "libtrtmc_runtime.so"; - runtime_context.cohort_id = kRuntimeCohort; + runtime_context.loaded_runtime_root = loaded_libraries; runtime_context.runtime_path = (runtime_test_root / "family-only").string(); - touch(runtime_test_root / "family-only" / "libtrtmc_model_gpt2.so"); - check(resolve_throws(runtime_bundle, runtime_context, discovery_error), + std::filesystem::create_directories(runtime_test_root / "family-only"); + check(resolve_throws(runtime_bundle, runtime_context, matches, discovery_error), "runtime discovery never combines partial roots"); - check(discovery_error.find("libtrtmc_backend_trt.so") != std::string::npos && - discovery_error.find("libtrtmc_model_gpt2.so") != std::string::npos && + check(discovery_error.find("bundle 'gpt2/trt'") != std::string::npos && discovery_error.find("--runtime-root") != std::string::npos, - "runtime discovery failure identifies required files and explicit override"); + "runtime discovery failure identifies the bundle and explicit override"); std::filesystem::remove_all(runtime_test_root); if (argc == 4) check_cli_runtime_discovery(argv[1], argv[2], argv[3]); diff --git a/core/runtime/include/trtmc/runtime/runtime_root.h b/core/runtime/include/trtmc/runtime/runtime_root.h new file mode 100644 index 0000000000..a2329fe9d5 --- /dev/null +++ b/core/runtime/include/trtmc/runtime/runtime_root.h @@ -0,0 +1,26 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include "trtmc/bundle.h" + +#include + +namespace trtmc { + +// Return the directory containing the runtime loader used by this process. +// Applications may use this as a discovery candidate; load_task remains the +// only operation that loads a family and backend. +std::string loaded_runtime_root(); + +// Return whether runtime_root contains one complete build cohort for bundle +// that matches the core and runtime loader already active in this process. +// This function validates one explicit candidate and never searches, loads, or +// falls back to another directory. +bool runtime_root_matches_loaded_build(const BundleInfo& bundle, const std::string& runtime_root, + bool require_byok = false); + +} // namespace trtmc diff --git a/core/runtime/loader/family_loader.cpp b/core/runtime/loader/family_loader.cpp index 76c419e633..09fe82ec70 100644 --- a/core/runtime/loader/family_loader.cpp +++ b/core/runtime/loader/family_loader.cpp @@ -7,10 +7,16 @@ #include "runtime/bundle/bundle_format.h" #include "trtmc/runtime/family_factory.h" +#include "trtmc/runtime/runtime_root.h" #include "trtmc/runtime/trt_backend.h" +#include +#include +#include #include +#include #include +#include #include #include #include @@ -28,6 +34,95 @@ namespace fs = std::filesystem; using CreateBackendFn = IBackend* (*)(); using DestroyBackendFn = void (*)(IBackend*); +std::string backend_library_name(const std::string& backend_id) { + return "libtrtmc_backend_" + backend_id + ".so"; +} + +std::string family_library_name(const std::string& family_id) { + return "libtrtmc_model_" + family_id + ".so"; +} + +constexpr std::uint64_t kMaxElfStringTableSize = 16ULL * 1024ULL * 1024ULL; + +bool read_at(std::ifstream& input, std::uintmax_t file_size, std::uint64_t offset, + void* destination, std::size_t size) { + if (offset > file_size || size > file_size - offset) + return false; + input.clear(); + input.seekg(static_cast(offset)); + input.read(static_cast(destination), static_cast(size)); + return input.good(); +} + +bool has_supported_elf_identity(const Elf64_Ehdr& header) { + return header.e_ident[EI_MAG0] == ELFMAG0 && header.e_ident[EI_MAG1] == ELFMAG1 && + header.e_ident[EI_MAG2] == ELFMAG2 && header.e_ident[EI_MAG3] == ELFMAG3 && + header.e_ident[EI_CLASS] == ELFCLASS64 && header.e_ident[EI_DATA] == ELFDATA2LSB; +} + +bool has_valid_section_table(const Elf64_Ehdr& header, std::uintmax_t file_size) { + return header.e_shentsize == sizeof(Elf64_Shdr) && header.e_shnum != 0 && + header.e_shstrndx < header.e_shnum && header.e_shoff <= file_size && + header.e_shnum <= (file_size - header.e_shoff) / sizeof(Elf64_Shdr); +} + +bool read_section_header(std::ifstream& input, std::uintmax_t file_size, const Elf64_Ehdr& header, + std::size_t index, Elf64_Shdr& section) { + const std::uint64_t offset = header.e_shoff + index * sizeof(Elf64_Shdr); + return read_at(input, file_size, offset, §ion, sizeof(section)); +} + +bool read_string_table(std::ifstream& input, std::uintmax_t file_size, const Elf64_Shdr& section, + std::string& contents) { + if (section.sh_size > kMaxElfStringTableSize) + return false; + contents.assign(static_cast(section.sh_size), '\0'); + return read_at(input, file_size, section.sh_offset, contents.data(), contents.size()); +} + +bool section_has_name(const Elf64_Shdr& section, const std::string& names, const char* expected) { + if (section.sh_name >= names.size()) + return false; + const auto end = names.find('\0', section.sh_name); + return end != std::string::npos && + names.compare(section.sh_name, end - section.sh_name, expected) == 0; +} + +bool is_lower_hex(unsigned char character) { + return (character >= '0' && character <= '9') || (character >= 'a' && character <= 'f'); +} + +bool read_named_string_table(std::ifstream& input, std::uintmax_t file_size, + const Elf64_Ehdr& header, const std::string& section_names, + const char* expected_name, std::string& contents) { + for (std::size_t index = 0; index < header.e_shnum; ++index) { + Elf64_Shdr section{}; + if (!read_section_header(input, file_size, header, index, section)) + return false; + if (!section_has_name(section, section_names, expected_name)) + continue; + return read_string_table(input, file_size, section, contents); + } + return false; +} + +std::string find_build_cohort(const std::string& strings) { + static constexpr char prefix[] = "trtmc_build_cohort_"; + static constexpr std::size_t id_size = 32; + std::size_t position = strings.find(prefix); + while (position != std::string::npos) { + const std::size_t id_begin = position + sizeof(prefix) - 1; + const std::size_t marker_end = id_begin + id_size; + if (marker_end < strings.size() && strings[marker_end] == '\0' && + std::all_of(strings.begin() + static_cast(id_begin), + strings.begin() + static_cast(marker_end), is_lower_hex)) { + return strings.substr(id_begin, id_size); + } + position = strings.find(prefix, position + 1); + } + return {}; +} + bool is_safe_id(const std::string& value) { if (value.empty() || value.front() < 'a' || value.front() > 'z') return false; @@ -59,6 +154,72 @@ fs::path explicit_runtime_root(const std::string& runtime_root) { return root.lexically_normal(); } +std::string read_build_cohort(const fs::path& library) { + std::ifstream input(library, std::ios::binary); + if (!input) + return {}; + + std::error_code error; + const std::uintmax_t file_size = fs::file_size(library, error); + if (error || file_size < sizeof(Elf64_Ehdr)) + return {}; + + Elf64_Ehdr header{}; + if (!read_at(input, file_size, 0, &header, sizeof(header)) || + !has_supported_elf_identity(header) || !has_valid_section_table(header, file_size)) { + return {}; + } + + Elf64_Shdr names_header{}; + if (!read_section_header(input, file_size, header, header.e_shstrndx, names_header)) + return {}; + std::string names; + if (!read_string_table(input, file_size, names_header, names)) + return {}; + + std::string strings; + if (!read_named_string_table(input, file_size, header, names, ".dynstr", strings)) + return {}; + return find_build_cohort(strings); +} + +fs::path loaded_library_path(const void* symbol) { + Dl_info info{}; + if (dladdr(symbol, &info) == 0 || info.dli_fname == nullptr) + return {}; + std::error_code error; + fs::path path = fs::absolute(info.dli_fname, error); + if (error) + return info.dli_fname; + fs::path normalized = fs::weakly_canonical(path, error); + return error ? path.lexically_normal() : normalized; +} + +struct LoadedBuild { + fs::path runtime_library; + std::string cohort_id; +}; + +const LoadedBuild& loaded_build() { + static const LoadedBuild build = [] { + using InspectBundleFn = BundleInfo (*)(const std::string&); + using LoadTaskFn = std::unique_ptr (*)(const std::string&, const std::string&, + std::uint64_t, const std::string&, bool); + const auto inspect_bundle_function = static_cast(&InspectBundle); + const auto load_task_function = static_cast(&load_task); + const fs::path core_library = + loaded_library_path(reinterpret_cast(inspect_bundle_function)); + const fs::path runtime_library = + loaded_library_path(reinterpret_cast(load_task_function)); + const std::string core_cohort = read_build_cohort(core_library); + const std::string runtime_cohort = read_build_cohort(runtime_library); + return LoadedBuild{runtime_library, !core_cohort.empty() && core_cohort == runtime_cohort + ? core_cohort + : std::string{}}; + }(); + return build; +} + class SharedLibrary { public: explicit SharedLibrary(const fs::path& path) : path_(path.string()) { @@ -98,7 +259,7 @@ class SharedLibrary { class BackendLibrary { public: BackendLibrary(const fs::path& runtime_root, const std::string& backend_id) - : library_(runtime_root / ("libtrtmc_backend_" + backend_id + ".so")) { + : library_(runtime_root / backend_library_name(backend_id)) { const auto create = reinterpret_cast(library_.require_symbol("trtmc_create_backend")); destroy_ = @@ -136,7 +297,7 @@ class BackendLibrary { class FamilyLibrary { public: FamilyLibrary(const fs::path& runtime_root, const std::string& family_id) - : library_(runtime_root / ("libtrtmc_model_" + family_id + ".so")), + : library_(runtime_root / family_library_name(family_id)), create_(reinterpret_cast(library_.require_symbol(kCreateFamilySymbol))) {} FamilyLibrary(const FamilyLibrary&) = delete; @@ -228,7 +389,7 @@ RuntimeLibraryCache& runtime_library_cache() { } IBackend& cached_backend(const fs::path& runtime_root, const std::string& backend_id) { - const std::string path = (runtime_root / ("libtrtmc_backend_" + backend_id + ".so")).string(); + const std::string path = (runtime_root / backend_library_name(backend_id)).string(); auto& cache = runtime_library_cache(); std::lock_guard lock(cache.mutex); const auto found = cache.backends.find(path); @@ -258,7 +419,7 @@ IBackend& cached_configured_backend(IBackend& backend, const std::string& runtim } FamilyLibrary& cached_family(const fs::path& runtime_root, const std::string& family_id) { - const std::string path = (runtime_root / ("libtrtmc_model_" + family_id + ".so")).string(); + const std::string path = (runtime_root / family_library_name(family_id)).string(); auto& cache = runtime_library_cache(); std::lock_guard lock(cache.mutex); const auto found = cache.families.find(path); @@ -283,6 +444,43 @@ void require_matching_task(const BundleInfo& info, const ITask& task) { } // namespace +std::string loaded_runtime_root() { + const fs::path& runtime_library = loaded_build().runtime_library; + if (runtime_library.empty()) + throw std::runtime_error("Unable to locate the active TRTMC runtime loader"); + return runtime_library.parent_path().string(); +} + +bool runtime_root_matches_loaded_build(const BundleInfo& bundle, const std::string& runtime_root, + bool require_byok) { + require_safe_id("family", bundle.family); + require_safe_id("backend", bundle.backend); + const std::string& cohort_id = loaded_build().cohort_id; + if (runtime_root.empty() || cohort_id.empty()) + return false; + + std::error_code error; + fs::path root = fs::absolute(runtime_root, error); + if (error) + return false; + root = root.lexically_normal(); + + std::vector required{ + "libtrtmc_core.so", + "libtrtmc_runtime.so", + backend_library_name(bundle.backend), + family_library_name(bundle.family), + }; + if (require_byok) + required.emplace_back("libtrtmc_byok_tvm_ffi.so"); + + return std::all_of(required.begin(), required.end(), [&](const std::string& library) { + std::error_code library_error; + const fs::path path = root / library; + return fs::is_regular_file(path, library_error) && read_build_cohort(path) == cohort_id; + }); +} + std::unique_ptr load_task(const std::string& bundle_path, const std::string& runtime_root, std::uint64_t kv_cache_size_bytes, const std::string& runtime_cache_path, bool cuda_graphs) { diff --git a/core/runtime/tests/test_runtime_root.cpp b/core/runtime/tests/test_runtime_root.cpp new file mode 100644 index 0000000000..492cc9c137 --- /dev/null +++ b/core/runtime/tests/test_runtime_root.cpp @@ -0,0 +1,114 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "trtmc/runtime/runtime_root.h" + +#include +#include +#include +#include +#include +#include + +namespace { + +namespace fs = std::filesystem; + +int failures = 0; + +void check(bool condition, const char* name) { + if (!condition) { + std::cerr << "FAIL: " << name << '\n'; + ++failures; + } +} + +void copy_library(const fs::path& source, const fs::path& root, const char* name) { + fs::copy_file(source, root / name, fs::copy_options::overwrite_existing); +} + +void change_build_cohort(const fs::path& library) { + static constexpr char marker[] = "trtmc_build_cohort_"; + std::ifstream input(library, std::ios::binary); + std::string contents{std::istreambuf_iterator(input), std::istreambuf_iterator()}; + const std::size_t marker_position = contents.find(marker); + if (marker_position == std::string::npos) + throw std::runtime_error("test library has no build-cohort marker"); + const std::size_t id_position = marker_position + sizeof(marker) - 1; + contents[id_position] = contents[id_position] == '0' ? '1' : '0'; + std::ofstream output(library, std::ios::binary | std::ios::trunc); + output.write(contents.data(), static_cast(contents.size())); +} + +} // namespace + +int main(int argc, char** argv) { + if (argc != 6) { + std::cerr << "usage: test_runtime_root ROOT CORE RUNTIME BACKEND FAMILY\n"; + return 2; + } + + const fs::path root = argv[1]; + const fs::path core_library = argv[2]; + const fs::path runtime_library = argv[3]; + const fs::path backend_library = argv[4]; + const fs::path family_library = argv[5]; + fs::remove_all(root); + fs::create_directories(root); + + copy_library(core_library, root, "libtrtmc_core.so"); + copy_library(runtime_library, root, "libtrtmc_runtime.so"); + copy_library(backend_library, root, "libtrtmc_backend_fake.so"); + copy_library(family_library, root, "libtrtmc_model_fake.so"); + const trtmc::BundleInfo bundle{1, "fake", "time_series_forecast", "fake", {}}; + + std::error_code error; + check(fs::equivalent(trtmc::loaded_runtime_root(), runtime_library.parent_path(), error), + "runtime loader reports its active installation root"); + check(trtmc::runtime_root_matches_loaded_build(bundle, root.string()), + "loader contract accepts one complete matching build cohort"); + + fs::remove(root / "libtrtmc_backend_fake.so"); + check(!trtmc::runtime_root_matches_loaded_build(bundle, root.string()), + "loader contract rejects an incomplete root"); + copy_library(backend_library, root, "libtrtmc_backend_fake.so"); + + check(!trtmc::runtime_root_matches_loaded_build(bundle, root.string(), true), + "loader contract requires the BYOK DSO when requested"); + copy_library(core_library, root, "libtrtmc_byok_tvm_ffi.so"); + check(trtmc::runtime_root_matches_loaded_build(bundle, root.string(), true), + "loader contract accepts a matching BYOK DSO"); + + change_build_cohort(root / "libtrtmc_model_fake.so"); + check(!trtmc::runtime_root_matches_loaded_build(bundle, root.string()), + "loader contract rejects a family from another build cohort"); + copy_library(family_library, root, "libtrtmc_model_fake.so"); + + { + std::ofstream malformed(root / "libtrtmc_model_fake.so", + std::ios::binary | std::ios::trunc); + malformed << "not an ELF file"; + } + fs::resize_file(root / "libtrtmc_model_fake.so", 64ULL * 1024ULL * 1024ULL); + check(!trtmc::runtime_root_matches_loaded_build(bundle, root.string()), + "loader contract rejects a large malformed candidate with bounded reads"); + + bool unsafe_rejected = false; + try { + const trtmc::BundleInfo unsafe{1, "../fake", "time_series_forecast", "fake", {}}; + (void)trtmc::runtime_root_matches_loaded_build(unsafe, root.string()); + } catch (const std::runtime_error&) { + unsafe_rejected = true; + } + check(unsafe_rejected, "loader contract owns safe family and backend validation"); + + fs::remove_all(root); + if (failures != 0) { + std::cerr << failures << " runtime-root test(s) failed\n"; + return 1; + } + std::cout << "runtime-root tests passed\n"; + return 0; +} diff --git a/tools/tests/test_architecture.py b/tools/tests/test_architecture.py index f3969e24db..9bdecb34bd 100644 --- a/tools/tests/test_architecture.py +++ b/tools/tests/test_architecture.py @@ -303,6 +303,7 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "core/runtime/include/trtmc/runtime/device_tensor.h", "core/runtime/include/trtmc/runtime/family_factory.h", "core/runtime/include/trtmc/runtime/family_loader.h", + "core/runtime/include/trtmc/runtime/runtime_root.h", "core/runtime/include/trtmc/runtime/tensor.h", "core/runtime/include/trtmc/runtime/trt_backend.h", "core/runtime/include/trtmc/runtime/trt_module.h", @@ -311,6 +312,7 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "core/runtime/tests/test_bundle_format_v1.cpp", "core/runtime/tests/test_byok_shape_spec.cpp", "core/runtime/tests/test_family_loader.cpp", + "core/runtime/tests/test_runtime_root.cpp", "core/runtime/tests/test_task_api.cpp", "core/runtime/tests/test_trt_module_dynamic_input.cpp", } diff --git a/website/docs/architecture/ai-native-horizontal-scaling.md b/website/docs/architecture/ai-native-horizontal-scaling.md index 1d13406df2..130e2728f7 100644 --- a/website/docs/architecture/ai-native-horizontal-scaling.md +++ b/website/docs/architecture/ai-native-horizontal-scaling.md @@ -120,7 +120,7 @@ After each transfer, core no longer participates in model behavior. | Component | Owns | Explicitly does not own | | --- | --- | --- | | Native Core (`libtrtmc_core.so`) | bounded bundle reads, device tensors, stable engine primitives | `dlopen`, model config, weight mapping, preprocessing, request loops | -| Runtime Loader (`libtrtmc_runtime.so`) | safe family/backend names, explicit runtime root, exact `dlopen`, one control transfer | model pipelines, preprocessing, policy dispatch, family fallback | +| Runtime Loader (`libtrtmc_runtime.so`) | safe family/backend names, explicit runtime root, runtime-root/build-cohort validation, exact `dlopen`, one control transfer | path search policy, model pipelines, preprocessing, policy dispatch, family fallback | | Family | checkpoint identity, tasks/default, graph build, weights, section semantics, native pipeline, dispatch, bindings, pre/postprocessing | sibling families, shared model policy | | Bundle | header and named byte sections with bounded streaming I/O | model schema, section semantics, content hashes | | Task API | user behavior such as text, image, audio, embedding, and segmentation | family names, TensorRT objects, backend details | @@ -518,15 +518,21 @@ Runtime dispatch occurs once: Core, family, and backend DSOs are produced by one product build. There is no ABI negotiation, version translation, old-symbol alias, or compatibility shim. The human-facing `trtmc` CLI may discover a complete runtime cohort before this -control transfer. It prefers the current directory, then the runtime belonging -to the running CLI installation, followed by explicitly configured runtime -library paths. The public C++ load API still receives one explicit root, and -the loader never combines or falls back across roots. Every native artifact -carries the build-cohort identity generated when CMake configures the build. -Automatic -candidates must contain that same identity in core, runtime, backend, family, -and optional BYOK DSOs, preventing another build cohort from being selected -implicitly. +control transfer. The CLI owns only candidate enumeration and search order: it +prefers the current directory, then the runtime belonging to the running CLI +installation, followed by explicitly configured runtime library paths. For +each candidate, it asks the model-agnostic Runtime Loader contract to validate +the safe bundle identifiers, required DSO set, and build cohort. The CLI does +not derive DSO names or parse native artifact metadata. The public C++ load API +still receives one explicit root, and the loader never combines or falls back +across roots. + +Every native artifact carries the build-cohort identity generated when CMake +configures the build. Automatic candidates must contain that same identity in +core, runtime, backend, family, and optional BYOK DSOs, preventing another +build cohort from being selected implicitly. This is strict product-build +identity, not ABI compatibility negotiation: there is no compatible-version +selection, translation, or fallback. ### Task API diff --git a/website/docs/getting-started/quick-start.md b/website/docs/getting-started/quick-start.md index 10583595a0..71bd162ffd 100644 --- a/website/docs/getting-started/quick-start.md +++ b/website/docs/getting-started/quick-start.md @@ -61,9 +61,10 @@ single-directory runtime in this order: A complete GPT-2 TensorRT runtime contains matching `libtrtmc_core.so`, `libtrtmc_runtime.so`, `libtrtmc_backend_trt.so`, and `libtrtmc_model_gpt2.so` files. Candidates are never combined across -directories, and the CLI prints the automatically selected directory. If more -than one installed wheel runtime matches, select one with `--runtime-root DIR`. -An explicit root bypasses discovery. +directories: the CLI enumerates paths, and the Runtime Loader contract validates +each candidate without loading it. The CLI prints the automatically selected +directory. If more than one installed wheel runtime matches, select one with +`--runtime-root DIR`. An explicit root bypasses discovery. Every native artifact carries a build-cohort identity, and automatic discovery accepts a directory only when the identity matches the core and runtime already From 2b9e7f3801131f97bf197c498d5e5b6933014710 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Mon, 7 Sep 2026 15:01:06 +0000 Subject: [PATCH 04/13] refactor(runtime): validate exact plugin builds Move plugin-root structure and load validation into the Runtime Loader. Replace discovery-time ELF inspection with exact build, kind, and ID descriptors while preserving one-product-build and no-fallback behavior. Signed-off-by: chaofengw --- CMakeLists.txt | 77 ++++- apps/cli/cli.cpp | 49 +-- apps/cli/cli.h | 1 - apps/cli/tests/test_cli.cpp | 44 ++- core/runtime/byok/byok.cpp | 2 + core/runtime/include/trtmc/byok.h | 2 + .../include/trtmc/runtime/family_factory.h | 4 + .../include/trtmc/runtime/family_loader.h | 5 + .../include/trtmc/runtime/plugin_abi.h | 53 ++++ .../include/trtmc/runtime/runtime_root.h | 12 +- .../include/trtmc/runtime/trt_backend.h | 5 + core/runtime/loader/family_loader.cpp | 279 +++++++++--------- core/runtime/primitives/build_identity.cpp | 14 + core/runtime/tensorrt/rtx_backend.cpp | 2 + core/runtime/tensorrt/trt_backend.cpp | 2 + core/runtime/tests/fake_backend.cpp | 11 + core/runtime/tests/fake_family.cpp | 2 + core/runtime/tests/test_family_loader.cpp | 60 +++- core/runtime/tests/test_runtime_root.cpp | 61 ++-- families/albert/runtime/plugin.cpp | 2 + families/bark/runtime/plugin.cpp | 2 + families/bart/runtime/plugin.cpp | 2 + families/bert/runtime/plugin.cpp | 2 + families/bloom/runtime/plugin.cpp | 2 + families/canary/runtime/plugin.cpp | 2 + families/chronos_bolt/runtime/plugin.cpp | 2 + families/codegen/runtime/plugin.cpp | 2 + families/convbert/runtime/plugin.cpp | 2 + families/cosmos3/runtime/plugin.cpp | 2 + families/deberta/runtime/plugin.cpp | 2 + families/deepseek_ocr/runtime/plugin.cpp | 2 + families/deepseek_v2/runtime/plugin.cpp | 2 + families/dinov3/runtime/plugin.cpp | 2 + families/distilbert/runtime/plugin.cpp | 2 + families/dpr/runtime/plugin.cpp | 2 + families/eagle_vlm/runtime/plugin.cpp | 2 + families/electra/runtime/plugin.cpp | 2 + families/elf_flow/runtime/plugin.cpp | 2 + families/falcon/runtime/plugin.cpp | 2 + .../fast_foundation_stereo/runtime/plugin.cpp | 2 + families/flux/runtime/plugin.cpp | 2 + families/fnet/runtime/plugin.cpp | 2 + families/foundationpose/runtime/plugin.cpp | 2 + families/gemma/runtime/plugin.cpp | 2 + families/glm/runtime/plugin.cpp | 2 + families/gpt2/runtime/plugin.cpp | 2 + families/gpt_neo/runtime/plugin.cpp | 2 + families/gpt_neox/runtime/plugin.cpp | 2 + families/gpt_oss/runtime/plugin.cpp | 2 + families/granite/runtime/plugin.cpp | 2 + families/internlm/runtime/plugin.cpp | 2 + families/internvl/runtime/plugin.cpp | 2 + families/k2_horizon/runtime/plugin.cpp | 2 + families/lance/runtime/plugin.cpp | 2 + families/lerobot_act/runtime/plugin.cpp | 2 + families/lfm2/runtime/plugin.cpp | 2 + families/llama/runtime/plugin.cpp | 2 + families/locateanything/runtime/plugin.cpp | 2 + families/ltx_video/runtime/plugin.cpp | 2 + families/m2m_100/runtime/plugin.cpp | 2 + families/magpie_tts/runtime/plugin.cpp | 2 + families/mamba/runtime/plugin.cpp | 2 + families/marian/runtime/plugin.cpp | 2 + families/minimax_h3/runtime/plugin.cpp | 2 + families/mistral/runtime/plugin.cpp | 2 + families/mixtral/runtime/plugin.cpp | 2 + families/modernbert/runtime/plugin.cpp | 2 + families/moge/runtime/plugin.cpp | 2 + families/mpnet/runtime/plugin.cpp | 2 + families/nemotron/runtime/plugin.cpp | 2 + families/nemotron_h/runtime/plugin.cpp | 2 + .../runtime/plugin.cpp | 2 + .../runtime/plugin.cpp | 2 + .../nemotron_voicechat/runtime/plugin.cpp | 2 + families/olmo/runtime/plugin.cpp | 2 + families/olmo2/runtime/plugin.cpp | 2 + families/opt/runtime/plugin.cpp | 2 + families/patchtsmixer/runtime/plugin.cpp | 2 + families/patchtst/runtime/plugin.cpp | 2 + families/personaplex/runtime/plugin.cpp | 2 + families/phi/runtime/plugin.cpp | 2 + families/phi4_multimodal/runtime/plugin.cpp | 2 + families/phi_moe/runtime/plugin.cpp | 2 + families/pixart/runtime/plugin.cpp | 2 + families/qwen/runtime/plugin.cpp | 2 + families/qwen3_5/runtime/plugin.cpp | 2 + families/qwen3_8/runtime/plugin.cpp | 2 + families/qwen3_omni/runtime/plugin.cpp | 2 + families/qwen_image/runtime/plugin.cpp | 2 + families/qwen_moe/runtime/plugin.cpp | 2 + families/qwen_vl/runtime/plugin.cpp | 2 + families/roberta/runtime/plugin.cpp | 2 + families/rwkv/runtime/plugin.cpp | 2 + families/sam/runtime/plugin.cpp | 2 + families/sam2/runtime/plugin.cpp | 2 + families/sam3/runtime/plugin.cpp | 2 + families/sana_wm/runtime/plugin.cpp | 2 + families/segformer/runtime/plugin.cpp | 2 + families/stablelm/runtime/plugin.cpp | 2 + families/starcoder2/runtime/plugin.cpp | 2 + families/t5/runtime/plugin.cpp | 2 + families/timesfm/runtime/plugin.cpp | 2 + families/timm_densenet/runtime/plugin.cpp | 2 + families/timm_efficientnet/runtime/plugin.cpp | 2 + families/timm_inception/runtime/plugin.cpp | 2 + families/timm_mnasnet/runtime/plugin.cpp | 2 + families/timm_mobilenetv2/runtime/plugin.cpp | 2 + families/timm_mobilenetv3/runtime/plugin.cpp | 2 + families/timm_repvgg/runtime/plugin.cpp | 2 + families/timm_resnet/runtime/plugin.cpp | 2 + families/timm_vgg/runtime/plugin.cpp | 2 + families/timm_vit/runtime/plugin.cpp | 2 + families/wan2_2_ti2v/runtime/plugin.cpp | 2 + families/wan_t2v/runtime/plugin.cpp | 2 + families/whisper/runtime/plugin.cpp | 2 + families/xglm/runtime/plugin.cpp | 2 + families/xlnet/runtime/plugin.cpp | 2 + families/z_image/runtime/plugin.cpp | 2 + tools/ci/package.py | 62 +++- tools/tests/test_architecture.py | 38 ++- tools/tests/test_new_ci.py | 54 +++- website/docs/api/cli-reference.md | 15 +- website/docs/api/overview.md | 8 +- .../ai-native-horizontal-scaling.md | 42 +-- .../docs/architecture/runtime-lifecycle.md | 12 +- website/docs/architecture/runtime-plugins.md | 36 ++- website/docs/getting-started/quick-start.md | 36 ++- website/docs/user-guides/run-inference.md | 11 +- 128 files changed, 837 insertions(+), 360 deletions(-) create mode 100644 core/runtime/include/trtmc/runtime/plugin_abi.h create mode 100644 core/runtime/primitives/build_identity.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 47ff1ab0b9..dd3e267554 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,19 +12,18 @@ set(CMAKE_CUDA_STANDARD_REQUIRED ON) set(CMAKE_POSITION_INDEPENDENT_CODE ON) set(CMAKE_BUILD_RPATH_USE_ORIGIN TRUE) -if(NOT TRTMC_BUILD_COHORT_ID) - string(RANDOM LENGTH 32 ALPHABET 0123456789abcdef TRTMC_BUILD_COHORT_ID) +set(TRTMC_BUILD_ID "" CACHE STRING "Exact product-build identity embedded in runtime plugins") +if(NOT TRTMC_BUILD_ID) + string(RANDOM LENGTH 32 ALPHABET 0123456789abcdef _trtmc_generated_build_id) + set(TRTMC_BUILD_ID "${_trtmc_generated_build_id}" CACHE STRING + "Exact product-build identity embedded in runtime plugins" FORCE + ) endif() -string(LENGTH "${TRTMC_BUILD_COHORT_ID}" _trtmc_build_cohort_id_length) -if(NOT _trtmc_build_cohort_id_length EQUAL 32 OR - NOT TRTMC_BUILD_COHORT_ID MATCHES "^[0-9a-f]+$") - message(FATAL_ERROR "TRTMC_BUILD_COHORT_ID must contain exactly 32 lowercase hex characters") +string(LENGTH "${TRTMC_BUILD_ID}" _trtmc_build_id_length) +if(NOT _trtmc_build_id_length EQUAL 32 OR NOT TRTMC_BUILD_ID MATCHES "^[0-9a-f]+$") + message(FATAL_ERROR "TRTMC_BUILD_ID must contain exactly 32 lowercase hex characters") endif() -set(_trtmc_build_cohort_symbol "trtmc_build_cohort_${TRTMC_BUILD_COHORT_ID}") -add_link_options( - "LINKER:--defsym=${_trtmc_build_cohort_symbol}=0" - "LINKER:--export-dynamic-symbol=${_trtmc_build_cohort_symbol}" -) +add_compile_definitions(TRTMC_BUILD_ID="${TRTMC_BUILD_ID}") include(GNUInstallDirs) find_package(CUDAToolkit REQUIRED) @@ -95,6 +94,7 @@ set(TRTMC_CUDART_LIBRARY CUDA::cudart) add_library(trtmc_core SHARED core/runtime/bundle/bundle_format.cpp + core/runtime/primitives/build_identity.cpp core/runtime/primitives/cuda_common.cpp core/runtime/primitives/device_tensor.cpp core/runtime/primitives/trt_common.cpp @@ -364,8 +364,6 @@ if(TRTMC_BUILD_TESTS) target_compile_options(test_cli PRIVATE -Wall -Wextra -Wpedantic) add_test(NAME cli COMMAND test_cli "${_trtmc_test_runtime_root}" - $ - $ ) add_library(trtmc_test_backend_fake SHARED core/runtime/tests/fake_backend.cpp) @@ -389,6 +387,48 @@ if(TRTMC_BUILD_TESTS) LIBRARY_OUTPUT_DIRECTORY "${_trtmc_test_runtime_root}" ) + add_library(trtmc_test_backend_incompatible SHARED core/runtime/tests/fake_backend.cpp) + target_include_directories(trtmc_test_backend_incompatible PRIVATE + ${PROJECT_SOURCE_DIR}/core/runtime/include + ) + target_link_libraries(trtmc_test_backend_incompatible PRIVATE CUDA::cudart) + target_compile_definitions(trtmc_test_backend_incompatible PRIVATE + TRTMC_FAKE_BACKEND_NAME="incompatible" + TRTMC_FAKE_INCOMPATIBLE_BUILD=1 + ) + set_target_properties(trtmc_test_backend_incompatible PROPERTIES + OUTPUT_NAME trtmc_backend_incompatible + LIBRARY_OUTPUT_DIRECTORY "${_trtmc_test_runtime_root}" + ) + + add_library(trtmc_test_core_incompatible SHARED + core/runtime/bundle/bundle_format.cpp + core/runtime/primitives/build_identity.cpp + core/runtime/primitives/cuda_common.cpp + core/runtime/primitives/device_tensor.cpp + core/runtime/primitives/trt_common.cpp + ) + target_include_directories(trtmc_test_core_incompatible + PUBLIC ${PROJECT_SOURCE_DIR}/core/runtime/include + PRIVATE ${PROJECT_SOURCE_DIR}/core + ) + target_include_directories(trtmc_test_core_incompatible SYSTEM PRIVATE + ${CUDAToolkit_INCLUDE_DIRS} + ) + target_link_libraries(trtmc_test_core_incompatible + PUBLIC CUDA::cudart + PRIVATE nlohmann_json::nlohmann_json + ) + target_compile_definitions(trtmc_test_core_incompatible PRIVATE + TRTMC_FAKE_INCOMPATIBLE_BUILD=1 + ) + target_compile_options(trtmc_test_core_incompatible PRIVATE -Wall -Wextra -Wpedantic) + set_target_properties(trtmc_test_core_incompatible PROPERTIES + OUTPUT_NAME trtmc_core + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tests/incompatible-core" + BUILD_RPATH "\$ORIGIN" + ) + add_library(trtmc_test_family_fake SHARED core/runtime/tests/fake_family.cpp) target_include_directories(trtmc_test_family_fake PRIVATE ${PROJECT_SOURCE_DIR}/core/runtime/include) target_link_libraries(trtmc_test_family_fake PRIVATE trtmc_core) @@ -406,7 +446,6 @@ if(TRTMC_BUILD_TESTS) add_dependencies(test_runtime_root trtmc_test_backend_fake trtmc_test_family_fake) add_test(NAME runtime_root COMMAND test_runtime_root "${CMAKE_BINARY_DIR}/tests/runtime-root-validator" - $ $ $ $ @@ -422,9 +461,18 @@ if(TRTMC_BUILD_TESTS) add_dependencies(test_family_loader trtmc_test_backend_fake trtmc_test_backend_fake_rtx + trtmc_test_backend_incompatible + trtmc_test_core_incompatible trtmc_test_family_fake ) add_test(NAME family_loader COMMAND test_family_loader "${_trtmc_test_runtime_root}") + add_test(NAME family_loader_incompatible_core + COMMAND ${CMAKE_COMMAND} -E env + "LD_LIBRARY_PATH=$" + $ + "${_trtmc_test_runtime_root}" + --expect-core-mismatch + ) add_executable(test_trt_module_dynamic_input core/runtime/tests/test_trt_module_dynamic_input.cpp @@ -577,6 +625,7 @@ install(FILES core/runtime/include/trtmc/runtime/device_tensor.h core/runtime/include/trtmc/runtime/family_factory.h core/runtime/include/trtmc/runtime/family_loader.h + core/runtime/include/trtmc/runtime/plugin_abi.h core/runtime/include/trtmc/runtime/runtime_root.h core/runtime/include/trtmc/runtime/tensor.h core/runtime/include/trtmc/runtime/trt_backend.h diff --git a/apps/cli/cli.cpp b/apps/cli/cli.cpp index 18e0b027da..e9fc1515e6 100644 --- a/apps/cli/cli.cpp +++ b/apps/cli/cli.cpp @@ -15,7 +15,6 @@ #include #include #include -#include #include #include #include @@ -203,8 +202,6 @@ void append_python_package_runtime_roots(std::vector& candidates, RuntimeRootSearchContext runtime_root_search_context() { RuntimeRootSearchContext context; - std::error_code error; - context.current_directory = fs::current_path(error); context.loaded_runtime_root = loaded_runtime_root(); std::vector executable(4096, '\0'); @@ -219,31 +216,6 @@ RuntimeRootSearchContext runtime_root_search_context() { return context; } -void load_byok_extension(const Command& command) { - using LoadKernelFn = const char* (*)(const char*, const char*, const char*) noexcept; - const fs::path extension = fs::path(command.runtime_root) / "libtrtmc_byok_tvm_ffi.so"; - dlerror(); - void* handle = dlopen(extension.c_str(), RTLD_NOW | RTLD_LOCAL); - if (handle == nullptr) { - const char* error = dlerror(); - throw std::runtime_error("unable to load BYOK extension '" + extension.string() + - "': " + (error != nullptr ? error : "unknown dlopen error")); - } - static auto* handles = new std::vector; - handles->push_back(handle); - dlerror(); - auto load = reinterpret_cast(dlsym(handle, "trtmc_load_byok_kernel")); - if (const char* error = dlerror(); error != nullptr || load == nullptr) { - throw std::runtime_error("BYOK extension is missing trtmc_load_byok_kernel"); - } - if (const char* error = load(command.options.at("--byok-library").c_str(), - command.options.at("--byok-function").c_str(), - command.options.at("--byok-name").c_str())) { - const std::string message = error; - throw std::runtime_error(message); - } -} - std::string take_value(int argc, char** argv, int& index, const std::string& option) { if (index + 1 >= argc) throw std::invalid_argument(option + " requires a value"); @@ -747,12 +719,10 @@ std::string resolve_runtime_root(const BundleInfo& bundle, const std::string& ex if (!matches) throw std::logic_error("runtime-root discovery requires a candidate matcher"); - std::vector current_candidates; std::vector installed_candidates; std::vector wheel_candidates; std::vector configured_candidates; std::set seen; - append_candidate(current_candidates, seen, context.current_directory); append_candidate(installed_candidates, seen, context.loaded_runtime_root); if (!context.executable.empty()) { @@ -769,13 +739,6 @@ std::string resolve_runtime_root(const BundleInfo& bundle, const std::string& ex append_path_list(configured_candidates, seen, context.runtime_path); std::vector searched; - if (!current_candidates.empty()) { - const auto& current = current_candidates.front(); - searched.push_back(current); - if (matches(bundle, current, require_byok)) { - return current.string(); - } - } for (const auto& candidate : installed_candidates) { searched.push_back(candidate); if (matches(bundle, candidate, require_byok)) @@ -1459,9 +1422,9 @@ void print_usage(std::ostream& output) { " [--kv-cache-size BYTES|GB|GiB]\n\n" "TensorRT-RTX runtime options:\n" " [--runtime-cache PATH] [--cuda-graphs]\n\n" - "Runtime discovery: current directory, the active trtmc installation, then\n" - "TRTMC_RUNTIME_PATH. LD_LIBRARY_PATH can select the active cohort before startup.\n" - "--runtime-root overrides discovery.\n"; + "Runtime discovery: the active trtmc installation, then TRTMC_RUNTIME_PATH.\n" + "The current directory is not searched; use TRTMC_RUNTIME_PATH=. explicitly.\n" + "--runtime-root selects one exact root without fallback.\n"; } int run(int argc, char** argv, std::ostream& output, std::ostream& error) { @@ -1503,13 +1466,15 @@ int run(int argc, char** argv, std::ostream& output, std::ostream& error) { resolve_runtime_root(bundle, {}, has_byok_library, runtime_root_search_context(), [](const BundleInfo& candidate_bundle, const fs::path& candidate, bool require_byok) { - return runtime_root_matches_loaded_build( + return runtime_root_contains_bundle( candidate_bundle, candidate.string(), require_byok); }); error << "Using TRTMC runtime: " << command.runtime_root << '\n'; } if (has_byok_library) - load_byok_extension(command); + load_byok_kernel_from_runtime( + command.runtime_root, command.options.at("--byok-library"), + command.options.at("--byok-function"), command.options.at("--byok-name")); std::unique_ptr task = load_task(command.bundle, command.runtime_root, command.kv_cache_size_bytes, command.runtime_cache_path, command.cuda_graphs); diff --git a/apps/cli/cli.h b/apps/cli/cli.h index 46e882e416..998fd816cc 100644 --- a/apps/cli/cli.h +++ b/apps/cli/cli.h @@ -64,7 +64,6 @@ struct Command { }; struct RuntimeRootSearchContext { - std::filesystem::path current_directory; std::filesystem::path loaded_runtime_root; std::filesystem::path executable; std::string runtime_path; diff --git a/apps/cli/tests/test_cli.cpp b/apps/cli/tests/test_cli.cpp index eafa6409bb..c0655acf81 100644 --- a/apps/cli/tests/test_cli.cpp +++ b/apps/cli/tests/test_cli.cpp @@ -7,6 +7,7 @@ #include "cli/io.h" #include +#include #include #include #include @@ -71,16 +72,10 @@ void write_fake_bundle(const std::filesystem::path& path) { output.write("{}PLAN", 6); } -void check_cli_runtime_discovery(const std::filesystem::path& runtime_root, - const std::filesystem::path& core_library, - const std::filesystem::path& runtime_library) { +void check_cli_runtime_discovery(const std::filesystem::path& runtime_root) { const auto discovered_root = runtime_root.parent_path() / "cli-e2e-runtime"; std::filesystem::remove_all(discovered_root); std::filesystem::create_directories(discovered_root); - std::filesystem::copy_file(core_library, discovered_root / "libtrtmc_core.so", - std::filesystem::copy_options::overwrite_existing); - std::filesystem::copy_file(runtime_library, discovered_root / "libtrtmc_runtime.so", - std::filesystem::copy_options::overwrite_existing); std::filesystem::copy_file(runtime_root / "libtrtmc_backend_fake.so", discovered_root / "libtrtmc_backend_fake.so"); std::filesystem::copy_file(runtime_root / "libtrtmc_model_fake.so", @@ -94,8 +89,10 @@ void check_cli_runtime_discovery(const std::filesystem::path& runtime_root, output.write(reinterpret_cast(values), sizeof(values)); } - const auto previous_directory = std::filesystem::current_path(); - std::filesystem::current_path(discovered_root); + const char* previous_runtime_path_value = std::getenv("TRTMC_RUNTIME_PATH"); + const bool had_runtime_path = previous_runtime_path_value != nullptr; + const std::string previous_runtime_path = had_runtime_path ? previous_runtime_path_value : ""; + setenv("TRTMC_RUNTIME_PATH", discovered_root.c_str(), 1); std::vector arguments{"trtmc", "forecast", bundle.string(), "--input", input.string()}; std::vector argv; @@ -104,9 +101,12 @@ void check_cli_runtime_discovery(const std::filesystem::path& runtime_root, std::ostringstream output; std::ostringstream error; const int result = trtmc::cli::run(static_cast(argv.size()), argv.data(), output, error); - std::filesystem::current_path(previous_directory); + if (had_runtime_path) + setenv("TRTMC_RUNTIME_PATH", previous_runtime_path.c_str(), 1); + else + unsetenv("TRTMC_RUNTIME_PATH"); check(result == 0 && error.str() == "Using TRTMC runtime: " + discovered_root.string() + "\n", - "CLI executes with a runtime discovered from the current directory"); + "CLI executes with a runtime discovered from TRTMC_RUNTIME_PATH"); check(output.str().find("\"shape\":[1,3]") != std::string::npos, "automatically discovered runtime dispatches the bundle task"); @@ -389,18 +389,15 @@ int main(int argc, char** argv) { "duplicate runtime root rejected"); const std::filesystem::path runtime_test_root = - argc == 4 ? std::filesystem::path(argv[1]).parent_path() / "cli-runtime-root-test" + argc == 2 ? std::filesystem::path(argv[1]).parent_path() / "cli-runtime-root-test" : std::filesystem::temp_directory_path() / "trtmc-cli-runtime-root-test"; std::filesystem::remove_all(runtime_test_root); const trtmc::BundleInfo runtime_bundle{1, "gpt2", "text_generation", "trt", {}}; - const auto current_root = runtime_test_root / "current"; const auto loaded_runtime_root = runtime_test_root / "loaded-runtime"; const auto optional_root = runtime_test_root / "optional"; - std::filesystem::create_directories(current_root); std::filesystem::create_directories(loaded_runtime_root); std::filesystem::create_directories(optional_root); std::set complete_roots{ - path_key(current_root), path_key(loaded_runtime_root), path_key(optional_root), }; @@ -412,17 +409,17 @@ int main(int argc, char** argv) { }; trtmc::cli::RuntimeRootSearchContext runtime_context; - runtime_context.current_directory = current_root; runtime_context.loaded_runtime_root = loaded_runtime_root; runtime_context.runtime_path = optional_root.string(); check(trtmc::cli::resolve_runtime_root(runtime_bundle, {}, false, runtime_context, matches) == - current_root.string(), - "current directory wins automatic runtime discovery"); + loaded_runtime_root.string(), + "the active runtime installation wins automatic discovery"); - complete_roots.erase(path_key(current_root)); + complete_roots.erase(path_key(loaded_runtime_root)); check(trtmc::cli::resolve_runtime_root(runtime_bundle, {}, false, runtime_context, matches) == - loaded_runtime_root.string(), - "an invalid current-directory candidate is skipped as a whole"); + optional_root.string(), + "the configured runtime path is used when the active installation is incomplete"); + complete_roots.insert(path_key(loaded_runtime_root)); const auto wheel_prefix = runtime_test_root / "wheel"; const auto wheel_bin = wheel_prefix / "bin"; @@ -479,7 +476,6 @@ int main(int argc, char** argv) { "explicit runtime root bypasses automatic fallback"); runtime_context = {}; - runtime_context.current_directory = first_optional; runtime_context.loaded_runtime_root = loaded_libraries; runtime_context.runtime_path = (runtime_test_root / "family-only").string(); std::filesystem::create_directories(runtime_test_root / "family-only"); @@ -489,8 +485,8 @@ int main(int argc, char** argv) { discovery_error.find("--runtime-root") != std::string::npos, "runtime discovery failure identifies the bundle and explicit override"); std::filesystem::remove_all(runtime_test_root); - if (argc == 4) - check_cli_runtime_discovery(argv[1], argv[2], argv[3]); + if (argc == 2) + check_cli_runtime_discovery(argv[1]); const auto dynamic_kv = parse({"trtmc", "run", "model.bundle", "--runtime-root", "lib", "--kv-cache-size", "1GiB"}); check(dynamic_kv.kv_cache_size_bytes == 1024ULL * 1024ULL * 1024ULL, diff --git a/core/runtime/byok/byok.cpp b/core/runtime/byok/byok.cpp index 1277c06bcb..fe0fcd033e 100644 --- a/core/runtime/byok/byok.cpp +++ b/core/runtime/byok/byok.cpp @@ -121,6 +121,8 @@ void load_kernel(const std::string& library, const std::string& function, } // namespace trtmc +TRTMC_DEFINE_PLUGIN_DESCRIPTOR_V1(::trtmc::PluginKind::kRuntimeExtension, "tvm_ffi") + extern "C" const char* trtmc_load_byok_kernel(const char* library, const char* function, const char* kernel_name) noexcept { static thread_local std::string error; diff --git a/core/runtime/include/trtmc/byok.h b/core/runtime/include/trtmc/byok.h index 8ac679800e..63c8f60fd0 100644 --- a/core/runtime/include/trtmc/byok.h +++ b/core/runtime/include/trtmc/byok.h @@ -5,6 +5,8 @@ #pragma once +#include "trtmc/runtime/plugin_abi.h" + #include #include diff --git a/core/runtime/include/trtmc/runtime/family_factory.h b/core/runtime/include/trtmc/runtime/family_factory.h index c0927b50b0..1a0d707cd8 100644 --- a/core/runtime/include/trtmc/runtime/family_factory.h +++ b/core/runtime/include/trtmc/runtime/family_factory.h @@ -6,6 +6,7 @@ #pragma once #include "trtmc/bundle.h" +#include "trtmc/runtime/plugin_abi.h" #include "trtmc/task.h" #include @@ -27,3 +28,6 @@ inline constexpr const char* kCreateFamilySymbol = "trtmc_create_family"; } // namespace trtmc extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context); + +#define TRTMC_DEFINE_FAMILY_PLUGIN_V1(family_id) \ + TRTMC_DEFINE_PLUGIN_DESCRIPTOR_V1(::trtmc::PluginKind::kFamily, family_id) diff --git a/core/runtime/include/trtmc/runtime/family_loader.h b/core/runtime/include/trtmc/runtime/family_loader.h index 78ccba3f3b..5f044a2b81 100644 --- a/core/runtime/include/trtmc/runtime/family_loader.h +++ b/core/runtime/include/trtmc/runtime/family_loader.h @@ -23,4 +23,9 @@ std::unique_ptr load_task(const std::string& bundle_path, const std::stri const std::string& runtime_cache_path = {}, bool cuda_graphs = false); +// Load the exact-build-checked TVM-FFI runtime extension from runtime_root, then +// publish one BYOK kernel. The extension remains resident for process lifetime. +void load_byok_kernel_from_runtime(const std::string& runtime_root, const std::string& library, + const std::string& function, const std::string& kernel_name); + } // namespace trtmc diff --git a/core/runtime/include/trtmc/runtime/plugin_abi.h b/core/runtime/include/trtmc/runtime/plugin_abi.h new file mode 100644 index 0000000000..a66f20f4f1 --- /dev/null +++ b/core/runtime/include/trtmc/runtime/plugin_abi.h @@ -0,0 +1,53 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +namespace trtmc { + +inline constexpr std::uint32_t kPluginDescriptorVersion = 1; +inline constexpr const char* kPluginDescriptorSymbol = "trtmc_plugin_descriptor_v1"; + +#ifdef TRTMC_BUILD_ID +inline constexpr const char* kPluginBuildId = TRTMC_BUILD_ID; +#else +// Installed headers remain consumable, but plugins built outside a coordinated +// TRTMC product build cannot accidentally match a configured runtime. +inline constexpr const char* kPluginBuildId = "unconfigured"; +#endif + +enum class PluginKind : std::uint32_t { + kBackend = 1, + kFamily = 2, + kRuntimeExtension = 3, +}; + +// Model-agnostic identity returned by every backend, family, and extension DSO. +// V1 is an exact descriptor layout, not C++ ABI compatibility negotiation. +struct PluginDescriptorV1 { + std::uint32_t struct_size; + std::uint32_t descriptor_version; + PluginKind kind; + const char* id; + const char* build_id; +}; + +using PluginDescriptorFn = const PluginDescriptorV1* (*)() noexcept; + +} // namespace trtmc + +extern "C" const trtmc::PluginDescriptorV1* trtmc_plugin_descriptor_v1() noexcept; +extern "C" const char* trtmc_core_build_id() noexcept; +extern "C" const char* trtmc_runtime_build_id() noexcept; + +#define TRTMC_DEFINE_PLUGIN_DESCRIPTOR_V1(plugin_kind, plugin_id) \ + extern "C" const trtmc::PluginDescriptorV1* trtmc_plugin_descriptor_v1() noexcept { \ + static const trtmc::PluginDescriptorV1 descriptor{ \ + sizeof(trtmc::PluginDescriptorV1), trtmc::kPluginDescriptorVersion, plugin_kind, \ + plugin_id, trtmc::kPluginBuildId}; \ + return &descriptor; \ + } diff --git a/core/runtime/include/trtmc/runtime/runtime_root.h b/core/runtime/include/trtmc/runtime/runtime_root.h index a2329fe9d5..a66212c0db 100644 --- a/core/runtime/include/trtmc/runtime/runtime_root.h +++ b/core/runtime/include/trtmc/runtime/runtime_root.h @@ -16,11 +16,11 @@ namespace trtmc { // only operation that loads a family and backend. std::string loaded_runtime_root(); -// Return whether runtime_root contains one complete build cohort for bundle -// that matches the core and runtime loader already active in this process. -// This function validates one explicit candidate and never searches, loads, or -// falls back to another directory. -bool runtime_root_matches_loaded_build(const BundleInfo& bundle, const std::string& runtime_root, - bool require_byok = false); +// Return whether runtime_root contains the root-local backend and family DSOs +// named by bundle, plus BYOK when requested. This structural check never +// searches, loads, or falls back. Exact product-build and plugin identities are +// validated when the selected DSOs are loaded. +bool runtime_root_contains_bundle(const BundleInfo& bundle, const std::string& runtime_root, + bool require_byok = false); } // namespace trtmc diff --git a/core/runtime/include/trtmc/runtime/trt_backend.h b/core/runtime/include/trtmc/runtime/trt_backend.h index ace743553e..4200a9a49f 100644 --- a/core/runtime/include/trtmc/runtime/trt_backend.h +++ b/core/runtime/include/trtmc/runtime/trt_backend.h @@ -5,6 +5,8 @@ #pragma once +#include "trtmc/runtime/plugin_abi.h" + // IBackend: the narrow interface implemented by the TensorRT backend DSO. #include "trtmc/runtime/trt_module.h" @@ -70,3 +72,6 @@ extern "C" { trtmc::IBackend* trtmc_create_backend(); void trtmc_destroy_backend(trtmc::IBackend* backend); } + +#define TRTMC_DEFINE_BACKEND_PLUGIN_V1(backend_id) \ + TRTMC_DEFINE_PLUGIN_DESCRIPTOR_V1(::trtmc::PluginKind::kBackend, backend_id) diff --git a/core/runtime/loader/family_loader.cpp b/core/runtime/loader/family_loader.cpp index 09fe82ec70..f6d837df8d 100644 --- a/core/runtime/loader/family_loader.cpp +++ b/core/runtime/loader/family_loader.cpp @@ -7,6 +7,7 @@ #include "runtime/bundle/bundle_format.h" #include "trtmc/runtime/family_factory.h" +#include "trtmc/runtime/plugin_abi.h" #include "trtmc/runtime/runtime_root.h" #include "trtmc/runtime/trt_backend.h" @@ -14,9 +15,7 @@ #include #include #include -#include #include -#include #include #include #include @@ -24,6 +23,7 @@ #include #include #include +#include namespace trtmc { @@ -42,85 +42,20 @@ std::string family_library_name(const std::string& family_id) { return "libtrtmc_model_" + family_id + ".so"; } -constexpr std::uint64_t kMaxElfStringTableSize = 16ULL * 1024ULL * 1024ULL; - -bool read_at(std::ifstream& input, std::uintmax_t file_size, std::uint64_t offset, - void* destination, std::size_t size) { - if (offset > file_size || size > file_size - offset) - return false; - input.clear(); - input.seekg(static_cast(offset)); - input.read(static_cast(destination), static_cast(size)); - return input.good(); -} - -bool has_supported_elf_identity(const Elf64_Ehdr& header) { - return header.e_ident[EI_MAG0] == ELFMAG0 && header.e_ident[EI_MAG1] == ELFMAG1 && - header.e_ident[EI_MAG2] == ELFMAG2 && header.e_ident[EI_MAG3] == ELFMAG3 && - header.e_ident[EI_CLASS] == ELFCLASS64 && header.e_ident[EI_DATA] == ELFDATA2LSB; -} - -bool has_valid_section_table(const Elf64_Ehdr& header, std::uintmax_t file_size) { - return header.e_shentsize == sizeof(Elf64_Shdr) && header.e_shnum != 0 && - header.e_shstrndx < header.e_shnum && header.e_shoff <= file_size && - header.e_shnum <= (file_size - header.e_shoff) / sizeof(Elf64_Shdr); -} - -bool read_section_header(std::ifstream& input, std::uintmax_t file_size, const Elf64_Ehdr& header, - std::size_t index, Elf64_Shdr& section) { - const std::uint64_t offset = header.e_shoff + index * sizeof(Elf64_Shdr); - return read_at(input, file_size, offset, §ion, sizeof(section)); -} - -bool read_string_table(std::ifstream& input, std::uintmax_t file_size, const Elf64_Shdr& section, - std::string& contents) { - if (section.sh_size > kMaxElfStringTableSize) - return false; - contents.assign(static_cast(section.sh_size), '\0'); - return read_at(input, file_size, section.sh_offset, contents.data(), contents.size()); -} - -bool section_has_name(const Elf64_Shdr& section, const std::string& names, const char* expected) { - if (section.sh_name >= names.size()) - return false; - const auto end = names.find('\0', section.sh_name); - return end != std::string::npos && - names.compare(section.sh_name, end - section.sh_name, expected) == 0; +std::string runtime_extension_library_name(const std::string& extension_id) { + return "libtrtmc_byok_" + extension_id + ".so"; } -bool is_lower_hex(unsigned char character) { - return (character >= '0' && character <= '9') || (character >= 'a' && character <= 'f'); -} - -bool read_named_string_table(std::ifstream& input, std::uintmax_t file_size, - const Elf64_Ehdr& header, const std::string& section_names, - const char* expected_name, std::string& contents) { - for (std::size_t index = 0; index < header.e_shnum; ++index) { - Elf64_Shdr section{}; - if (!read_section_header(input, file_size, header, index, section)) - return false; - if (!section_has_name(section, section_names, expected_name)) - continue; - return read_string_table(input, file_size, section, contents); +const char* plugin_kind_name(PluginKind kind) { + switch (kind) { + case PluginKind::kBackend: + return "backend"; + case PluginKind::kFamily: + return "family"; + case PluginKind::kRuntimeExtension: + return "runtime extension"; } - return false; -} - -std::string find_build_cohort(const std::string& strings) { - static constexpr char prefix[] = "trtmc_build_cohort_"; - static constexpr std::size_t id_size = 32; - std::size_t position = strings.find(prefix); - while (position != std::string::npos) { - const std::size_t id_begin = position + sizeof(prefix) - 1; - const std::size_t marker_end = id_begin + id_size; - if (marker_end < strings.size() && strings[marker_end] == '\0' && - std::all_of(strings.begin() + static_cast(id_begin), - strings.begin() + static_cast(marker_end), is_lower_hex)) { - return strings.substr(id_begin, id_size); - } - position = strings.find(prefix, position + 1); - } - return {}; + return "unknown"; } bool is_safe_id(const std::string& value) { @@ -154,35 +89,6 @@ fs::path explicit_runtime_root(const std::string& runtime_root) { return root.lexically_normal(); } -std::string read_build_cohort(const fs::path& library) { - std::ifstream input(library, std::ios::binary); - if (!input) - return {}; - - std::error_code error; - const std::uintmax_t file_size = fs::file_size(library, error); - if (error || file_size < sizeof(Elf64_Ehdr)) - return {}; - - Elf64_Ehdr header{}; - if (!read_at(input, file_size, 0, &header, sizeof(header)) || - !has_supported_elf_identity(header) || !has_valid_section_table(header, file_size)) { - return {}; - } - - Elf64_Shdr names_header{}; - if (!read_section_header(input, file_size, header, header.e_shstrndx, names_header)) - return {}; - std::string names; - if (!read_string_table(input, file_size, names_header, names)) - return {}; - - std::string strings; - if (!read_named_string_table(input, file_size, header, names, ".dynstr", strings)) - return {}; - return find_build_cohort(strings); -} - fs::path loaded_library_path(const void* symbol) { Dl_info info{}; if (dladdr(symbol, &info) == 0 || info.dli_fname == nullptr) @@ -195,36 +101,35 @@ fs::path loaded_library_path(const void* symbol) { return error ? path.lexically_normal() : normalized; } -struct LoadedBuild { - fs::path runtime_library; - std::string cohort_id; -}; +bool contains_root_local_library(const fs::path& root, const std::string& library) { + std::error_code error; + const fs::path resolved = fs::canonical(root / library, error); + return !error && resolved.parent_path() == root && fs::is_regular_file(resolved, error) && + !error; +} + +void require_matching_build(const std::string& path, const PluginDescriptorV1& descriptor) { + if (descriptor.build_id != nullptr && std::string(descriptor.build_id) == kPluginBuildId) + return; + const std::string actual = descriptor.build_id != nullptr ? descriptor.build_id : ""; + throw std::runtime_error("Library '" + path + "' belongs to product build '" + actual + + "'; active runtime requires '" + kPluginBuildId + "'"); +} -const LoadedBuild& loaded_build() { - static const LoadedBuild build = [] { - using InspectBundleFn = BundleInfo (*)(const std::string&); - using LoadTaskFn = std::unique_ptr (*)(const std::string&, const std::string&, - std::uint64_t, const std::string&, bool); - const auto inspect_bundle_function = static_cast(&InspectBundle); - const auto load_task_function = static_cast(&load_task); - const fs::path core_library = - loaded_library_path(reinterpret_cast(inspect_bundle_function)); - const fs::path runtime_library = - loaded_library_path(reinterpret_cast(load_task_function)); - const std::string core_cohort = read_build_cohort(core_library); - const std::string runtime_cohort = read_build_cohort(runtime_library); - return LoadedBuild{runtime_library, !core_cohort.empty() && core_cohort == runtime_cohort - ? core_cohort - : std::string{}}; - }(); - return build; +void require_matching_core_build() { + const char* core_build_id = trtmc_core_build_id(); + if (core_build_id != nullptr && std::string(core_build_id) == kPluginBuildId) + return; + const std::string actual = core_build_id != nullptr ? core_build_id : ""; + throw std::runtime_error("Active libtrtmc_core.so belongs to product build '" + actual + + "'; libtrtmc_runtime.so requires '" + kPluginBuildId + "'"); } class SharedLibrary { public: explicit SharedLibrary(const fs::path& path) : path_(path.string()) { dlerror(); - handle_ = dlopen(path_.c_str(), RTLD_NOW | RTLD_LOCAL); + handle_ = dlopen(path_.c_str(), RTLD_NOW | RTLD_LOCAL | RTLD_NODELETE); if (handle_ == nullptr) { const char* error = dlerror(); throw std::runtime_error("Unable to load '" + path_ + @@ -251,6 +156,37 @@ class SharedLibrary { return symbol; } + void require_plugin(PluginKind expected_kind, const std::string& expected_id) const { + const auto descriptor_function = + reinterpret_cast(require_symbol(kPluginDescriptorSymbol)); + const PluginDescriptorV1* descriptor = descriptor_function(); + if (descriptor == nullptr) { + throw std::runtime_error("Library '" + path_ + "' returned a null plugin descriptor"); + } + if (descriptor->struct_size != sizeof(PluginDescriptorV1)) { + throw std::runtime_error("Library '" + path_ + "' declares descriptor size " + + std::to_string(descriptor->struct_size) + "; expected " + + std::to_string(sizeof(PluginDescriptorV1))); + } + if (descriptor->descriptor_version != kPluginDescriptorVersion) { + throw std::runtime_error("Library '" + path_ + "' declares plugin descriptor version " + + std::to_string(descriptor->descriptor_version) + + "; expected " + std::to_string(kPluginDescriptorVersion)); + } + if (descriptor->kind != expected_kind) { + throw std::runtime_error("Library '" + path_ + "' declares a " + + plugin_kind_name(descriptor->kind) + " plugin; expected " + + plugin_kind_name(expected_kind)); + } + if (descriptor->id == nullptr || expected_id != descriptor->id) { + const std::string actual = descriptor->id != nullptr ? descriptor->id : ""; + throw std::runtime_error("Library '" + path_ + "' declares " + + plugin_kind_name(expected_kind) + " '" + actual + + "'; expected '" + expected_id + "'"); + } + require_matching_build(path_, *descriptor); + } + private: std::string path_; void* handle_{nullptr}; @@ -260,6 +196,7 @@ class BackendLibrary { public: BackendLibrary(const fs::path& runtime_root, const std::string& backend_id) : library_(runtime_root / backend_library_name(backend_id)) { + library_.require_plugin(PluginKind::kBackend, backend_id); const auto create = reinterpret_cast(library_.require_symbol("trtmc_create_backend")); destroy_ = @@ -297,8 +234,10 @@ class BackendLibrary { class FamilyLibrary { public: FamilyLibrary(const fs::path& runtime_root, const std::string& family_id) - : library_(runtime_root / family_library_name(family_id)), - create_(reinterpret_cast(library_.require_symbol(kCreateFamilySymbol))) {} + : library_(runtime_root / family_library_name(family_id)) { + library_.require_plugin(PluginKind::kFamily, family_id); + create_ = reinterpret_cast(library_.require_symbol(kCreateFamilySymbol)); + } FamilyLibrary(const FamilyLibrary&) = delete; FamilyLibrary& operator=(const FamilyLibrary&) = delete; @@ -310,6 +249,27 @@ class FamilyLibrary { CreateFamilyFn create_{nullptr}; }; +class RuntimeExtensionLibrary { + public: + explicit RuntimeExtensionLibrary(const fs::path& runtime_root) + : library_(runtime_root / runtime_extension_library_name("tvm_ffi")) { + library_.require_plugin(PluginKind::kRuntimeExtension, "tvm_ffi"); + load_ = reinterpret_cast(library_.require_symbol("trtmc_load_byok_kernel")); + } + + void load(const std::string& library, const std::string& function, + const std::string& kernel_name) const { + if (const char* error = load_(library.c_str(), function.c_str(), kernel_name.c_str())) + throw std::runtime_error(error); + } + + private: + using LoadKernelFn = const char* (*)(const char*, const char*, const char*) noexcept; + + SharedLibrary library_; + LoadKernelFn load_{nullptr}; +}; + class RuntimeOptionsBackend final : public IBackend { public: RuntimeOptionsBackend(IBackend& backend, std::string runtime_cache_path, bool cuda_graphs) @@ -375,6 +335,7 @@ struct RuntimeLibraryCache { std::mutex mutex; std::unordered_map> backends; std::unordered_map> families; + std::unordered_map> extensions; std::unordered_map, ConfiguredBackendKeyHash> configured_backends; @@ -432,6 +393,20 @@ FamilyLibrary& cached_family(const fs::path& runtime_root, const std::string& fa return family; } +RuntimeExtensionLibrary& cached_runtime_extension(const fs::path& runtime_root) { + const std::string path = (runtime_root / runtime_extension_library_name("tvm_ffi")).string(); + auto& cache = runtime_library_cache(); + std::lock_guard lock(cache.mutex); + const auto found = cache.extensions.find(path); + if (found != cache.extensions.end()) + return *found->second; + + auto library = std::make_unique(runtime_root); + RuntimeExtensionLibrary& extension = *library; + cache.extensions.emplace(path, std::move(library)); + return extension; +} + void require_matching_task(const BundleInfo& info, const ITask& task) { const char* actual = task.task(); if (actual == nullptr || info.task != actual) { @@ -445,45 +420,53 @@ void require_matching_task(const BundleInfo& info, const ITask& task) { } // namespace std::string loaded_runtime_root() { - const fs::path& runtime_library = loaded_build().runtime_library; + using LoadTaskFn = std::unique_ptr (*)(const std::string&, const std::string&, + std::uint64_t, const std::string&, bool); + const auto load_task_function = static_cast(&load_task); + const fs::path runtime_library = + loaded_library_path(reinterpret_cast(load_task_function)); if (runtime_library.empty()) throw std::runtime_error("Unable to locate the active TRTMC runtime loader"); return runtime_library.parent_path().string(); } -bool runtime_root_matches_loaded_build(const BundleInfo& bundle, const std::string& runtime_root, - bool require_byok) { +bool runtime_root_contains_bundle(const BundleInfo& bundle, const std::string& runtime_root, + bool require_byok) { + require_matching_core_build(); require_safe_id("family", bundle.family); require_safe_id("backend", bundle.backend); - const std::string& cohort_id = loaded_build().cohort_id; - if (runtime_root.empty() || cohort_id.empty()) + if (runtime_root.empty()) return false; std::error_code error; - fs::path root = fs::absolute(runtime_root, error); - if (error) + fs::path root = fs::canonical(fs::absolute(runtime_root, error), error); + if (error || !fs::is_directory(root, error) || error) return false; - root = root.lexically_normal(); std::vector required{ - "libtrtmc_core.so", - "libtrtmc_runtime.so", backend_library_name(bundle.backend), family_library_name(bundle.family), }; if (require_byok) - required.emplace_back("libtrtmc_byok_tvm_ffi.so"); + required.emplace_back(runtime_extension_library_name("tvm_ffi")); return std::all_of(required.begin(), required.end(), [&](const std::string& library) { - std::error_code library_error; - const fs::path path = root / library; - return fs::is_regular_file(path, library_error) && read_build_cohort(path) == cohort_id; + return contains_root_local_library(root, library); }); } +void load_byok_kernel_from_runtime(const std::string& runtime_root, const std::string& library, + const std::string& function, const std::string& kernel_name) { + require_matching_core_build(); + RuntimeExtensionLibrary& extension = + cached_runtime_extension(explicit_runtime_root(runtime_root)); + extension.load(library, function, kernel_name); +} + std::unique_ptr load_task(const std::string& bundle_path, const std::string& runtime_root, std::uint64_t kv_cache_size_bytes, const std::string& runtime_cache_path, bool cuda_graphs) { + require_matching_core_build(); const BundleReader reader(bundle_path); const BundleInfo& info = reader.info(); require_safe_id("family", info.family); @@ -508,3 +491,7 @@ std::unique_ptr load_task(const std::string& bundle_path, const std::stri } } // namespace trtmc + +extern "C" const char* trtmc_runtime_build_id() noexcept { + return trtmc::kPluginBuildId; +} diff --git a/core/runtime/primitives/build_identity.cpp b/core/runtime/primitives/build_identity.cpp new file mode 100644 index 0000000000..52f9308e9d --- /dev/null +++ b/core/runtime/primitives/build_identity.cpp @@ -0,0 +1,14 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "trtmc/runtime/plugin_abi.h" + +extern "C" const char* trtmc_core_build_id() noexcept { +#ifdef TRTMC_FAKE_INCOMPATIBLE_BUILD + return "00000000000000000000000000000000"; +#else + return trtmc::kPluginBuildId; +#endif +} diff --git a/core/runtime/tensorrt/rtx_backend.cpp b/core/runtime/tensorrt/rtx_backend.cpp index 5bb004cd14..ec0a41bd02 100644 --- a/core/runtime/tensorrt/rtx_backend.cpp +++ b/core/runtime/tensorrt/rtx_backend.cpp @@ -217,6 +217,8 @@ class RtxBackend final : public IBackend { } // namespace trtmc +TRTMC_DEFINE_BACKEND_PLUGIN_V1("trt_rtx") + extern "C" trtmc::IBackend* trtmc_create_backend() { try { return new trtmc::RtxBackend(); diff --git a/core/runtime/tensorrt/trt_backend.cpp b/core/runtime/tensorrt/trt_backend.cpp index caf20f03a0..6259602e7f 100644 --- a/core/runtime/tensorrt/trt_backend.cpp +++ b/core/runtime/tensorrt/trt_backend.cpp @@ -143,6 +143,8 @@ class TrtBackend final : public IBackend { } // namespace trtmc +TRTMC_DEFINE_BACKEND_PLUGIN_V1("trt") + extern "C" trtmc::IBackend* trtmc_create_backend() { try { return new trtmc::TrtBackend(); diff --git a/core/runtime/tests/fake_backend.cpp b/core/runtime/tests/fake_backend.cpp index f97107743f..f2750b25e8 100644 --- a/core/runtime/tests/fake_backend.cpp +++ b/core/runtime/tests/fake_backend.cpp @@ -47,6 +47,17 @@ class FakeBackend final : public trtmc::IBackend { } // namespace +#ifdef TRTMC_FAKE_INCOMPATIBLE_BUILD +extern "C" const trtmc::PluginDescriptorV1* trtmc_plugin_descriptor_v1() noexcept { + static const trtmc::PluginDescriptorV1 descriptor{ + sizeof(trtmc::PluginDescriptorV1), trtmc::kPluginDescriptorVersion, + trtmc::PluginKind::kBackend, TRTMC_FAKE_BACKEND_NAME, "00000000000000000000000000000000"}; + return &descriptor; +} +#else +TRTMC_DEFINE_BACKEND_PLUGIN_V1(TRTMC_FAKE_BACKEND_NAME) +#endif + extern "C" trtmc::IBackend* trtmc_create_backend() { ++create_count; if (create_count != 1) diff --git a/core/runtime/tests/fake_family.cpp b/core/runtime/tests/fake_family.cpp index 289084c219..0c5e60d13e 100644 --- a/core/runtime/tests/fake_family.cpp +++ b/core/runtime/tests/fake_family.cpp @@ -44,6 +44,8 @@ class FakeForecast final : public trtmc::ITimeSeriesForecast { } // namespace +TRTMC_DEFINE_FAMILY_PLUGIN_V1("fake") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.reader.info().family != "fake") throw std::runtime_error("unexpected family"); diff --git a/core/runtime/tests/test_family_loader.cpp b/core/runtime/tests/test_family_loader.cpp index 1f77d55db2..c2fb2a24eb 100644 --- a/core/runtime/tests/test_family_loader.cpp +++ b/core/runtime/tests/test_family_loader.cpp @@ -42,15 +42,19 @@ void write_bundle(const std::filesystem::path& path, const std::string& family, output.write("{}PLAN", 6); } -bool load_throws(const std::filesystem::path& bundle, const std::string& runtime_root) { +std::string load_error(const std::filesystem::path& bundle, const std::string& runtime_root) { try { (void)trtmc::load_task(bundle.string(), runtime_root); - return false; - } catch (const std::exception&) { - return true; + return {}; + } catch (const std::exception& error) { + return error.what(); } } +bool load_throws(const std::filesystem::path& bundle, const std::string& runtime_root) { + return !load_error(bundle, runtime_root).empty(); +} + bool rtx_options_throw(const std::filesystem::path& bundle, const std::string& runtime_root) { try { (void)trtmc::load_task(bundle.string(), runtime_root, 0, "runtime.cache", true); @@ -87,14 +91,24 @@ void check_rtx_options(const std::filesystem::path& runtime_root, } // namespace int main(int argc, char** argv) { - if (argc != 2) { - std::cerr << "usage: test_family_loader RUNTIME_ROOT\n"; + if (argc != 2 && argc != 3) { + std::cerr << "usage: test_family_loader RUNTIME_ROOT [--expect-core-mismatch]\n"; return 2; } const std::filesystem::path runtime_root(argv[1]); const auto bundle_path = runtime_root / "fake.bundle"; write_bundle(bundle_path, "fake"); + if (argc == 3) { + const std::string core_error = load_error(bundle_path, runtime_root.string()); + check(std::string(argv[2]) == "--expect-core-mismatch" && + core_error.find("Active libtrtmc_core.so belongs to product build") != + std::string::npos, + "runtime rejects a core DSO from a different product build"); + std::filesystem::remove(bundle_path); + return failures; + } + auto task = trtmc::load_task(bundle_path.string(), runtime_root.string()); auto* forecast = dynamic_cast(task.get()); check(forecast != nullptr, "load returns forecast interface"); @@ -147,6 +161,35 @@ int main(int argc, char** argv) { check(load_throws(bundle_path, (runtime_root / "missing").string()), "loader does not search outside explicit root"); + const auto wrong_backend_library = runtime_root / "libtrtmc_backend_other.so"; + const auto wrong_backend_bundle = runtime_root / "wrong-backend.bundle"; + std::filesystem::copy_file(runtime_root / "libtrtmc_backend_fake.so", wrong_backend_library, + std::filesystem::copy_options::overwrite_existing); + write_bundle(wrong_backend_bundle, "fake", "time_series_forecast", "other"); + const std::string backend_descriptor_error = + load_error(wrong_backend_bundle, runtime_root.string()); + check(backend_descriptor_error.find("declares backend 'fake'; expected 'other'") != + std::string::npos, + "explicit root validates backend plugin identity before its factory"); + + const auto incompatible_bundle = runtime_root / "incompatible.bundle"; + write_bundle(incompatible_bundle, "fake", "time_series_forecast", "incompatible"); + const std::string incompatible_build_error = + load_error(incompatible_bundle, runtime_root.string()); + check(incompatible_build_error.find("belongs to product build") != std::string::npos, + "loader rejects a plugin from a different product build before its factory"); + + const auto wrong_family_library = runtime_root / "libtrtmc_model_other.so"; + const auto wrong_family_bundle = runtime_root / "wrong-family.bundle"; + std::filesystem::copy_file(runtime_root / "libtrtmc_model_fake.so", wrong_family_library, + std::filesystem::copy_options::overwrite_existing); + write_bundle(wrong_family_bundle, "other"); + const std::string family_descriptor_error = + load_error(wrong_family_bundle, runtime_root.string()); + check(family_descriptor_error.find("declares family 'fake'; expected 'other'") != + std::string::npos, + "explicit root validates family plugin identity before its factory"); + const auto unsafe_bundle = runtime_root / "unsafe.bundle"; write_bundle(unsafe_bundle, "../fake"); check(load_throws(unsafe_bundle, runtime_root.string()), "unsafe family id rejected"); @@ -160,6 +203,11 @@ int main(int argc, char** argv) { std::filesystem::remove(unsafe_bundle); std::filesystem::remove(mismatch_bundle); std::filesystem::remove(rtx_bundle); + std::filesystem::remove(wrong_backend_bundle); + std::filesystem::remove(wrong_backend_library); + std::filesystem::remove(incompatible_bundle); + std::filesystem::remove(wrong_family_bundle); + std::filesystem::remove(wrong_family_library); std::cerr << (failures == 0 ? "ALL PASSED\n" : "SOME FAILED\n"); return failures; } diff --git a/core/runtime/tests/test_runtime_root.cpp b/core/runtime/tests/test_runtime_root.cpp index 492cc9c137..0bf7060c6c 100644 --- a/core/runtime/tests/test_runtime_root.cpp +++ b/core/runtime/tests/test_runtime_root.cpp @@ -6,9 +6,7 @@ #include "trtmc/runtime/runtime_root.h" #include -#include #include -#include #include #include @@ -29,37 +27,21 @@ void copy_library(const fs::path& source, const fs::path& root, const char* name fs::copy_file(source, root / name, fs::copy_options::overwrite_existing); } -void change_build_cohort(const fs::path& library) { - static constexpr char marker[] = "trtmc_build_cohort_"; - std::ifstream input(library, std::ios::binary); - std::string contents{std::istreambuf_iterator(input), std::istreambuf_iterator()}; - const std::size_t marker_position = contents.find(marker); - if (marker_position == std::string::npos) - throw std::runtime_error("test library has no build-cohort marker"); - const std::size_t id_position = marker_position + sizeof(marker) - 1; - contents[id_position] = contents[id_position] == '0' ? '1' : '0'; - std::ofstream output(library, std::ios::binary | std::ios::trunc); - output.write(contents.data(), static_cast(contents.size())); -} - } // namespace int main(int argc, char** argv) { - if (argc != 6) { - std::cerr << "usage: test_runtime_root ROOT CORE RUNTIME BACKEND FAMILY\n"; + if (argc != 5) { + std::cerr << "usage: test_runtime_root ROOT RUNTIME BACKEND FAMILY\n"; return 2; } const fs::path root = argv[1]; - const fs::path core_library = argv[2]; - const fs::path runtime_library = argv[3]; - const fs::path backend_library = argv[4]; - const fs::path family_library = argv[5]; + const fs::path runtime_library = argv[2]; + const fs::path backend_library = argv[3]; + const fs::path family_library = argv[4]; fs::remove_all(root); fs::create_directories(root); - copy_library(core_library, root, "libtrtmc_core.so"); - copy_library(runtime_library, root, "libtrtmc_runtime.so"); copy_library(backend_library, root, "libtrtmc_backend_fake.so"); copy_library(family_library, root, "libtrtmc_model_fake.so"); const trtmc::BundleInfo bundle{1, "fake", "time_series_forecast", "fake", {}}; @@ -67,38 +49,29 @@ int main(int argc, char** argv) { std::error_code error; check(fs::equivalent(trtmc::loaded_runtime_root(), runtime_library.parent_path(), error), "runtime loader reports its active installation root"); - check(trtmc::runtime_root_matches_loaded_build(bundle, root.string()), - "loader contract accepts one complete matching build cohort"); + check(trtmc::runtime_root_contains_bundle(bundle, root.string()), + "loader contract accepts one complete plugin root"); fs::remove(root / "libtrtmc_backend_fake.so"); - check(!trtmc::runtime_root_matches_loaded_build(bundle, root.string()), + check(!trtmc::runtime_root_contains_bundle(bundle, root.string()), "loader contract rejects an incomplete root"); copy_library(backend_library, root, "libtrtmc_backend_fake.so"); - check(!trtmc::runtime_root_matches_loaded_build(bundle, root.string(), true), + check(!trtmc::runtime_root_contains_bundle(bundle, root.string(), true), "loader contract requires the BYOK DSO when requested"); - copy_library(core_library, root, "libtrtmc_byok_tvm_ffi.so"); - check(trtmc::runtime_root_matches_loaded_build(bundle, root.string(), true), - "loader contract accepts a matching BYOK DSO"); + copy_library(runtime_library, root, "libtrtmc_byok_tvm_ffi.so"); + check(trtmc::runtime_root_contains_bundle(bundle, root.string(), true), + "loader contract accepts a root-local BYOK DSO"); - change_build_cohort(root / "libtrtmc_model_fake.so"); - check(!trtmc::runtime_root_matches_loaded_build(bundle, root.string()), - "loader contract rejects a family from another build cohort"); - copy_library(family_library, root, "libtrtmc_model_fake.so"); - - { - std::ofstream malformed(root / "libtrtmc_model_fake.so", - std::ios::binary | std::ios::trunc); - malformed << "not an ELF file"; - } - fs::resize_file(root / "libtrtmc_model_fake.so", 64ULL * 1024ULL * 1024ULL); - check(!trtmc::runtime_root_matches_loaded_build(bundle, root.string()), - "loader contract rejects a large malformed candidate with bounded reads"); + fs::remove(root / "libtrtmc_model_fake.so"); + fs::create_symlink(family_library, root / "libtrtmc_model_fake.so"); + check(!trtmc::runtime_root_contains_bundle(bundle, root.string()), + "loader contract rejects a DSO symlink that escapes the selected root"); bool unsafe_rejected = false; try { const trtmc::BundleInfo unsafe{1, "../fake", "time_series_forecast", "fake", {}}; - (void)trtmc::runtime_root_matches_loaded_build(unsafe, root.string()); + (void)trtmc::runtime_root_contains_bundle(unsafe, root.string()); } catch (const std::runtime_error&) { unsafe_rejected = true; } diff --git a/families/albert/runtime/plugin.cpp b/families/albert/runtime/plugin.cpp index 70548ff353..df98ccb9e0 100644 --- a/families/albert/runtime/plugin.cpp +++ b/families/albert/runtime/plugin.cpp @@ -55,6 +55,8 @@ std::string require_task(const BundleInfo& info) { } // namespace } // namespace trtmc::albert_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("albert") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("albert does not support --kv-cache-size"); diff --git a/families/bark/runtime/plugin.cpp b/families/bark/runtime/plugin.cpp index 02d489d468..2f3414a86b 100644 --- a/families/bark/runtime/plugin.cpp +++ b/families/bark/runtime/plugin.cpp @@ -70,6 +70,8 @@ BarkConfig parse_config(const nlohmann::json& json) { } // namespace } // namespace trtmc::bark_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("bark") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("bark does not support --kv-cache-size"); diff --git a/families/bart/runtime/plugin.cpp b/families/bart/runtime/plugin.cpp index c0639033c0..7946d359dd 100644 --- a/families/bart/runtime/plugin.cpp +++ b/families/bart/runtime/plugin.cpp @@ -358,6 +358,8 @@ ITask* create_bart(const FamilyContext& context) { } // namespace trtmc +TRTMC_DEFINE_FAMILY_PLUGIN_V1("bart") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("bart does not support --kv-cache-size"); diff --git a/families/bert/runtime/plugin.cpp b/families/bert/runtime/plugin.cpp index 4d20f2374a..17751f7224 100644 --- a/families/bert/runtime/plugin.cpp +++ b/families/bert/runtime/plugin.cpp @@ -55,6 +55,8 @@ std::string require_task(const BundleInfo& info) { } // namespace } // namespace trtmc::bert_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("bert") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("bert does not support --kv-cache-size"); diff --git a/families/bloom/runtime/plugin.cpp b/families/bloom/runtime/plugin.cpp index 49cbc07eee..79acca59ff 100644 --- a/families/bloom/runtime/plugin.cpp +++ b/families/bloom/runtime/plugin.cpp @@ -208,6 +208,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::bloom +TRTMC_DEFINE_FAMILY_PLUGIN_V1("bloom") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("bloom does not support --kv-cache-size"); diff --git a/families/canary/runtime/plugin.cpp b/families/canary/runtime/plugin.cpp index 85cad3b8dc..5ccf8fe8ca 100644 --- a/families/canary/runtime/plugin.cpp +++ b/families/canary/runtime/plugin.cpp @@ -28,6 +28,8 @@ std::vector require_section(const BundleReader& bundle, const char* name) } // namespace } // namespace trtmc::canary_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("canary") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("canary does not support --kv-cache-size"); diff --git a/families/chronos_bolt/runtime/plugin.cpp b/families/chronos_bolt/runtime/plugin.cpp index d6def3a036..428de4d21d 100644 --- a/families/chronos_bolt/runtime/plugin.cpp +++ b/families/chronos_bolt/runtime/plugin.cpp @@ -69,6 +69,8 @@ std::unique_ptr load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_module(IBackend& backend, const std::vector require_section(const BundleReader& bundle, const char* name) } // namespace } // namespace trtmc::magpie_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("magpie_tts") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("magpie_tts does not support --kv-cache-size"); diff --git a/families/mamba/runtime/plugin.cpp b/families/mamba/runtime/plugin.cpp index 1fafc43e39..178e05f81b 100644 --- a/families/mamba/runtime/plugin.cpp +++ b/families/mamba/runtime/plugin.cpp @@ -119,6 +119,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::mamba +TRTMC_DEFINE_FAMILY_PLUGIN_V1("mamba") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("mamba does not support --kv-cache-size"); diff --git a/families/marian/runtime/plugin.cpp b/families/marian/runtime/plugin.cpp index e2b75c30e6..f5cef4d54c 100644 --- a/families/marian/runtime/plugin.cpp +++ b/families/marian/runtime/plugin.cpp @@ -411,6 +411,8 @@ ITask* create_marian(const FamilyContext& context) { } // namespace trtmc +TRTMC_DEFINE_FAMILY_PLUGIN_V1("marian") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("marian does not support --kv-cache-size"); diff --git a/families/minimax_h3/runtime/plugin.cpp b/families/minimax_h3/runtime/plugin.cpp index bef6a2fc4e..3724a44a8f 100644 --- a/families/minimax_h3/runtime/plugin.cpp +++ b/families/minimax_h3/runtime/plugin.cpp @@ -71,6 +71,8 @@ MiniMaxH3ModuleLoader make_loader(IBackend& backend, PlanMap plans) { } // namespace } // namespace trtmc::minimax_h3_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("minimax_h3") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("minimax_h3 does not support --kv-cache-size"); diff --git a/families/mistral/runtime/plugin.cpp b/families/mistral/runtime/plugin.cpp index a9598bd6d5..c6e2e7d89d 100644 --- a/families/mistral/runtime/plugin.cpp +++ b/families/mistral/runtime/plugin.cpp @@ -171,6 +171,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::mistral +TRTMC_DEFINE_FAMILY_PLUGIN_V1("mistral") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("mistral does not support --kv-cache-size"); diff --git a/families/mixtral/runtime/plugin.cpp b/families/mixtral/runtime/plugin.cpp index 0a0ff108e4..16946dd6cb 100644 --- a/families/mixtral/runtime/plugin.cpp +++ b/families/mixtral/runtime/plugin.cpp @@ -214,6 +214,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::mixtral +TRTMC_DEFINE_FAMILY_PLUGIN_V1("mixtral") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("mixtral does not support --kv-cache-size"); diff --git a/families/modernbert/runtime/plugin.cpp b/families/modernbert/runtime/plugin.cpp index d221cb2be3..68bc1d7b5e 100644 --- a/families/modernbert/runtime/plugin.cpp +++ b/families/modernbert/runtime/plugin.cpp @@ -55,6 +55,8 @@ std::string require_task(const BundleInfo& info) { } // namespace } // namespace trtmc::modernbert_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("modernbert") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("modernbert does not support --kv-cache-size"); diff --git a/families/moge/runtime/plugin.cpp b/families/moge/runtime/plugin.cpp index 946489a569..5ba09e207e 100644 --- a/families/moge/runtime/plugin.cpp +++ b/families/moge/runtime/plugin.cpp @@ -40,6 +40,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::moge +TRTMC_DEFINE_FAMILY_PLUGIN_V1("moge") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("moge does not support --kv-cache-size"); diff --git a/families/mpnet/runtime/plugin.cpp b/families/mpnet/runtime/plugin.cpp index 6fb5b30048..6066325951 100644 --- a/families/mpnet/runtime/plugin.cpp +++ b/families/mpnet/runtime/plugin.cpp @@ -55,6 +55,8 @@ std::string require_task(const BundleInfo& info) { } // namespace } // namespace trtmc::mpnet_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("mpnet") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("mpnet does not support --kv-cache-size"); diff --git a/families/nemotron/runtime/plugin.cpp b/families/nemotron/runtime/plugin.cpp index 436cd825e5..ce9cc0a6a8 100644 --- a/families/nemotron/runtime/plugin.cpp +++ b/families/nemotron/runtime/plugin.cpp @@ -171,6 +171,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::nemotron +TRTMC_DEFINE_FAMILY_PLUGIN_V1("nemotron") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("nemotron does not support --kv-cache-size"); diff --git a/families/nemotron_h/runtime/plugin.cpp b/families/nemotron_h/runtime/plugin.cpp index 7ab2faaff9..a986a8e083 100644 --- a/families/nemotron_h/runtime/plugin.cpp +++ b/families/nemotron_h/runtime/plugin.cpp @@ -212,6 +212,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::nemotron_h +TRTMC_DEFINE_FAMILY_PLUGIN_V1("nemotron_h") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("nemotron_h does not support --kv-cache-size"); diff --git a/families/nemotron_labs_diffusion/runtime/plugin.cpp b/families/nemotron_labs_diffusion/runtime/plugin.cpp index 89a0098fbc..a2b451604e 100644 --- a/families/nemotron_labs_diffusion/runtime/plugin.cpp +++ b/families/nemotron_labs_diffusion/runtime/plugin.cpp @@ -187,6 +187,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::nemotron_labs_diffusion +TRTMC_DEFINE_FAMILY_PLUGIN_V1("nemotron_labs_diffusion") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("nemotron_labs_diffusion does not support --kv-cache-size"); diff --git a/families/nemotron_speech_streaming/runtime/plugin.cpp b/families/nemotron_speech_streaming/runtime/plugin.cpp index 74e4119241..057f95119d 100644 --- a/families/nemotron_speech_streaming/runtime/plugin.cpp +++ b/families/nemotron_speech_streaming/runtime/plugin.cpp @@ -70,6 +70,8 @@ RnntConfig parse_config(const nlohmann::json& json) { } // namespace } // namespace trtmc::nemotron_streaming_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("nemotron_speech_streaming") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("nemotron_speech_streaming does not support --kv-cache-size"); diff --git a/families/nemotron_voicechat/runtime/plugin.cpp b/families/nemotron_voicechat/runtime/plugin.cpp index 1a0cb1fd86..44c658f346 100644 --- a/families/nemotron_voicechat/runtime/plugin.cpp +++ b/families/nemotron_voicechat/runtime/plugin.cpp @@ -144,6 +144,8 @@ VoiceChatTtsPrompt load_tts_prompt(const BundleReader& bundle, } // namespace } // namespace trtmc::voicechat_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("nemotron_voicechat") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("nemotron_voicechat does not support --kv-cache-size"); diff --git a/families/olmo/runtime/plugin.cpp b/families/olmo/runtime/plugin.cpp index 2ce4280186..bf5cfeae1c 100644 --- a/families/olmo/runtime/plugin.cpp +++ b/families/olmo/runtime/plugin.cpp @@ -208,6 +208,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::olmo +TRTMC_DEFINE_FAMILY_PLUGIN_V1("olmo") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("olmo does not support --kv-cache-size"); diff --git a/families/olmo2/runtime/plugin.cpp b/families/olmo2/runtime/plugin.cpp index 813ed600b3..d64b2ffc26 100644 --- a/families/olmo2/runtime/plugin.cpp +++ b/families/olmo2/runtime/plugin.cpp @@ -208,6 +208,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::olmo2 +TRTMC_DEFINE_FAMILY_PLUGIN_V1("olmo2") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("olmo2 does not support --kv-cache-size"); diff --git a/families/opt/runtime/plugin.cpp b/families/opt/runtime/plugin.cpp index 21b4558207..c61bedea2d 100644 --- a/families/opt/runtime/plugin.cpp +++ b/families/opt/runtime/plugin.cpp @@ -208,6 +208,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::opt +TRTMC_DEFINE_FAMILY_PLUGIN_V1("opt") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("opt does not support --kv-cache-size"); diff --git a/families/patchtsmixer/runtime/plugin.cpp b/families/patchtsmixer/runtime/plugin.cpp index 1e7aa3aac9..9f7ccafc0a 100644 --- a/families/patchtsmixer/runtime/plugin.cpp +++ b/families/patchtsmixer/runtime/plugin.cpp @@ -11,6 +11,8 @@ #include #include +TRTMC_DEFINE_FAMILY_PLUGIN_V1("patchtsmixer") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("patchtsmixer does not support --kv-cache-size"); diff --git a/families/patchtst/runtime/plugin.cpp b/families/patchtst/runtime/plugin.cpp index 200fa8035d..e4a44d52f4 100644 --- a/families/patchtst/runtime/plugin.cpp +++ b/families/patchtst/runtime/plugin.cpp @@ -76,6 +76,8 @@ std::unique_ptr load_module(IBackend& backend, const std::vector require_section(const BundleReader& bundle, const char* name) } // namespace } // namespace trtmc::pixart_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("pixart") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("pixart does not support --kv-cache-size"); diff --git a/families/qwen/runtime/plugin.cpp b/families/qwen/runtime/plugin.cpp index fe8d8a30bb..a314c7b977 100644 --- a/families/qwen/runtime/plugin.cpp +++ b/families/qwen/runtime/plugin.cpp @@ -233,6 +233,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::qwen +TRTMC_DEFINE_FAMILY_PLUGIN_V1("qwen") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("qwen does not support --kv-cache-size"); diff --git a/families/qwen3_5/runtime/plugin.cpp b/families/qwen3_5/runtime/plugin.cpp index 7786e34b43..2119cb307c 100644 --- a/families/qwen3_5/runtime/plugin.cpp +++ b/families/qwen3_5/runtime/plugin.cpp @@ -160,6 +160,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::qwen3_5 +TRTMC_DEFINE_FAMILY_PLUGIN_V1("qwen3_5") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("qwen3_5 does not support --kv-cache-size"); diff --git a/families/qwen3_8/runtime/plugin.cpp b/families/qwen3_8/runtime/plugin.cpp index 49a8b4c57f..66349b8629 100644 --- a/families/qwen3_8/runtime/plugin.cpp +++ b/families/qwen3_8/runtime/plugin.cpp @@ -160,6 +160,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::qwen3_8 +TRTMC_DEFINE_FAMILY_PLUGIN_V1("qwen3_8") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("qwen3_8 does not support --kv-cache-size"); diff --git a/families/qwen3_omni/runtime/plugin.cpp b/families/qwen3_omni/runtime/plugin.cpp index 15fd0bb118..b7da0bd2af 100644 --- a/families/qwen3_omni/runtime/plugin.cpp +++ b/families/qwen3_omni/runtime/plugin.cpp @@ -101,6 +101,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::qwen3_omni +TRTMC_DEFINE_FAMILY_PLUGIN_V1("qwen3_omni") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("qwen3_omni does not support --kv-cache-size"); diff --git a/families/qwen_image/runtime/plugin.cpp b/families/qwen_image/runtime/plugin.cpp index 5b3196b08d..a32e856f0a 100644 --- a/families/qwen_image/runtime/plugin.cpp +++ b/families/qwen_image/runtime/plugin.cpp @@ -33,6 +33,8 @@ std::unique_ptr load(IBackend& backend, const BundleReader& bundle, } // namespace } // namespace trtmc::qwen_image_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("qwen_image") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("qwen_image does not support --kv-cache-size"); diff --git a/families/qwen_moe/runtime/plugin.cpp b/families/qwen_moe/runtime/plugin.cpp index 6488b8263f..c35c7090cf 100644 --- a/families/qwen_moe/runtime/plugin.cpp +++ b/families/qwen_moe/runtime/plugin.cpp @@ -216,6 +216,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::qwen_moe +TRTMC_DEFINE_FAMILY_PLUGIN_V1("qwen_moe") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("qwen_moe does not support --kv-cache-size"); diff --git a/families/qwen_vl/runtime/plugin.cpp b/families/qwen_vl/runtime/plugin.cpp index 295189918a..8b1fe23ea4 100644 --- a/families/qwen_vl/runtime/plugin.cpp +++ b/families/qwen_vl/runtime/plugin.cpp @@ -62,6 +62,8 @@ std::vector lora_contract(const ITrtModule& module) { } // namespace } // namespace trtmc::qwen_vl_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("qwen_vl") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("qwen_vl does not support --kv-cache-size"); diff --git a/families/roberta/runtime/plugin.cpp b/families/roberta/runtime/plugin.cpp index 0eec0266b1..2bde4f0923 100644 --- a/families/roberta/runtime/plugin.cpp +++ b/families/roberta/runtime/plugin.cpp @@ -55,6 +55,8 @@ std::string require_task(const BundleInfo& info) { } // namespace } // namespace trtmc::roberta_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("roberta") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("roberta does not support --kv-cache-size"); diff --git a/families/rwkv/runtime/plugin.cpp b/families/rwkv/runtime/plugin.cpp index b231eacf1a..40451ced97 100644 --- a/families/rwkv/runtime/plugin.cpp +++ b/families/rwkv/runtime/plugin.cpp @@ -114,6 +114,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::rwkv +TRTMC_DEFINE_FAMILY_PLUGIN_V1("rwkv") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("rwkv does not support --kv-cache-size"); diff --git a/families/sam/runtime/plugin.cpp b/families/sam/runtime/plugin.cpp index 98adea4216..3903757829 100644 --- a/families/sam/runtime/plugin.cpp +++ b/families/sam/runtime/plugin.cpp @@ -54,6 +54,8 @@ SamConfig parse_config(const std::vector& data, std::int32_t& tp_size) { } // namespace } // namespace trtmc::sam_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("sam") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("sam does not support --kv-cache-size"); diff --git a/families/sam2/runtime/plugin.cpp b/families/sam2/runtime/plugin.cpp index 1aedd5fa2a..ac8215f795 100644 --- a/families/sam2/runtime/plugin.cpp +++ b/families/sam2/runtime/plugin.cpp @@ -40,6 +40,8 @@ NativePlanModuleFactory makeModuleFactory(IBackend& backend) { } // namespace } // namespace trtmc::sam2 +TRTMC_DEFINE_FAMILY_PLUGIN_V1("sam2") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("sam2 does not support --kv-cache-size"); diff --git a/families/sam3/runtime/plugin.cpp b/families/sam3/runtime/plugin.cpp index 4ab33f0428..fe854d4606 100644 --- a/families/sam3/runtime/plugin.cpp +++ b/families/sam3/runtime/plugin.cpp @@ -82,6 +82,8 @@ std::unique_ptr load(IBackend& backend, const BundleReader& bundle, } // namespace } // namespace trtmc::sam3_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("sam3") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("sam3 does not support --kv-cache-size"); diff --git a/families/sana_wm/runtime/plugin.cpp b/families/sana_wm/runtime/plugin.cpp index 3a669a1ec5..79c1ea7261 100644 --- a/families/sana_wm/runtime/plugin.cpp +++ b/families/sana_wm/runtime/plugin.cpp @@ -35,6 +35,8 @@ std::shared_ptr load_tokenizer(const BundleReader& bundle, const cha } // namespace } // namespace trtmc::sana_wm_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("sana_wm") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("sana_wm does not support --kv-cache-size"); diff --git a/families/segformer/runtime/plugin.cpp b/families/segformer/runtime/plugin.cpp index 07dc71fe5b..3f3b9c6b6b 100644 --- a/families/segformer/runtime/plugin.cpp +++ b/families/segformer/runtime/plugin.cpp @@ -46,6 +46,8 @@ SegformerPreprocessConfig parse_config(const std::vector& data, std::int32 } // namespace } // namespace trtmc::segformer +TRTMC_DEFINE_FAMILY_PLUGIN_V1("segformer") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("segformer does not support --kv-cache-size"); diff --git a/families/stablelm/runtime/plugin.cpp b/families/stablelm/runtime/plugin.cpp index d098b83688..2199f858d7 100644 --- a/families/stablelm/runtime/plugin.cpp +++ b/families/stablelm/runtime/plugin.cpp @@ -208,6 +208,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::stablelm +TRTMC_DEFINE_FAMILY_PLUGIN_V1("stablelm") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("stablelm does not support --kv-cache-size"); diff --git a/families/starcoder2/runtime/plugin.cpp b/families/starcoder2/runtime/plugin.cpp index 7762bd3144..80542183d4 100644 --- a/families/starcoder2/runtime/plugin.cpp +++ b/families/starcoder2/runtime/plugin.cpp @@ -208,6 +208,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::starcoder2 +TRTMC_DEFINE_FAMILY_PLUGIN_V1("starcoder2") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("starcoder2 does not support --kv-cache-size"); diff --git a/families/t5/runtime/plugin.cpp b/families/t5/runtime/plugin.cpp index 8e3e617b4d..f2127b2e19 100644 --- a/families/t5/runtime/plugin.cpp +++ b/families/t5/runtime/plugin.cpp @@ -430,6 +430,8 @@ ITask* create_t5(const FamilyContext& context) { } // namespace trtmc +TRTMC_DEFINE_FAMILY_PLUGIN_V1("t5") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("t5 does not support --kv-cache-size"); diff --git a/families/timesfm/runtime/plugin.cpp b/families/timesfm/runtime/plugin.cpp index 23764aadae..c7e8acb5bc 100644 --- a/families/timesfm/runtime/plugin.cpp +++ b/families/timesfm/runtime/plugin.cpp @@ -67,6 +67,8 @@ std::unique_ptr load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_tokenizer(const BundleReader& bundle) { } // namespace } // namespace trtmc::wan22_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("wan2_2_ti2v") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("wan2_2_ti2v does not support --kv-cache-size"); diff --git a/families/wan_t2v/runtime/plugin.cpp b/families/wan_t2v/runtime/plugin.cpp index bcdb7636ad..790a7a469c 100644 --- a/families/wan_t2v/runtime/plugin.cpp +++ b/families/wan_t2v/runtime/plugin.cpp @@ -27,6 +27,8 @@ std::vector require_section(const BundleReader& bundle, const char* name) } // namespace } // namespace trtmc::wan_t2v +TRTMC_DEFINE_FAMILY_PLUGIN_V1("wan_t2v") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("wan_t2v does not support --kv-cache-size"); diff --git a/families/whisper/runtime/plugin.cpp b/families/whisper/runtime/plugin.cpp index 38789139e9..6c0c5d183d 100644 --- a/families/whisper/runtime/plugin.cpp +++ b/families/whisper/runtime/plugin.cpp @@ -27,6 +27,8 @@ std::vector require_section(const BundleReader& bundle, const char* name) } // namespace } // namespace trtmc::whisper_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("whisper") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("whisper does not support --kv-cache-size"); diff --git a/families/xglm/runtime/plugin.cpp b/families/xglm/runtime/plugin.cpp index 3a6c72ecd4..1382b0435c 100644 --- a/families/xglm/runtime/plugin.cpp +++ b/families/xglm/runtime/plugin.cpp @@ -208,6 +208,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::xglm +TRTMC_DEFINE_FAMILY_PLUGIN_V1("xglm") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("xglm does not support --kv-cache-size"); diff --git a/families/xlnet/runtime/plugin.cpp b/families/xlnet/runtime/plugin.cpp index 693cc0c50b..94bc51a575 100644 --- a/families/xlnet/runtime/plugin.cpp +++ b/families/xlnet/runtime/plugin.cpp @@ -55,6 +55,8 @@ std::string require_task(const BundleInfo& info) { } // namespace } // namespace trtmc::xlnet_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("xlnet") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("xlnet does not support --kv-cache-size"); diff --git a/families/z_image/runtime/plugin.cpp b/families/z_image/runtime/plugin.cpp index 6528c4fa1b..12a858c3d8 100644 --- a/families/z_image/runtime/plugin.cpp +++ b/families/z_image/runtime/plugin.cpp @@ -59,6 +59,8 @@ ZImagePreprocessorWeights parse_weights(const std::vector& data) { } // namespace } // namespace trtmc::z_image_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("z_image") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("z_image does not support --kv-cache-size"); diff --git a/tools/ci/package.py b/tools/ci/package.py index dc44f3c43e..6ebd14c29b 100644 --- a/tools/ci/package.py +++ b/tools/ci/package.py @@ -74,11 +74,67 @@ def load_native_libraries(bin_dir: Path, families: tuple[str, ...]) -> None: script = """ import ctypes import os +from pathlib import Path import sys -ctypes.CDLL(sys.argv[1], mode=os.RTLD_NOW | ctypes.RTLD_GLOBAL) -for path in sys.argv[2:]: - ctypes.CDLL(path, mode=os.RTLD_NOW | ctypes.RTLD_LOCAL) +core = ctypes.CDLL(sys.argv[1], mode=os.RTLD_NOW | ctypes.RTLD_GLOBAL) +runtime = ctypes.CDLL(sys.argv[2], mode=os.RTLD_NOW | ctypes.RTLD_LOCAL) + +def build_identity(library, symbol): + function = getattr(library, symbol) + function.restype = ctypes.c_char_p + raw_identity = function() + identity = raw_identity.decode() if raw_identity is not None else "" + if len(identity) != 32 or any(character not in "0123456789abcdef" for character in identity): + raise RuntimeError(f"invalid TRTMC product-build identity from {symbol}: {identity}") + return identity + +core_build_id = build_identity(core, "trtmc_core_build_id") +runtime_build_id = build_identity(runtime, "trtmc_runtime_build_id") +if core_build_id != runtime_build_id: + raise RuntimeError( + f"installed TRTMC core/runtime build mismatch: core={core_build_id} " + f"runtime={runtime_build_id}" + ) + +class PluginDescriptorV1(ctypes.Structure): + _fields_ = [ + ("struct_size", ctypes.c_uint32), + ("descriptor_version", ctypes.c_uint32), + ("kind", ctypes.c_uint32), + ("id", ctypes.c_char_p), + ("build_id", ctypes.c_char_p), + ] + +for raw_path in sys.argv[3:]: + path = Path(raw_path) + library = ctypes.CDLL(path, mode=os.RTLD_NOW | ctypes.RTLD_LOCAL) + descriptor_function = library.trtmc_plugin_descriptor_v1 + descriptor_function.restype = ctypes.POINTER(PluginDescriptorV1) + descriptor = descriptor_function().contents + name = path.name + if name.startswith("libtrtmc_backend_"): + expected_kind, expected_id = 1, name.removeprefix("libtrtmc_backend_").removesuffix(".so") + elif name.startswith("libtrtmc_model_"): + expected_kind, expected_id = 2, name.removeprefix("libtrtmc_model_").removesuffix(".so") + elif name == "libtrtmc_byok_tvm_ffi.so": + expected_kind, expected_id = 3, "tvm_ffi" + else: + raise RuntimeError(f"unknown TRTMC plugin library: {path}") + actual_id = descriptor.id.decode() if descriptor.id is not None else "" + actual_build_id = descriptor.build_id.decode() if descriptor.build_id is not None else "" + if ( + descriptor.struct_size != ctypes.sizeof(PluginDescriptorV1) + or descriptor.descriptor_version != 1 + or descriptor.kind != expected_kind + or actual_id != expected_id + or actual_build_id != runtime_build_id + ): + raise RuntimeError( + f"invalid TRTMC plugin descriptor: {path}: " + f"size={descriptor.struct_size} version={descriptor.descriptor_version} " + f"kind={descriptor.kind} id={actual_id} build={actual_build_id}" + ) """ environment = os.environ.copy() environment["LD_LIBRARY_PATH"] = ":".join( diff --git a/tools/tests/test_architecture.py b/tools/tests/test_architecture.py index 9bdecb34bd..ca2fc53513 100644 --- a/tools/tests/test_architecture.py +++ b/tools/tests/test_architecture.py @@ -293,6 +293,7 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "core/runtime/byok/tvm_ffi_kernel_plugin.h", "core/runtime/primitives/cuda_common.cpp", "core/runtime/primitives/cuda_common.h", + "core/runtime/primitives/build_identity.cpp", "core/runtime/primitives/device_tensor.cpp", "core/runtime/primitives/trt_common.cpp", "core/runtime/primitives/trt_common.h", @@ -303,6 +304,7 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "core/runtime/include/trtmc/runtime/device_tensor.h", "core/runtime/include/trtmc/runtime/family_factory.h", "core/runtime/include/trtmc/runtime/family_loader.h", + "core/runtime/include/trtmc/runtime/plugin_abi.h", "core/runtime/include/trtmc/runtime/runtime_root.h", "core/runtime/include/trtmc/runtime/tensor.h", "core/runtime/include/trtmc/runtime/trt_backend.h", @@ -1123,7 +1125,7 @@ def test_rtx_backend_is_an_explicit_optional_dso() -> None: assert "runtime cache and whole-graph capture require a TensorRT-RTX bundle" in loader -def test_every_runtime_exports_only_the_task_factory_contract() -> None: +def test_every_runtime_exports_task_factory_and_abi_descriptor() -> None: forbidden = ( "IPipeline", "PipelineContext", @@ -1148,6 +1150,8 @@ def test_every_runtime_exports_only_the_task_factory_contract() -> None: source = factory.read_text(encoding="utf-8", errors="ignore") if "trtmc_create_family" not in source or "trtmc::ITask*" not in source: violations.append(f"{family.name}:factory") + if f'TRTMC_DEFINE_FAMILY_PLUGIN_V1("{family.name}")' not in source: + violations.append(f"{family.name}:plugin-descriptor") cmake = (runtime / "CMakeLists.txt").read_text(encoding="utf-8") if f"trtmc_model_{family.name}" not in cmake: violations.append(f"{family.name}:target") @@ -1166,6 +1170,38 @@ def test_every_runtime_exports_only_the_task_factory_contract() -> None: assert violations == [] +def test_runtime_plugins_publish_one_exact_build_descriptor() -> None: + descriptor = (REPO / "core/runtime/include/trtmc/runtime/plugin_abi.h").read_text( + encoding="utf-8" + ) + loader = (REPO / "core/runtime/loader/family_loader.cpp").read_text(encoding="utf-8") + cmake = (REPO / "CMakeLists.txt").read_text(encoding="utf-8") + + assert "struct PluginDescriptorV1" in descriptor + assert 'kPluginDescriptorSymbol = "trtmc_plugin_descriptor_v1"' in descriptor + assert "kPluginBuildId = TRTMC_BUILD_ID" in descriptor + assert "trtmc_core_build_id" in descriptor + assert "trtmc_runtime_build_id" in descriptor + assert "require_plugin(PluginKind expected_kind" in loader + assert "require_matching_core_build();" in loader + assert "active runtime requires" in loader + assert "" not in loader + assert "build_cohort" not in loader + assert "TRTMC_BUILD_COHORT_ID" not in cmake + assert "TRTMC_BUILD_ID" in cmake + + backends = { + "core/runtime/tensorrt/trt_backend.cpp": "trt", + "core/runtime/tensorrt/rtx_backend.cpp": "trt_rtx", + } + for path, backend in backends.items(): + source = (REPO / path).read_text(encoding="utf-8") + assert f'TRTMC_DEFINE_BACKEND_PLUGIN_V1("{backend}")' in source + + byok = (REPO / "core/runtime/byok/byok.cpp").read_text(encoding="utf-8") + assert "PluginKind::kRuntimeExtension" in byok + + def test_family_factory_receives_only_direct_runtime_inputs() -> None: factory_header = (REPO / "core/runtime/include/trtmc/runtime/family_factory.h").read_text( encoding="utf-8" diff --git a/tools/tests/test_new_ci.py b/tools/tests/test_new_ci.py index cab2bc05d3..be484c51ea 100644 --- a/tools/tests/test_new_ci.py +++ b/tools/tests/test_new_ci.py @@ -885,17 +885,57 @@ def compile_library(name: str, source: str) -> None: check=True, ) - compile_library("libtrtmc_core.so", "void core_symbol(void) {}\n") - compile_library("libtrtmc_runtime.so", "void runtime_symbol(void) {}\n") - compile_library("libtrtmc_backend_trt.so", "void backend_symbol(void) {}\n") - compile_library("libtrtmc_byok_tvm_ffi.so", "void byok_symbol(void) {}\n") - compile_library("libtrtmc_model_alpha.so", "void alpha_symbol(void) {}\n") - compile_library("libtrtmc_model_beta.so", "void beta_symbol(void) {}\n") + def plugin_source( + kind: int, + plugin_id: str, + implementation: str, + build_id: str = "1234567890abcdef1234567890abcdef", + ) -> str: + return f""" +#include +struct PluginDescriptorV1 {{ + uint32_t struct_size; + uint32_t descriptor_version; + uint32_t kind; + const char *id; + const char *build_id; +}}; +static const struct PluginDescriptorV1 descriptor = {{ + sizeof(struct PluginDescriptorV1), 1, {kind}, "{plugin_id}", "{build_id}" +}}; +const struct PluginDescriptorV1 *trtmc_plugin_descriptor_v1(void) {{ return &descriptor; }} +{implementation} +""" + + build_id = "1234567890abcdef1234567890abcdef" + compile_library( + "libtrtmc_core.so", + f'const char *trtmc_core_build_id(void) {{ return "{build_id}"; }}\n', + ) + compile_library( + "libtrtmc_runtime.so", + f'const char *trtmc_runtime_build_id(void) {{ return "{build_id}"; }}\n', + ) + compile_library( + "libtrtmc_backend_trt.so", plugin_source(1, "trt", "void backend_symbol(void) {}") + ) + compile_library( + "libtrtmc_byok_tvm_ffi.so", + plugin_source(3, "tvm_ffi", "void byok_symbol(void) {}"), + ) + compile_library( + "libtrtmc_model_alpha.so", plugin_source(2, "alpha", "void alpha_symbol(void) {}") + ) + compile_library("libtrtmc_model_beta.so", plugin_source(2, "beta", "void beta_symbol(void) {}")) load_native_libraries(tmp_path, ("alpha", "beta")) compile_library( "libtrtmc_model_beta.so", - "extern void missing_symbol(void); void beta_symbol(void) { missing_symbol(); }\n", + plugin_source( + 2, + "beta", + "extern void missing_symbol(void); void beta_symbol(void) { missing_symbol(); }", + ), ) with pytest.raises(CiError, match="undefined symbol: missing_symbol"): load_native_libraries(tmp_path, ("alpha", "beta")) diff --git a/website/docs/api/cli-reference.md b/website/docs/api/cli-reference.md index 062f946e57..e02c18913d 100644 --- a/website/docs/api/cli-reference.md +++ b/website/docs/api/cli-reference.md @@ -56,17 +56,19 @@ backend, and section bounds. Family-owned section payloads are not decoded. Every execution command has this shape: ```bash -trtmc COMMAND MODEL.bundle --runtime-root DIR [OPTIONS] +trtmc COMMAND MODEL.bundle [--runtime-root DIR] [OPTIONS] ``` -`--runtime-root` is always required. It must contain the matching -`libtrtmc_core.so`, `libtrtmc_runtime.so`, backend DSO, and selected family DSO. -The CLI never searches the current directory, environment variables, or an -installed fallback. Common load options are: +When `--runtime-root` is omitted, the CLI searches the active runtime and CLI +installation, followed by colon-separated `TRTMC_RUNTIME_PATH` entries. It +does not search the current directory unless `.` is explicitly present in that +variable. One selected root must contain the requested backend and family DSOs; +their descriptors must declare the active product-build identity, expected +kind, and bundle ID before either factory is called. Common load options are: | Option | Contract | | --- | --- | -| `--runtime-root DIR` | Required exact DSO root. | +| `--runtime-root DIR` | Select one exact plugin root without search fallback; exact-build validation still applies. | | `--kv-cache-size BYTES\|GB\|GiB` | Runtime-sized KV capacity for a compatible bundle. | | `--runtime-cache PATH` | TensorRT-RTX cache path; rejected by the standard TensorRT backend. | | `--cuda-graphs` | Enable TensorRT-RTX CUDA graphs; rejected by the standard backend. | @@ -101,7 +103,6 @@ text-diffusion replay inputs, and a paired `--lora-adapter` / ```bash trtmc run qwen.bundle \ - --runtime-root /opt/trtmc/lib \ --prompt "Hello" \ --max-new-tokens 32 \ --temperature 0 \ diff --git a/website/docs/api/overview.md b/website/docs/api/overview.md index 6a6aa7d032..4121d1a73d 100644 --- a/website/docs/api/overview.md +++ b/website/docs/api/overview.md @@ -20,16 +20,16 @@ The build and runtime entry points are intentionally separate. The Python builder resolves exactly one `families//support.py`, imports only that family's `model.py`, and writes a bundle. The native loader reads the bundle's `family`, `task`, and `backend`, then loads exactly one family DSO and one -backend DSO from the explicit runtime root. +backend DSO from one selected plugin root. ```text Hugging Face model ID or local snapshot -> python -m tensorrt_model_connect build -> model.bundle - -> trtmc::load_task() or trtmc TASK --runtime-root DIR + -> trtmc::load_task() with an explicit root, or trtmc TASK with CLI discovery -> task-specific output ``` There is no Python runtime wrapper, central model registry, runtime-strategy -switch, backend search path, or fallback runtime discovery in the current -architecture. +switch, sibling-family probe, or load-time fallback. CLI discovery selects one +root before the Runtime Loader performs an exact load. diff --git a/website/docs/architecture/ai-native-horizontal-scaling.md b/website/docs/architecture/ai-native-horizontal-scaling.md index 130e2728f7..9cfa63c925 100644 --- a/website/docs/architecture/ai-native-horizontal-scaling.md +++ b/website/docs/architecture/ai-native-horizontal-scaling.md @@ -120,7 +120,7 @@ After each transfer, core no longer participates in model behavior. | Component | Owns | Explicitly does not own | | --- | --- | --- | | Native Core (`libtrtmc_core.so`) | bounded bundle reads, device tensors, stable engine primitives | `dlopen`, model config, weight mapping, preprocessing, request loops | -| Runtime Loader (`libtrtmc_runtime.so`) | safe family/backend names, explicit runtime root, runtime-root/build-cohort validation, exact `dlopen`, one control transfer | path search policy, model pipelines, preprocessing, policy dispatch, family fallback | +| Runtime Loader (`libtrtmc_runtime.so`) | safe family/backend names, explicit runtime root, plugin-root validation, exact product-build and plugin identity, exact `dlopen`, one control transfer | path search policy, model pipelines, preprocessing, policy dispatch, family fallback | | Family | checkpoint identity, tasks/default, graph build, weights, section semantics, native pipeline, dispatch, bindings, pre/postprocessing | sibling families, shared model policy | | Bundle | header and named byte sections with bounded streaming I/O | model schema, section semantics, content hashes | | Task API | user behavior such as text, image, audio, embedding, and segmentation | family names, TensorRT objects, backend details | @@ -506,8 +506,8 @@ Runtime dispatch occurs once: 1. The user calls `load(bundle)`. 2. Runtime reads the fixed header and bounded section table. -3. Runtime derives and loads `libtrtmc_model_.so` from the explicit - runtime root. +3. Runtime loads the named backend and family DSOs from the explicit runtime + root and validates their product-build identities, kinds, and IDs. 4. Runtime finds the fixed family-factory symbol and passes BundleReader, Engine API, and the direct runtime KV budget. Backend-only RTX cache and whole-graph options are applied through the Engine API wrapper. @@ -515,24 +515,24 @@ Runtime dispatch occurs once: 6. Runtime returns the abstract Task interface. Later requests call the family pipeline directly. -Core, family, and backend DSOs are produced by one product build. There is no -ABI negotiation, version translation, old-symbol alias, or compatibility shim. -The human-facing `trtmc` CLI may discover a complete runtime cohort before this -control transfer. The CLI owns only candidate enumeration and search order: it -prefers the current directory, then the runtime belonging to the running CLI -installation, followed by explicitly configured runtime library paths. For -each candidate, it asks the model-agnostic Runtime Loader contract to validate -the safe bundle identifiers, required DSO set, and build cohort. The CLI does -not derive DSO names or parse native artifact metadata. The public C++ load API -still receives one explicit root, and the loader never combines or falls back -across roots. - -Every native artifact carries the build-cohort identity generated when CMake -configures the build. Automatic candidates must contain that same identity in -core, runtime, backend, family, and optional BYOK DSOs, preventing another -build cohort from being selected implicitly. This is strict product-build -identity, not ABI compatibility negotiation: there is no compatible-version -selection, translation, or fallback. +Core, runtime, family, backend, and extension DSOs are produced by one product +build. Core and runtime expose the same exact build identity; each plugin +descriptor carries it alongside the plugin kind and ID. The runtime verifies +core before crossing their C++ seam, then verifies every selected plugin. There +is no ABI negotiation, compatible-version selection, translation, old-symbol +alias, or fallback. Source work remains family-local, but native artifacts from +separate product builds are never mixed. + +The human-facing `trtmc` CLI may discover one plugin root before this control +transfer. The CLI owns only candidate enumeration and search order: the active +runtime, the installation belonging to the running executable, then explicitly +configured `TRTMC_RUNTIME_PATH` entries. The current directory participates +only when a user explicitly adds `.` to that variable. For each candidate, the +CLI asks the model-agnostic Runtime Loader contract whether the requested +root-local backend, family, and optional BYOK files exist. It does not derive +DSO names, inspect native metadata, or load discovery candidates. The public +C++ load interface still receives one explicit root, and a selected-root load +failure never triggers another search. ### Task API diff --git a/website/docs/architecture/runtime-lifecycle.md b/website/docs/architecture/runtime-lifecycle.md index b82b68eb40..af41d99f96 100644 --- a/website/docs/architecture/runtime-lifecycle.md +++ b/website/docs/architecture/runtime-lifecycle.md @@ -14,12 +14,16 @@ auto task = trtmc::load_task("model.bundle", "/opt/trtmc/lib"); 2. The loader validates the `family` and `backend` names as safe DSO tokens. 3. It loads `libtrtmc_backend_.so` from the explicit runtime root. 4. It loads `libtrtmc_model_.so` from that same root. -5. It resolves the single `trtmc_create_family` factory and passes a +5. It requires both plugin descriptors to declare the active product build, + expected kind, and identity. +6. It resolves the single `trtmc_create_family` factory and passes a `FamilyContext` containing the read-only bundle reader and abstract backend. -6. It verifies that the returned `ITask::task()` matches the bundle header. +7. It verifies that the backend name and returned `ITask::task()` match the + bundle header. -There is no current-directory search, environment fallback, registry lookup, -strategy switch, sibling-family probe, or load retry. +The Runtime Loader performs no current-directory search, environment fallback, +registry lookup, strategy switch, sibling-family probe, or load retry. The CLI +may select one explicit root before entering this sequence. ## Ownership after transfer diff --git a/website/docs/architecture/runtime-plugins.md b/website/docs/architecture/runtime-plugins.md index 874f8fb874..9bf9cee4bc 100644 --- a/website/docs/architecture/runtime-plugins.md +++ b/website/docs/architecture/runtime-plugins.md @@ -5,13 +5,31 @@ title: Family Runtime DSOs The pre-#1093 runtime-plugin registry no longer exists. The stable load unit is one family DSO named `libtrtmc_model_.so`. -The bundle header names exactly one family and backend. The loader opens that -family DSO from the explicit runtime root, resolves `trtmc_create_family`, and -receives an implementation of an abstract Task interface. A family owns its -factory, pipeline, preprocessing, postprocessing, dispatch, bindings, state, -samplers, and any genuinely model-specific TensorRT plugin. +The bundle header names exactly one family and backend. Each DSO publishes the +model-agnostic `trtmc_plugin_descriptor_v1` interface with its product-build +identity, kind, and ID. The loader opens the exact backend and family paths +from one explicit plugin root, validates both descriptors, resolves +`trtmc_create_family`, and receives an implementation of an abstract Task +interface. -There is no registrar macro, runtime-strategy map, central manifest, sibling -fallback, or hot plugin marketplace. To extend an existing family, edit only -its `families//runtime/` implementation and family-owned tests. To add -a new family, follow [Add a Model Family](../extend/add-model-family.md). +A family owns its descriptor declaration, factory, pipeline, preprocessing, +postprocessing, dispatch, bindings, state, samplers, and any genuinely +model-specific TensorRT plugin. Source ownership remains family-local, while +the resulting DSO is released as part of one coordinated product build. + +Descriptor v1 is an exact metadata layout, not ABI negotiation. The factory, +context, backend, module, and Task seams contain C++ interfaces, so plugins from +another product build are rejected even when they expose the same descriptor +version. Core and runtime also expose and compare their product-build identities +before the runtime calls a core C++ interface. + +Plugin roots are trusted native-code inputs. ELF constructors run during +`dlopen`, before the descriptor can be called. The loader therefore never opens +discovery candidates speculatively, never falls back after a selected load +fails, and keeps opened DSOs resident so TensorRT registrars cannot outlive +their defining code. + +There is no runtime-strategy map, central manifest, sibling fallback, or hot +plugin marketplace. To extend an existing family, edit only its +`families//runtime/` implementation and family-owned tests. To add a +new family, follow [Add a Model Family](../extend/add-model-family.md). diff --git a/website/docs/getting-started/quick-start.md b/website/docs/getting-started/quick-start.md index 71bd162ffd..3e5704aa53 100644 --- a/website/docs/getting-started/quick-start.md +++ b/website/docs/getting-started/quick-start.md @@ -50,23 +50,27 @@ trtmc run gpt2.bundle \ ``` The CLI reads the bundle family and backend, then selects the first complete, -single-directory runtime in this order: +single-directory plugin root in this order: -1. the current directory, when all required libraries match the active build - cohort; -2. the runtime belonging to the active `trtmc` selected through `PATH`, +1. the directory containing the active `libtrtmc_runtime.so`; +2. the installation belonging to the active `trtmc` selected through `PATH`, including native CMake and wheel install layouts; 3. colon-separated directories in `TRTMC_RUNTIME_PATH`. -A complete GPT-2 TensorRT runtime contains matching `libtrtmc_core.so`, -`libtrtmc_runtime.so`, `libtrtmc_backend_trt.so`, and -`libtrtmc_model_gpt2.so` files. Candidates are never combined across -directories: the CLI enumerates paths, and the Runtime Loader contract validates -each candidate without loading it. The CLI prints the automatically selected -directory. If more than one installed wheel runtime matches, select one with -`--runtime-root DIR`. An explicit root bypasses discovery. - -Every native artifact carries a build-cohort identity, and automatic discovery -accepts a directory only when the identity matches the core and runtime already -loaded by `trtmc`. The platform loader evaluates `LD_LIBRARY_PATH` before the -CLI starts, so it can determine that active cohort before the search above. +A complete GPT-2 TensorRT plugin root contains root-local +`libtrtmc_backend_trt.so` and `libtrtmc_model_gpt2.so` files. Candidates are +never combined across directories, and discovery never loads a candidate just +to inspect it. After selection, the Runtime Loader loads those exact paths and +requires their descriptors to match the active product build, plugin kinds, +and bundle IDs before it calls either factory. A mismatched selected root fails +immediately without falling back to another installation. + +The Runtime Loader also verifies that its already loaded Core belongs to the +same product build before reading the bundle. + +The CLI prints the automatically selected directory. If more than one installed +wheel root matches structurally, select one with `--runtime-root DIR`. An +explicit root bypasses discovery but not build and identity validation. The +current directory is not searched implicitly; use `TRTMC_RUNTIME_PATH=.` when +that behavior is intended. `LD_LIBRARY_PATH` remains a platform-loader setting +evaluated before the CLI starts. diff --git a/website/docs/user-guides/run-inference.md b/website/docs/user-guides/run-inference.md index 29b88c31da..b01794fe2f 100644 --- a/website/docs/user-guides/run-inference.md +++ b/website/docs/user-guides/run-inference.md @@ -4,7 +4,8 @@ description: Select the native command matching the bundle's declared Task. --- Inspect the bundle, then call the Task named by its family manifest/header. -Every execution command requires `--runtime-root DIR`. +Execution commands discover the active installation by default. Use +`--runtime-root DIR` only to select one exact plugin root. | Task | Command | Primary result | | --- | --- | --- | @@ -19,13 +20,13 @@ Every execution command requires `--runtime-root DIR`. ```bash trtmc run qwen3-0.6b.bundle \ - --runtime-root /opt/trtmc/lib \ --prompt "What is the capital of France? Answer in one word." \ --max-new-tokens 10 \ --temperature 0 \ --top-k 1 ``` -The CLI loads exactly one family DSO and backend DSO from the runtime root. A -wrong Task command fails instead of attempting another family or interface. -See the [CLI Reference](../api/cli-reference.md) for command-specific inputs. +The CLI selects one root, then the Runtime Loader validates and loads exactly +one family DSO and backend DSO from it. A wrong Task command or mismatched-build +plugin fails instead of attempting another root, family, or interface. See the +[CLI Reference](../api/cli-reference.md) for command-specific inputs. From 5c50039010a842bc9cbda5a39cdb58ed0e9db774 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Tue, 8 Sep 2026 07:56:34 +0000 Subject: [PATCH 05/13] refactor(runtime): enforce one product root Co-locate the wheel's native CLI, Core, Runtime, and plugins, and replace the installed console command with a thin exec adapter. Limit discovery to the active Runtime and TRTMC_RUNTIME_PATH, validate executable/Core/Runtime identity before crossing C++ interfaces, and keep selected-root loading exact and fail-closed. Signed-off-by: chaofengw --- CMakeLists.txt | 68 ++++++++-- apps/cli/cli.cpp | 98 +++----------- apps/cli/cli.h | 1 - apps/cli/tests/test_cli.cpp | 121 +++++++++++++----- cmake/product_build.h.in | 8 ++ conanfile.py | 27 ++-- .../tensorrt_model_connect/native_cli.py | 16 +++ core/builder/tests/test_native_cli.py | 45 +++++++ .../include/trtmc/runtime/plugin_abi.h | 4 + core/runtime/loader/family_loader.cpp | 31 ++++- core/runtime/primitives/build_identity.cpp | 4 - .../tests/fake_core_build_identity.cpp | 8 ++ core/runtime/tests/test_family_loader.cpp | 12 ++ pyproject.toml | 1 + tools/ci/package.py | 30 +++-- tools/tests/test_architecture.py | 7 +- tools/tests/test_new_ci.py | 47 ++++++- website/docs/api/cli-reference.md | 14 +- .../ai-native-horizontal-scaling.md | 28 ++-- website/docs/architecture/build-system.md | 10 +- website/docs/architecture/runtime-plugins.md | 14 +- website/docs/extend/add-model-family.md | 11 ++ website/docs/extend/add-optimized-runtime.md | 1 - website/docs/features/config-and-backends.md | 1 - website/docs/features/multi-device.md | 1 - website/docs/features/sampling.md | 2 +- website/docs/getting-started/build-and-run.md | 12 -- website/docs/getting-started/glossary.md | 4 +- website/docs/getting-started/quick-start.md | 26 ++-- .../docs/getting-started/troubleshooting.md | 4 +- website/docs/reference/profiling.md | 7 +- .../advanced/multi-device-inference.md | 1 - .../quantization-and-runtime-knobs.md | 1 - .../tutorials/beginner/inspect-bundles.md | 4 +- .../tutorials/beginner/text-generation.md | 1 - .../tutorials/intermediate/canary-decoding.md | 2 - .../intermediate/diffusion-and-time-series.md | 3 - .../intermediate/multimodal-and-speech.md | 5 - website/docs/user-guides/configure-runtime.md | 1 - .../user-guides/image-video-generation.md | 7 +- website/docs/user-guides/multimodal-speech.md | 5 - website/docs/user-guides/text-generation.md | 2 - website/docs/user-guides/time-series.md | 2 - 43 files changed, 451 insertions(+), 246 deletions(-) create mode 100644 cmake/product_build.h.in create mode 100644 core/builder/tensorrt_model_connect/native_cli.py create mode 100644 core/builder/tests/test_native_cli.py create mode 100644 core/runtime/tests/fake_core_build_identity.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index dd3e267554..db1ed031a3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,7 +23,13 @@ string(LENGTH "${TRTMC_BUILD_ID}" _trtmc_build_id_length) if(NOT _trtmc_build_id_length EQUAL 32 OR NOT TRTMC_BUILD_ID MATCHES "^[0-9a-f]+$") message(FATAL_ERROR "TRTMC_BUILD_ID must contain exactly 32 lowercase hex characters") endif() -add_compile_definitions(TRTMC_BUILD_ID="${TRTMC_BUILD_ID}") +set(_trtmc_generated_include_dir "${CMAKE_BINARY_DIR}/generated/include") +file(MAKE_DIRECTORY "${_trtmc_generated_include_dir}/trtmc/runtime") +configure_file( + cmake/product_build.h.in + "${_trtmc_generated_include_dir}/trtmc/runtime/product_build.h" + @ONLY +) include(GNUInstallDirs) find_package(CUDAToolkit REQUIRED) @@ -92,16 +98,20 @@ endif() set(TRTMC_CUDA_INCLUDE_DIR ${CUDAToolkit_INCLUDE_DIRS}) set(TRTMC_CUDART_LIBRARY CUDA::cudart) -add_library(trtmc_core SHARED +set(_trtmc_core_sources core/runtime/bundle/bundle_format.cpp - core/runtime/primitives/build_identity.cpp core/runtime/primitives/cuda_common.cpp core/runtime/primitives/device_tensor.cpp core/runtime/primitives/trt_common.cpp ) +add_library(trtmc_core SHARED + ${_trtmc_core_sources} + core/runtime/primitives/build_identity.cpp +) target_include_directories(trtmc_core PUBLIC $ + $ $ PRIVATE ${PROJECT_SOURCE_DIR}/core @@ -232,6 +242,7 @@ if(TRTMC_HAS_TVM_FFI) $ PRIVATE ${PROJECT_SOURCE_DIR}/core + ${_trtmc_generated_include_dir} ) target_include_directories(trtmc_byok_tvm_ffi SYSTEM PRIVATE ${TRTMC_TRT_INCLUDE_DIR} @@ -367,7 +378,10 @@ if(TRTMC_BUILD_TESTS) ) add_library(trtmc_test_backend_fake SHARED core/runtime/tests/fake_backend.cpp) - target_include_directories(trtmc_test_backend_fake PRIVATE ${PROJECT_SOURCE_DIR}/core/runtime/include) + target_include_directories(trtmc_test_backend_fake PRIVATE + ${PROJECT_SOURCE_DIR}/core/runtime/include + ${_trtmc_generated_include_dir} + ) target_link_libraries(trtmc_test_backend_fake PRIVATE CUDA::cudart) set_target_properties(trtmc_test_backend_fake PROPERTIES OUTPUT_NAME trtmc_backend_fake @@ -377,6 +391,7 @@ if(TRTMC_BUILD_TESTS) add_library(trtmc_test_backend_fake_rtx SHARED core/runtime/tests/fake_backend.cpp) target_include_directories(trtmc_test_backend_fake_rtx PRIVATE ${PROJECT_SOURCE_DIR}/core/runtime/include + ${_trtmc_generated_include_dir} ) target_link_libraries(trtmc_test_backend_fake_rtx PRIVATE CUDA::cudart) target_compile_definitions(trtmc_test_backend_fake_rtx PRIVATE @@ -390,6 +405,7 @@ if(TRTMC_BUILD_TESTS) add_library(trtmc_test_backend_incompatible SHARED core/runtime/tests/fake_backend.cpp) target_include_directories(trtmc_test_backend_incompatible PRIVATE ${PROJECT_SOURCE_DIR}/core/runtime/include + ${_trtmc_generated_include_dir} ) target_link_libraries(trtmc_test_backend_incompatible PRIVATE CUDA::cudart) target_compile_definitions(trtmc_test_backend_incompatible PRIVATE @@ -401,12 +417,25 @@ if(TRTMC_BUILD_TESTS) LIBRARY_OUTPUT_DIRECTORY "${_trtmc_test_runtime_root}" ) + add_library(trtmc_test_backend_compatible_incompatible SHARED + core/runtime/tests/fake_backend.cpp + ) + target_include_directories(trtmc_test_backend_compatible_incompatible PRIVATE + ${PROJECT_SOURCE_DIR}/core/runtime/include + ${_trtmc_generated_include_dir} + ) + target_link_libraries(trtmc_test_backend_compatible_incompatible PRIVATE CUDA::cudart) + target_compile_definitions(trtmc_test_backend_compatible_incompatible PRIVATE + TRTMC_FAKE_BACKEND_NAME="incompatible" + ) + set_target_properties(trtmc_test_backend_compatible_incompatible PROPERTIES + OUTPUT_NAME trtmc_backend_incompatible_compatible + LIBRARY_OUTPUT_DIRECTORY "${_trtmc_test_runtime_root}" + ) + add_library(trtmc_test_core_incompatible SHARED - core/runtime/bundle/bundle_format.cpp - core/runtime/primitives/build_identity.cpp - core/runtime/primitives/cuda_common.cpp - core/runtime/primitives/device_tensor.cpp - core/runtime/primitives/trt_common.cpp + ${_trtmc_core_sources} + core/runtime/tests/fake_core_build_identity.cpp ) target_include_directories(trtmc_test_core_incompatible PUBLIC ${PROJECT_SOURCE_DIR}/core/runtime/include @@ -419,9 +448,6 @@ if(TRTMC_BUILD_TESTS) PUBLIC CUDA::cudart PRIVATE nlohmann_json::nlohmann_json ) - target_compile_definitions(trtmc_test_core_incompatible PRIVATE - TRTMC_FAKE_INCOMPATIBLE_BUILD=1 - ) target_compile_options(trtmc_test_core_incompatible PRIVATE -Wall -Wextra -Wpedantic) set_target_properties(trtmc_test_core_incompatible PROPERTIES OUTPUT_NAME trtmc_core @@ -437,7 +463,13 @@ if(TRTMC_BUILD_TESTS) LIBRARY_OUTPUT_DIRECTORY "${_trtmc_test_runtime_root}" BUILD_RPATH "\$ORIGIN/../.." ) - add_dependencies(test_cli trtmc_test_backend_fake trtmc_test_family_fake) + add_dependencies(test_cli + trtmc_test_backend_fake + trtmc_test_backend_incompatible + trtmc_test_backend_compatible_incompatible + trtmc_test_core_incompatible + trtmc_test_family_fake + ) add_executable(test_runtime_root core/runtime/tests/test_runtime_root.cpp) target_include_directories(test_runtime_root PRIVATE ${PROJECT_SOURCE_DIR}/core/runtime/include) @@ -473,6 +505,16 @@ if(TRTMC_BUILD_TESTS) "${_trtmc_test_runtime_root}" --expect-core-mismatch ) + add_test(NAME cli_incompatible_core + COMMAND ${CMAKE_COMMAND} -E env + "LD_LIBRARY_PATH=$" + $ + --expect-core-mismatch + ) + set_tests_properties( + family_loader family_loader_incompatible_core cli cli_incompatible_core runtime_root + PROPERTIES RESOURCE_LOCK runtime-loader-fixtures + ) add_executable(test_trt_module_dynamic_input core/runtime/tests/test_trt_module_dynamic_input.cpp diff --git a/apps/cli/cli.cpp b/apps/cli/cli.cpp index e9fc1515e6..dbb74a9ae4 100644 --- a/apps/cli/cli.cpp +++ b/apps/cli/cli.cpp @@ -7,6 +7,7 @@ #include "cli/io.h" #include "trtmc/runtime/family_loader.h" +#include "trtmc/runtime/plugin_abi.h" #include "trtmc/runtime/runtime_root.h" #include @@ -26,7 +27,6 @@ #include #include #include -#include #include #include #include @@ -174,48 +174,28 @@ void append_path_list(std::vector& candidates, std::set& } } -void append_python_package_runtime_roots(std::vector& candidates, - std::set& seen, const fs::path& prefix) { - std::vector python_roots; - for (const auto& library_root : {prefix / "lib", prefix / "lib64"}) { - std::error_code error; - if (!fs::is_directory(library_root, error)) - continue; - for (fs::directory_iterator iterator(library_root, error), end; !error && iterator != end; - iterator.increment(error)) { - std::error_code entry_error; - if (!iterator->is_directory(entry_error)) - continue; - const std::string name = iterator->path().filename().string(); - if (name.rfind("python", 0) != 0) - continue; - for (const auto& packages : {"site-packages", "dist-packages"}) { - python_roots.push_back(iterator->path() / packages / "tensorrt_model_connect" / - "bin"); - } - } - } - std::sort(python_roots.begin(), python_roots.end()); - for (const auto& root : python_roots) - append_candidate(candidates, seen, root); -} - RuntimeRootSearchContext runtime_root_search_context() { RuntimeRootSearchContext context; context.loaded_runtime_root = loaded_runtime_root(); - - std::vector executable(4096, '\0'); - const ssize_t length = readlink("/proc/self/exe", executable.data(), executable.size() - 1); - if (length > 0) { - executable[static_cast(length)] = '\0'; - context.executable = executable.data(); - } - if (const char* value = std::getenv("TRTMC_RUNTIME_PATH")) context.runtime_path = value; return context; } +void require_matching_product_build() { + const std::string expected = trtmc::kPluginBuildId; + const auto require_module = [&](const char* module, const char* actual) { + if (actual == nullptr || expected != actual) { + throw std::runtime_error( + "TRTMC product build mismatch: CLI requires '" + expected + "' but active " + + module + " reports '" + + (actual != nullptr ? std::string(actual) : std::string("")) + "'"); + } + }; + require_module("Runtime", trtmc_runtime_build_id()); + require_module("Core", trtmc_core_build_id()); +} + std::string take_value(int argc, char** argv, int& index, const std::string& option) { if (index + 1 >= argc) throw std::invalid_argument(option + " requires a value"); @@ -719,50 +699,13 @@ std::string resolve_runtime_root(const BundleInfo& bundle, const std::string& ex if (!matches) throw std::logic_error("runtime-root discovery requires a candidate matcher"); - std::vector installed_candidates; - std::vector wheel_candidates; - std::vector configured_candidates; + std::vector candidates; std::set seen; - append_candidate(installed_candidates, seen, context.loaded_runtime_root); - - if (!context.executable.empty()) { - const fs::path executable_directory = context.executable.parent_path(); - append_candidate(installed_candidates, seen, executable_directory); - if (executable_directory.filename() == "bin") { - const fs::path prefix = executable_directory.parent_path(); - append_candidate(installed_candidates, seen, prefix / "lib"); - append_candidate(installed_candidates, seen, prefix / "lib64"); - append_python_package_runtime_roots(wheel_candidates, seen, prefix); - } - } - - append_path_list(configured_candidates, seen, context.runtime_path); + append_candidate(candidates, seen, context.loaded_runtime_root); + append_path_list(candidates, seen, context.runtime_path); std::vector searched; - for (const auto& candidate : installed_candidates) { - searched.push_back(candidate); - if (matches(bundle, candidate, require_byok)) - return candidate.string(); - } - - std::vector matching_wheels; - for (const auto& candidate : wheel_candidates) { - searched.push_back(candidate); - if (matches(bundle, candidate, require_byok)) - matching_wheels.push_back(candidate); - } - if (matching_wheels.size() == 1) - return matching_wheels.front().string(); - if (matching_wheels.size() > 1) { - std::ostringstream message; - message << "Multiple installed TRTMC runtimes match the running CLI:"; - for (const auto& candidate : matching_wheels) - message << " " << candidate.string(); - message << ". Pass --runtime-root DIR to select one."; - throw std::runtime_error(message.str()); - } - - for (const auto& candidate : configured_candidates) { + for (const auto& candidate : candidates) { searched.push_back(candidate); if (matches(bundle, candidate, require_byok)) return candidate.string(); @@ -1422,7 +1365,7 @@ void print_usage(std::ostream& output) { " [--kv-cache-size BYTES|GB|GiB]\n\n" "TensorRT-RTX runtime options:\n" " [--runtime-cache PATH] [--cuda-graphs]\n\n" - "Runtime discovery: the active trtmc installation, then TRTMC_RUNTIME_PATH.\n" + "Runtime discovery: the active Runtime directory, then TRTMC_RUNTIME_PATH.\n" "The current directory is not searched; use TRTMC_RUNTIME_PATH=. explicitly.\n" "--runtime-root selects one exact root without fallback.\n"; } @@ -1438,6 +1381,7 @@ int run(int argc, char** argv, std::ostream& output, std::ostream& error) { output << "trtmc " << TRTMC_VERSION_STRING << '\n'; return EXIT_SUCCESS; } + require_matching_product_build(); if (command.kind == CommandKind::kInspect) { const BundleInfo bundle = InspectBundle(command.bundle); nlohmann::json sections = nlohmann::json::object(); diff --git a/apps/cli/cli.h b/apps/cli/cli.h index 998fd816cc..350db11c17 100644 --- a/apps/cli/cli.h +++ b/apps/cli/cli.h @@ -65,7 +65,6 @@ struct Command { struct RuntimeRootSearchContext { std::filesystem::path loaded_runtime_root; - std::filesystem::path executable; std::string runtime_path; }; diff --git a/apps/cli/tests/test_cli.cpp b/apps/cli/tests/test_cli.cpp index c0655acf81..33d523f1e1 100644 --- a/apps/cli/tests/test_cli.cpp +++ b/apps/cli/tests/test_cli.cpp @@ -57,11 +57,13 @@ std::string path_key(const std::filesystem::path& path) { return (error ? absolute.lexically_normal() : canonical).string(); } -void write_fake_bundle(const std::filesystem::path& path) { +void write_fake_bundle(const std::filesystem::path& path, const std::string& backend = "fake") { static constexpr unsigned char magic[8] = {'B', 'U', 'N', 'D', 'L', 'E', '\x01', '\0'}; const std::string header = "{\"format\":1,\"family\":\"fake\",\"task\":\"time_series_forecast\"," - "\"backend\":\"fake\",\"sections\":{\"runtime.json\":{\"offset\":0,\"length\":2}," + "\"backend\":\"" + + backend + + "\",\"sections\":{\"runtime.json\":{\"offset\":0,\"length\":2}," "\"engine.plan\":{\"offset\":2,\"length\":4}}}"; std::ofstream output(path, std::ios::binary); output.write(reinterpret_cast(magic), 8); @@ -113,6 +115,58 @@ void check_cli_runtime_discovery(const std::filesystem::path& runtime_root) { std::filesystem::remove_all(discovered_root); } +void check_selected_root_never_falls_back(const std::filesystem::path& runtime_root) { + const auto base = runtime_root.parent_path() / "cli-no-fallback"; + const auto incompatible_root = base / "incompatible"; + const auto compatible_root = base / "compatible"; + std::filesystem::remove_all(base); + std::filesystem::create_directories(incompatible_root); + std::filesystem::create_directories(compatible_root); + for (const auto& root : {incompatible_root, compatible_root}) { + std::filesystem::copy_file(runtime_root / "libtrtmc_model_fake.so", + root / "libtrtmc_model_fake.so"); + } + std::filesystem::copy_file(runtime_root / "libtrtmc_backend_incompatible.so", + incompatible_root / "libtrtmc_backend_incompatible.so"); + std::filesystem::copy_file(runtime_root / "libtrtmc_backend_incompatible_compatible.so", + compatible_root / "libtrtmc_backend_incompatible.so"); + const auto bundle = base / "incompatible.bundle"; + const auto input = base / "input.f32"; + write_fake_bundle(bundle, "incompatible"); + { + const float values[] = {1.0F}; + std::ofstream output(input, std::ios::binary); + output.write(reinterpret_cast(values), sizeof(values)); + } + + const char* previous_runtime_path_value = std::getenv("TRTMC_RUNTIME_PATH"); + const bool had_runtime_path = previous_runtime_path_value != nullptr; + const std::string previous_runtime_path = had_runtime_path ? previous_runtime_path_value : ""; + const std::string configured_roots = + incompatible_root.string() + ":" + compatible_root.string(); + setenv("TRTMC_RUNTIME_PATH", configured_roots.c_str(), 1); + std::vector arguments{"trtmc", "forecast", bundle.string(), "--input", + input.string()}; + std::vector argv; + for (auto& argument : arguments) + argv.push_back(argument.data()); + std::ostringstream output; + std::ostringstream error; + const int result = trtmc::cli::run(static_cast(argv.size()), argv.data(), output, error); + if (had_runtime_path) + setenv("TRTMC_RUNTIME_PATH", previous_runtime_path.c_str(), 1); + else + unsetenv("TRTMC_RUNTIME_PATH"); + + check(result != 0 && error.str().find("belongs to product build") != std::string::npos, + "selected structurally complete root fails without fallback"); + check(error.str().find("Using TRTMC runtime: " + incompatible_root.string()) != + std::string::npos && + output.str().empty(), + "runtime path order cannot hide an incompatible selected product"); + std::filesystem::remove_all(base); +} + bool resolve_throws(const trtmc::BundleInfo& bundle, const trtmc::cli::RuntimeRootSearchContext& context, const trtmc::cli::RuntimeRootMatcher& matches, std::string& message) { @@ -349,6 +403,22 @@ bool dispatch_throws(const trtmc::cli::Command& command, trtmc::ITask& task) { } // namespace int main(int argc, char** argv) { + if (argc == 2 && std::string(argv[1]) == "--expect-core-mismatch") { + std::vector arguments{"trtmc", "inspect", "not-opened.bundle"}; + std::vector pointers; + for (auto& argument : arguments) + pointers.push_back(argument.data()); + std::ostringstream output; + std::ostringstream error; + const int result = + trtmc::cli::run(static_cast(pointers.size()), pointers.data(), output, error); + check(result != 0 && error.str().find("active Core reports") != std::string::npos, + "CLI rejects an incompatible Core before bundle inspection"); + check(error.str().find("not-opened.bundle") == std::string::npos, + "CLI does not cross the Core C++ seam before identity validation"); + return failures == 0 ? 0 : 1; + } + const std::vector execution_commands{ "run", "encode", @@ -402,9 +472,11 @@ int main(int argc, char** argv) { path_key(optional_root), }; std::set byok_roots; + std::vector observed_candidates; const trtmc::cli::RuntimeRootMatcher matches = [&](const trtmc::BundleInfo&, const std::filesystem::path& candidate, bool require_byok) { const std::string key = path_key(candidate); + observed_candidates.push_back(key); return complete_roots.count(key) != 0 && (!require_byok || byok_roots.count(key) != 0); }; @@ -421,32 +493,6 @@ int main(int argc, char** argv) { "the configured runtime path is used when the active installation is incomplete"); complete_roots.insert(path_key(loaded_runtime_root)); - const auto wheel_prefix = runtime_test_root / "wheel"; - const auto wheel_bin = wheel_prefix / "bin"; - const auto wheel_runtime = - wheel_prefix / "lib" / "python3.12" / "dist-packages" / "tensorrt_model_connect" / "bin"; - std::filesystem::create_directories(wheel_bin); - std::filesystem::create_directories(wheel_runtime); - complete_roots.insert(path_key(wheel_runtime)); - std::filesystem::create_directory_symlink("lib", wheel_prefix / "lib64"); - runtime_context = {}; - runtime_context.executable = wheel_bin / "trtmc"; - runtime_context.loaded_runtime_root = wheel_bin; - check(trtmc::cli::resolve_runtime_root(runtime_bundle, {}, false, runtime_context, matches) == - wheel_runtime.string(), - "wheel runtime is discovered once when lib64 aliases lib"); - - const auto second_wheel_runtime = - wheel_prefix / "lib" / "python3.13" / "site-packages" / "tensorrt_model_connect" / "bin"; - std::filesystem::create_directories(second_wheel_runtime); - complete_roots.insert(path_key(second_wheel_runtime)); - std::string discovery_error; - check(resolve_throws(runtime_bundle, runtime_context, matches, discovery_error) && - discovery_error.find("Multiple installed TRTMC runtimes") != std::string::npos, - "ambiguous matching wheel runtimes require an explicit selection"); - complete_roots.erase(path_key(second_wheel_runtime)); - std::filesystem::remove_all(second_wheel_runtime); - const auto first_optional = runtime_test_root / "first-optional"; const auto second_optional = runtime_test_root / "second-optional"; const auto loaded_libraries = runtime_test_root / "loaded-libraries"; @@ -461,6 +507,21 @@ int main(int argc, char** argv) { second_optional.string(), "dedicated runtime path preserves directory order"); + const auto current_directory_root = runtime_test_root / "current-directory"; + std::filesystem::create_directories(current_directory_root); + complete_roots.insert(path_key(current_directory_root)); + runtime_context.runtime_path.clear(); + const auto original_current_directory = std::filesystem::current_path(); + std::filesystem::current_path(current_directory_root); + observed_candidates.clear(); + std::string discovery_error; + check(resolve_throws(runtime_bundle, runtime_context, matches, discovery_error), + "runtime discovery does not search an unconfigured current directory"); + std::filesystem::current_path(original_current_directory); + check(std::find(observed_candidates.begin(), observed_candidates.end(), + path_key(current_directory_root)) == observed_candidates.end(), + "current directory participates only through explicit configuration"); + const auto byok_root = runtime_test_root / "byok"; std::filesystem::create_directories(byok_root); complete_roots.insert(path_key(byok_root)); @@ -485,8 +546,10 @@ int main(int argc, char** argv) { discovery_error.find("--runtime-root") != std::string::npos, "runtime discovery failure identifies the bundle and explicit override"); std::filesystem::remove_all(runtime_test_root); - if (argc == 2) + if (argc == 2) { check_cli_runtime_discovery(argv[1]); + check_selected_root_never_falls_back(argv[1]); + } const auto dynamic_kv = parse({"trtmc", "run", "model.bundle", "--runtime-root", "lib", "--kv-cache-size", "1GiB"}); check(dynamic_kv.kv_cache_size_bytes == 1024ULL * 1024ULL * 1024ULL, diff --git a/cmake/product_build.h.in b/cmake/product_build.h.in new file mode 100644 index 0000000000..236021f601 --- /dev/null +++ b/cmake/product_build.h.in @@ -0,0 +1,8 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#define TRTMC_BUILD_ID "@TRTMC_BUILD_ID@" diff --git a/conanfile.py b/conanfile.py index 4b69898a08..c3f22b7172 100644 --- a/conanfile.py +++ b/conanfile.py @@ -75,19 +75,16 @@ def package(self) -> None: build = Path(self.build_folder) package = Path(self.package_folder) module_bin = package / "tensorrt_model_connect" / "bin" - script_bin = package / f"{self.name.replace('-', '_')}-{self.version}.data" / "scripts" copy(self, "trtmc", src=str(build), dst=str(module_bin), keep_path=False) - copy(self, "trtmc", src=str(build), dst=str(script_bin), keep_path=False) - for destination in (module_bin, script_bin): - for library in ("libtrtmc_core.so", "libtrtmc_runtime.so"): - copy( - self, - library, - src=str(build), - dst=str(destination), - keep_path=False, - ) + for library in ("libtrtmc_core.so", "libtrtmc_runtime.so"): + copy( + self, + library, + src=str(build), + dst=str(module_bin), + keep_path=False, + ) copy( self, "libtrtmc_backend_trt*.so", @@ -146,11 +143,8 @@ def package(self) -> None: ) native = module_bin / "trtmc" - installed = script_bin / "trtmc" shared_runtime = [ - destination / library - for destination in (module_bin, script_bin) - for library in ("libtrtmc_core.so", "libtrtmc_runtime.so") + module_bin / library for library in ("libtrtmc_core.so", "libtrtmc_runtime.so") ] backend = module_bin / "libtrtmc_backend_trt.so" backends = sorted(module_bin.glob("libtrtmc_backend_trt*.so")) @@ -159,7 +153,6 @@ def package(self) -> None: dataset_benchmark = module_bin / "trtmc_dataset_benchmark" if ( not native.is_file() - or not installed.is_file() or not all(library.is_file() for library in shared_runtime) or not backend.is_file() or not byok.is_file() @@ -168,7 +161,7 @@ def package(self) -> None: ): raise ConanException("native runtime package is incomplete") - for executable in (native, installed, benchmark_worker, dataset_benchmark): + for executable in (native, benchmark_worker, dataset_benchmark): _make_executable(executable) _set_runpath(executable, "$ORIGIN") for library in shared_runtime: diff --git a/core/builder/tensorrt_model_connect/native_cli.py b/core/builder/tensorrt_model_connect/native_cli.py new file mode 100644 index 0000000000..83b3f6d617 --- /dev/null +++ b/core/builder/tensorrt_model_connect/native_cli.py @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import os +from pathlib import Path +import sys + + +def main() -> None: + """Replace the console adapter with its co-located native product CLI.""" + executable = Path(__file__).resolve().parent / "bin" / "trtmc" + if not executable.is_file() or not os.access(executable, os.X_OK): + raise RuntimeError(f"native trtmc executable is missing or not executable: {executable}") + os.execv(executable, [str(executable), *sys.argv[1:]]) diff --git a/core/builder/tests/test_native_cli.py b/core/builder/tests/test_native_cli.py new file mode 100644 index 0000000000..0284656532 --- /dev/null +++ b/core/builder/tests/test_native_cli.py @@ -0,0 +1,45 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from pathlib import Path +import sys + +import pytest + +from tensorrt_model_connect import native_cli + + +def test_main_executes_the_native_cli_in_the_package_bin( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + package = tmp_path / "tensorrt_model_connect" + executable = package / "bin" / "trtmc" + executable.parent.mkdir(parents=True) + executable.touch() + executable.chmod(0o755) + monkeypatch.setattr(native_cli, "__file__", str(package / "native_cli.py")) + monkeypatch.setattr(sys, "argv", ["trtmc", "version"]) + + called: tuple[Path, list[str]] | None = None + + def capture_execv(path: Path, arguments: list[str]) -> None: + nonlocal called + called = (path, arguments) + + monkeypatch.setattr(native_cli.os, "execv", capture_execv) + native_cli.main() + + assert called == (executable, [str(executable), "version"]) + + +def test_main_rejects_an_incomplete_native_product( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + package = tmp_path / "tensorrt_model_connect" + package.mkdir() + monkeypatch.setattr(native_cli, "__file__", str(package / "native_cli.py")) + + with pytest.raises(RuntimeError, match="native trtmc executable is missing"): + native_cli.main() diff --git a/core/runtime/include/trtmc/runtime/plugin_abi.h b/core/runtime/include/trtmc/runtime/plugin_abi.h index a66f20f4f1..109d01d614 100644 --- a/core/runtime/include/trtmc/runtime/plugin_abi.h +++ b/core/runtime/include/trtmc/runtime/plugin_abi.h @@ -5,6 +5,10 @@ #pragma once +#if __has_include("trtmc/runtime/product_build.h") +#include "trtmc/runtime/product_build.h" +#endif + #include namespace trtmc { diff --git a/core/runtime/loader/family_loader.cpp b/core/runtime/loader/family_loader.cpp index f6d837df8d..1bfcfe4f72 100644 --- a/core/runtime/loader/family_loader.cpp +++ b/core/runtime/loader/family_loader.cpp @@ -82,11 +82,17 @@ fs::path explicit_runtime_root(const std::string& runtime_root) { if (runtime_root.empty()) throw std::invalid_argument("runtime_root must be explicit and non-empty"); std::error_code error; - fs::path root = fs::absolute(fs::path(runtime_root), error); + const fs::path absolute = fs::absolute(fs::path(runtime_root), error); if (error) throw std::runtime_error("Unable to resolve runtime_root '" + runtime_root + "': " + error.message()); - return root.lexically_normal(); + fs::path root = fs::canonical(absolute, error); + if (error) + throw std::runtime_error("Unable to resolve runtime_root '" + runtime_root + + "': " + error.message()); + if (!fs::is_directory(root, error) || error) + throw std::runtime_error("runtime_root is not a directory: '" + runtime_root + "'"); + return root; } fs::path loaded_library_path(const void* symbol) { @@ -108,6 +114,20 @@ bool contains_root_local_library(const fs::path& root, const std::string& librar !error; } +fs::path require_root_local_library(const fs::path& root, const std::string& library) { + std::error_code error; + const fs::path resolved = fs::canonical(root / library, error); + if (error || !fs::is_regular_file(resolved, error) || error) { + throw std::runtime_error("Runtime root '" + root.string() + "' does not contain '" + + library + "'"); + } + if (resolved.parent_path() != root) { + throw std::runtime_error("Library '" + (root / library).string() + + "' escapes the selected runtime root"); + } + return resolved; +} + void require_matching_build(const std::string& path, const PluginDescriptorV1& descriptor) { if (descriptor.build_id != nullptr && std::string(descriptor.build_id) == kPluginBuildId) return; @@ -195,7 +215,7 @@ class SharedLibrary { class BackendLibrary { public: BackendLibrary(const fs::path& runtime_root, const std::string& backend_id) - : library_(runtime_root / backend_library_name(backend_id)) { + : library_(require_root_local_library(runtime_root, backend_library_name(backend_id))) { library_.require_plugin(PluginKind::kBackend, backend_id); const auto create = reinterpret_cast(library_.require_symbol("trtmc_create_backend")); @@ -234,7 +254,7 @@ class BackendLibrary { class FamilyLibrary { public: FamilyLibrary(const fs::path& runtime_root, const std::string& family_id) - : library_(runtime_root / family_library_name(family_id)) { + : library_(require_root_local_library(runtime_root, family_library_name(family_id))) { library_.require_plugin(PluginKind::kFamily, family_id); create_ = reinterpret_cast(library_.require_symbol(kCreateFamilySymbol)); } @@ -252,7 +272,8 @@ class FamilyLibrary { class RuntimeExtensionLibrary { public: explicit RuntimeExtensionLibrary(const fs::path& runtime_root) - : library_(runtime_root / runtime_extension_library_name("tvm_ffi")) { + : library_( + require_root_local_library(runtime_root, runtime_extension_library_name("tvm_ffi"))) { library_.require_plugin(PluginKind::kRuntimeExtension, "tvm_ffi"); load_ = reinterpret_cast(library_.require_symbol("trtmc_load_byok_kernel")); } diff --git a/core/runtime/primitives/build_identity.cpp b/core/runtime/primitives/build_identity.cpp index 52f9308e9d..7c2788401a 100644 --- a/core/runtime/primitives/build_identity.cpp +++ b/core/runtime/primitives/build_identity.cpp @@ -6,9 +6,5 @@ #include "trtmc/runtime/plugin_abi.h" extern "C" const char* trtmc_core_build_id() noexcept { -#ifdef TRTMC_FAKE_INCOMPATIBLE_BUILD - return "00000000000000000000000000000000"; -#else return trtmc::kPluginBuildId; -#endif } diff --git a/core/runtime/tests/fake_core_build_identity.cpp b/core/runtime/tests/fake_core_build_identity.cpp new file mode 100644 index 0000000000..aea43e98c3 --- /dev/null +++ b/core/runtime/tests/fake_core_build_identity.cpp @@ -0,0 +1,8 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +extern "C" const char* trtmc_core_build_id() noexcept { + return "00000000000000000000000000000000"; +} diff --git a/core/runtime/tests/test_family_loader.cpp b/core/runtime/tests/test_family_loader.cpp index c2fb2a24eb..bb3f9c96c9 100644 --- a/core/runtime/tests/test_family_loader.cpp +++ b/core/runtime/tests/test_family_loader.cpp @@ -179,6 +179,18 @@ int main(int argc, char** argv) { check(incompatible_build_error.find("belongs to product build") != std::string::npos, "loader rejects a plugin from a different product build before its factory"); + const auto escaped_root = runtime_root.parent_path() / "escaped-runtime-root"; + std::filesystem::remove_all(escaped_root); + std::filesystem::create_directories(escaped_root); + std::filesystem::create_symlink(runtime_root / "libtrtmc_backend_fake.so", + escaped_root / "libtrtmc_backend_fake.so"); + std::filesystem::create_symlink(runtime_root / "libtrtmc_model_fake.so", + escaped_root / "libtrtmc_model_fake.so"); + const std::string escaped_root_error = load_error(bundle_path, escaped_root.string()); + check(escaped_root_error.find("escapes the selected runtime root") != std::string::npos, + "explicit loading rejects a DSO symlink that escapes the selected root"); + std::filesystem::remove_all(escaped_root); + const auto wrong_family_library = runtime_root / "libtrtmc_model_other.so"; const auto wrong_family_bundle = runtime_root / "wrong-family.bundle"; std::filesystem::copy_file(runtime_root / "libtrtmc_model_fake.so", wrong_family_library, diff --git a/pyproject.toml b/pyproject.toml index c7435458d9..aa581ad519 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,7 @@ dependencies = [ ] [project.scripts] +trtmc = "tensorrt_model_connect.native_cli:main" trtmc-bench = "trtmc_benchmark.cli:main" [project.optional-dependencies] diff --git a/tools/ci/package.py b/tools/ci/package.py index 6ebd14c29b..49058afb39 100644 --- a/tools/ci/package.py +++ b/tools/ci/package.py @@ -173,6 +173,8 @@ def validate(self, wheels: list[Path]) -> None: raise CiError(f"{wheel}: generated Python cache files are packaged") if "tensorrt_model_connect/__init__.py" not in names: raise CiError(f"{wheel}: Python core package is missing") + if "tensorrt_model_connect/native_cli.py" not in names: + raise CiError(f"{wheel}: native CLI console adapter is missing") if "trtmc_benchmark/__init__.py" not in names: raise CiError(f"{wheel}: Python benchmark application is missing") source_suffixes = { @@ -318,20 +320,24 @@ def validate(self, wheels: list[Path]) -> None: raise CiError( f"{wheel}: expected only unaliased TensorRT backend DSOs, found {backend_dsos}" ) - scripts = [name for name in names if name.endswith(".data/scripts/trtmc")] - script_cores = [ - name for name in names if name.endswith(".data/scripts/libtrtmc_core.so") - ] - script_runtimes = [ - name for name in names if name.endswith(".data/scripts/libtrtmc_runtime.so") + duplicate_native_payload = [ + name + for name in names + if ".data/scripts/" in name + and Path(name).name in {"trtmc", "libtrtmc_core.so", "libtrtmc_runtime.so"} ] - if len(scripts) != 1 or len(script_cores) != 1 or len(script_runtimes) != 1: - raise CiError(f"{wheel}: installed CLI payload is incomplete") + if duplicate_native_payload: + raise CiError(f"{wheel}: native product payload is duplicated in wheel scripts") entry_points = [name for name in names if name.endswith(".dist-info/entry_points.txt")] - if len(entry_points) != 1 or "trtmc-bench" not in archive.read(entry_points[0]).decode( - "utf-8" - ): - raise CiError(f"{wheel}: trtmc-bench console entrypoint is missing") + if len(entry_points) != 1: + raise CiError(f"{wheel}: console entrypoints are missing") + entrypoint_text = archive.read(entry_points[0]).decode("utf-8") + required_entrypoints = ( + "trtmc = tensorrt_model_connect.native_cli:main", + "trtmc-bench = trtmc_benchmark.cli:main", + ) + if not all(entrypoint in entrypoint_text for entrypoint in required_entrypoints): + raise CiError(f"{wheel}: required console entrypoints are missing") print(f"validated wheel={wheel} families={len(expected_families)}") diff --git a/tools/tests/test_architecture.py b/tools/tests/test_architecture.py index ca2fc53513..2516ec0c83 100644 --- a/tools/tests/test_architecture.py +++ b/tools/tests/test_architecture.py @@ -268,6 +268,7 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "core/builder/tensorrt_model_connect/bundle_writer.py", "core/builder/tensorrt_model_connect/graph_transform.py", "core/builder/tensorrt_model_connect/model_support.py", + "core/builder/tensorrt_model_connect/native_cli.py", "core/builder/tests/__init__.py", "core/builder/tests/test_build.py", "core/builder/tests/test_build_cli.py", @@ -275,6 +276,7 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "core/builder/tests/test_byok.py", "core/builder/tests/test_graph_transform.py", "core/builder/tests/test_model_support.py", + "core/builder/tests/test_native_cli.py", } expected_native = { "core/runtime/bundle/bundle_format.cpp", @@ -310,6 +312,7 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "core/runtime/include/trtmc/runtime/trt_backend.h", "core/runtime/include/trtmc/runtime/trt_module.h", "core/runtime/tests/fake_backend.cpp", + "core/runtime/tests/fake_core_build_identity.cpp", "core/runtime/tests/fake_family.cpp", "core/runtime/tests/test_bundle_format_v1.cpp", "core/runtime/tests/test_byok_shape_spec.cpp", @@ -359,7 +362,7 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "tools/tests/test_pr_metadata.py", "tools/tests/test_public_source_hygiene.py", } - expected_cmake = {"cmake/trtmcConfig.cmake.in"} + expected_cmake = {"cmake/product_build.h.in", "cmake/trtmcConfig.cmake.in"} expected_third_party = { "third_party/stb/stb_image.h", "third_party/stb/stb_image_resize2.h", @@ -1189,6 +1192,8 @@ def test_runtime_plugins_publish_one_exact_build_descriptor() -> None: assert "build_cohort" not in loader assert "TRTMC_BUILD_COHORT_ID" not in cmake assert "TRTMC_BUILD_ID" in cmake + assert "cmake/product_build.h.in" in cmake + assert "add_compile_definitions(TRTMC_BUILD_ID" not in cmake backends = { "core/runtime/tensorrt/trt_backend.cpp": "trt", diff --git a/tools/tests/test_new_ci.py b/tools/tests/test_new_ci.py index be484c51ea..8c78bfda28 100644 --- a/tools/tests/test_new_ci.py +++ b/tools/tests/test_new_ci.py @@ -803,6 +803,7 @@ def test_wheel_validation_requires_exact_new_payload(tmp_path: Path) -> None: wheel = tmp_path / "package.whl" with zipfile.ZipFile(wheel, "w") as archive: archive.writestr("tensorrt_model_connect/__init__.py", "") + archive.writestr("tensorrt_model_connect/native_cli.py", "") archive.writestr("trtmc_benchmark/__init__.py", "") archive.writestr("families/__init__.py", "") archive.writestr("tensorrt_model_connect/bin/trtmc", "") @@ -814,7 +815,9 @@ def test_wheel_validation_requires_exact_new_payload(tmp_path: Path) -> None: archive.writestr("tensorrt_model_connect/bin/libtrtmc_byok_tvm_ffi.so", "") archive.writestr( "package-0.1.dist-info/entry_points.txt", - "[console_scripts]\ntrtmc-bench = trtmc_benchmark.cli:main\n", + "[console_scripts]\n" + "trtmc = tensorrt_model_connect.native_cli:main\n" + "trtmc-bench = trtmc_benchmark.cli:main\n", ) archive.writestr( "package-0.1.dist-info/METADATA", @@ -824,9 +827,6 @@ def test_wheel_validation_requires_exact_new_payload(tmp_path: Path) -> None: "Provides-Extra: cutedsl\n" "Provides-Extra: test\n", ) - archive.writestr("package-0.1.data/scripts/trtmc", "") - archive.writestr("package-0.1.data/scripts/libtrtmc_core.so", "") - archive.writestr("package-0.1.data/scripts/libtrtmc_runtime.so", "") for family in family_names: archive.writestr( f"families/{family}/model.py", @@ -836,6 +836,22 @@ def test_wheel_validation_requires_exact_new_payload(tmp_path: Path) -> None: WheelArchiveValidator(CiContext(tmp_path, {})).validate([wheel]) + without_launcher = tmp_path / "without-launcher.whl" + with zipfile.ZipFile(wheel) as source, zipfile.ZipFile(without_launcher, "w") as output: + for entry in source.infolist(): + if entry.filename != "tensorrt_model_connect/native_cli.py": + output.writestr(entry, source.read(entry.filename)) + with pytest.raises(CiError, match="console adapter is missing"): + WheelArchiveValidator(CiContext(tmp_path, {})).validate([without_launcher]) + + duplicated = tmp_path / "duplicated.whl" + with zipfile.ZipFile(wheel) as source, zipfile.ZipFile(duplicated, "w") as output: + for entry in source.infolist(): + output.writestr(entry, source.read(entry.filename)) + output.writestr("package-0.1.data/scripts/trtmc", "") + with pytest.raises(CiError, match="duplicated in wheel scripts"): + WheelArchiveValidator(CiContext(tmp_path, {})).validate([duplicated]) + corrupt = tmp_path / "corrupt.whl" with zipfile.ZipFile(wheel) as source, zipfile.ZipFile(corrupt, "w") as output: for entry in source.infolist(): @@ -890,6 +906,8 @@ def plugin_source( plugin_id: str, implementation: str, build_id: str = "1234567890abcdef1234567890abcdef", + struct_size: str = "sizeof(struct PluginDescriptorV1)", + descriptor_version: int = 1, ) -> str: return f""" #include @@ -901,7 +919,7 @@ def plugin_source( const char *build_id; }}; static const struct PluginDescriptorV1 descriptor = {{ - sizeof(struct PluginDescriptorV1), 1, {kind}, "{plugin_id}", "{build_id}" + {struct_size}, {descriptor_version}, {kind}, "{plugin_id}", "{build_id}" }}; const struct PluginDescriptorV1 *trtmc_plugin_descriptor_v1(void) {{ return &descriptor; }} {implementation} @@ -929,6 +947,25 @@ def plugin_source( compile_library("libtrtmc_model_beta.so", plugin_source(2, "beta", "void beta_symbol(void) {}")) load_native_libraries(tmp_path, ("alpha", "beta")) + invalid_descriptors = ( + ("size=0", plugin_source(2, "beta", "", struct_size="0")), + ("version=2", plugin_source(2, "beta", "", descriptor_version=2)), + ("kind=1", plugin_source(1, "beta", "")), + ("id=gamma", plugin_source(2, "gamma", "")), + ( + "build=00000000000000000000000000000000", + plugin_source(2, "beta", "", build_id="00000000000000000000000000000000"), + ), + ) + for expected_error, source in invalid_descriptors: + compile_library("libtrtmc_model_beta.so", source) + with pytest.raises(CiError, match=expected_error): + load_native_libraries(tmp_path, ("alpha", "beta")) + + compile_library("libtrtmc_model_beta.so", "void beta_symbol(void) {}") + with pytest.raises(CiError, match="trtmc_plugin_descriptor_v1"): + load_native_libraries(tmp_path, ("alpha", "beta")) + compile_library( "libtrtmc_model_beta.so", plugin_source( diff --git a/website/docs/api/cli-reference.md b/website/docs/api/cli-reference.md index e02c18913d..0a41768042 100644 --- a/website/docs/api/cli-reference.md +++ b/website/docs/api/cli-reference.md @@ -59,12 +59,14 @@ Every execution command has this shape: trtmc COMMAND MODEL.bundle [--runtime-root DIR] [OPTIONS] ``` -When `--runtime-root` is omitted, the CLI searches the active runtime and CLI -installation, followed by colon-separated `TRTMC_RUNTIME_PATH` entries. It -does not search the current directory unless `.` is explicitly present in that -variable. One selected root must contain the requested backend and family DSOs; -their descriptors must declare the active product-build identity, expected -kind, and bundle ID before either factory is called. Common load options are: +When `--runtime-root` is omitted, the CLI checks the active Runtime directory, +followed by colon-separated `TRTMC_RUNTIME_PATH` entries. A wheel console +command replaces itself with the native CLI in the wheel's single native root. +The CLI does not scan `PATH`, Python installation layouts, or the current +directory unless `.` is explicitly present in `TRTMC_RUNTIME_PATH`. One +selected root must contain the requested backend and family DSOs; their +descriptors must declare the active product-build identity, expected kind, and +bundle ID before either factory is called. Common load options are: | Option | Contract | | --- | --- | diff --git a/website/docs/architecture/ai-native-horizontal-scaling.md b/website/docs/architecture/ai-native-horizontal-scaling.md index 9cfa63c925..af52a80beb 100644 --- a/website/docs/architecture/ai-native-horizontal-scaling.md +++ b/website/docs/architecture/ai-native-horizontal-scaling.md @@ -524,15 +524,25 @@ alias, or fallback. Source work remains family-local, but native artifacts from separate product builds are never mixed. The human-facing `trtmc` CLI may discover one plugin root before this control -transfer. The CLI owns only candidate enumeration and search order: the active -runtime, the installation belonging to the running executable, then explicitly -configured `TRTMC_RUNTIME_PATH` entries. The current directory participates -only when a user explicitly adds `.` to that variable. For each candidate, the -CLI asks the model-agnostic Runtime Loader contract whether the requested -root-local backend, family, and optional BYOK files exist. It does not derive -DSO names, inspect native metadata, or load discovery candidates. The public -C++ load interface still receives one explicit root, and a selected-root load -failure never triggers another search. +transfer. A product installation owns one plugin root containing Core, Runtime, +backend, extension, and family artifacts. A native CMake install may keep the +CLI in its conventional sibling `bin` directory; the already loaded Runtime +still identifies the `lib` plugin root. A wheel console adapter instead +replaces itself with the native CLI inside its single native product directory. +The CLI therefore owns only candidate enumeration and search order: the active +Runtime directory, then explicitly configured `TRTMC_RUNTIME_PATH` entries. +The current directory participates only when a user explicitly adds `.` to +that variable. For each candidate, the CLI asks the model-agnostic Runtime +Loader contract whether the requested root-local backend, family, and optional +BYOK files exist. It does not derive DSO names, inspect native metadata, scan +installation layouts, or load discovery candidates. The public C++ load +interface still receives one explicit root, and a selected-root load failure +never triggers another search. + +Before calling bundle or loader C++ interfaces, the CLI compares its embedded +product-build identity with the C identity symbols exported by the active +Runtime and Core. This keeps executable validation at the application seam; +the Runtime Loader continues to own Core and plugin validation. ### Task API diff --git a/website/docs/architecture/build-system.md b/website/docs/architecture/build-system.md index 0e88a85da3..22904a2a23 100644 --- a/website/docs/architecture/build-system.md +++ b/website/docs/architecture/build-system.md @@ -23,11 +23,19 @@ private dependencies, warnings, tests, output name, and install rule. Adding a normal family never changes a central model source list. The wheel packages `core/builder/tensorrt_model_connect`, the top-level -`families` package, benchmark Python code, and installed native binaries/DSOs. +`families` package, benchmark Python code, and one native product directory at +`tensorrt_model_connect/bin`. The `trtmc` console entry point is a Python +adapter that uses `exec` to replace itself with the native CLI in that +directory; it does not duplicate native files into the wheel scripts area. Optional family dependencies remain in each `families//requirements.txt`; package validation does not import every family implementation. +CMake generates one private product-build header for all native targets in a +build tree. It is not installed. A release build may set the 32-character +`TRTMC_BUILD_ID` explicitly for coordinated reproducibility; never reuse that +identity across independently compiled native artifact sets. + ## Typical source build ```bash diff --git a/website/docs/architecture/runtime-plugins.md b/website/docs/architecture/runtime-plugins.md index 9bf9cee4bc..327cbab1e6 100644 --- a/website/docs/architecture/runtime-plugins.md +++ b/website/docs/architecture/runtime-plugins.md @@ -8,7 +8,7 @@ one family DSO named `libtrtmc_model_.so`. The bundle header names exactly one family and backend. Each DSO publishes the model-agnostic `trtmc_plugin_descriptor_v1` interface with its product-build identity, kind, and ID. The loader opens the exact backend and family paths -from one explicit plugin root, validates both descriptors, resolves +from one selected plugin root, validates both descriptors, resolves `trtmc_create_family`, and receives an implementation of an abstract Task interface. @@ -23,6 +23,11 @@ another product build are rejected even when they expose the same descriptor version. Core and runtime also expose and compare their product-build identities before the runtime calls a core C++ interface. +The native CLI also embeds that identity and compares it with the Runtime and +Core C symbols before calling bundle or loader C++ interfaces. This check is +application assembly, not family selection, so it remains outside the generic +Runtime Loader. + Plugin roots are trusted native-code inputs. ELF constructors run during `dlopen`, before the descriptor can be called. The loader therefore never opens discovery candidates speculatively, never falls back after a selected load @@ -33,3 +38,10 @@ There is no runtime-strategy map, central manifest, sibling fallback, or hot plugin marketplace. To extend an existing family, edit only its `families//runtime/` implementation and family-owned tests. To add a new family, follow [Add a Model Family](../extend/add-model-family.md). + +Every family factory declares +`TRTMC_DEFINE_FAMILY_PLUGIN_V1("")`; every backend declares +`TRTMC_DEFINE_BACKEND_PLUGIN_V1("")`. Runtime extensions use the +generic descriptor macro with `PluginKind::kRuntimeExtension`. These macros +only publish the fixed descriptor symbol and build identity. They are required +because calling a factory first would cross an unvalidated C++ interface. diff --git a/website/docs/extend/add-model-family.md b/website/docs/extend/add-model-family.md index 16e8c92cd9..743cbb52f1 100644 --- a/website/docs/extend/add-model-family.md +++ b/website/docs/extend/add-model-family.md @@ -76,6 +76,17 @@ concrete implementation of an abstract interface in `trtmc/task.h`. The family pipeline depends on and implements `trtmc/task.h`; `trtmc/task.h` never includes or links a family. +The same `plugin.cpp` must publish the model-agnostic descriptor next to its +factory: + +```cpp +TRTMC_DEFINE_FAMILY_PLUGIN_V1("my_family") +``` + +This declaration supplies the family ID, plugin kind, and coordinated product +build identity used before the factory crosses a C++ interface. It does not +register the family centrally or move any family behavior into shared code. + If the family graph contains distributed collectives, that same family owns its communicator setup and NCCL loading. A replicated plan that only selects a rank-specific section must not load NCCL. diff --git a/website/docs/extend/add-optimized-runtime.md b/website/docs/extend/add-optimized-runtime.md index 92fe8a32af..680aac4c31 100644 --- a/website/docs/extend/add-optimized-runtime.md +++ b/website/docs/extend/add-optimized-runtime.md @@ -38,7 +38,6 @@ python -m tensorrt_model_connect build MODEL \ -o model-rtx.bundle trtmc run model-rtx.bundle \ - --runtime-root /opt/trtmc/lib \ --runtime-cache /tmp/trtmc-rtx.cache \ --cuda-graphs \ --prompt "Hello" diff --git a/website/docs/features/config-and-backends.md b/website/docs/features/config-and-backends.md index f36bc18449..9c4041e01e 100644 --- a/website/docs/features/config-and-backends.md +++ b/website/docs/features/config-and-backends.md @@ -35,7 +35,6 @@ Both are loaded from the same required runtime root as the family DSO: ```bash trtmc run model.bundle \ - --runtime-root /opt/trtmc/lib \ --prompt "Hello" ``` diff --git a/website/docs/features/multi-device.md b/website/docs/features/multi-device.md index 35a3c996f7..0db5d30145 100644 --- a/website/docs/features/multi-device.md +++ b/website/docs/features/multi-device.md @@ -45,7 +45,6 @@ mpirun --tag-output -np 4 \ -x CUDA_VISIBLE_DEVICES \ -x TRTMC_NCCL_RENDEZVOUS \ trtmc run model-tp4.bundle \ - --runtime-root /opt/trtmc/lib \ --prompt "Hello" ``` diff --git a/website/docs/features/sampling.md b/website/docs/features/sampling.md index eb5c3b2afb..d1632840a2 100644 --- a/website/docs/features/sampling.md +++ b/website/docs/features/sampling.md @@ -47,7 +47,7 @@ to disabled top-p behavior. ### CLI ```bash -$TRTMC run bundle.bundle --runtime-root /opt/trtmc/lib \ +$TRTMC run bundle.bundle \ --prompt "Once upon a time" \ --temperature 0.7 --top-p 0.9 --min-p 0.05 --top-k 50 \ --repetition-penalty 1.05 --seed 42 diff --git a/website/docs/getting-started/build-and-run.md b/website/docs/getting-started/build-and-run.md index be3bfbdd0e..ed7d5b6f2f 100644 --- a/website/docs/getting-started/build-and-run.md +++ b/website/docs/getting-started/build-and-run.md @@ -7,12 +7,6 @@ checkpoint, task, precision, topology, dependency, and validation support is generated on [Models & Recipes](../models-recipes/overview.md) from the current family-owned manifests. -Set the runtime root once for the examples: - -```bash -export TRTMC_RUNTIME_ROOT=/opt/trtmc/lib -``` - ## Text generation ```bash @@ -22,7 +16,6 @@ python -m tensorrt_model_connect build Qwen/Qwen3-0.6B \ --output /tmp/qwen.bundle trtmc run /tmp/qwen.bundle \ - --runtime-root "$TRTMC_RUNTIME_ROOT" \ --prompt "What is the capital of France? Answer in one word." \ --max-new-tokens 10 \ --temperature 0 \ @@ -42,7 +35,6 @@ python -m tensorrt_model_connect build Qwen/Qwen2.5-VL-3B-Instruct \ --output /tmp/qwen25vl.bundle trtmc run /tmp/qwen25vl.bundle \ - --runtime-root "$TRTMC_RUNTIME_ROOT" \ --prompt "Describe this image." \ --image families/qwen_vl/tests/data/test_img.jpeg \ --max-new-tokens 48 @@ -59,7 +51,6 @@ python -m tensorrt_model_connect build openai/whisper-large-v3-turbo \ --output /tmp/whisper.bundle trtmc transcribe /tmp/whisper.bundle \ - --runtime-root "$TRTMC_RUNTIME_ROOT" \ --input families/whisper/tests/data/Recording.wav \ --max-output-tokens 224 ``` @@ -70,7 +61,6 @@ python -m tensorrt_model_connect build nvidia/magpie_tts_multilingual_357m \ --output /tmp/magpie.bundle trtmc generate-audio /tmp/magpie.bundle \ - --runtime-root "$TRTMC_RUNTIME_ROOT" \ --prompt "A clear short test sentence." \ --output /tmp/magpie.wav ``` @@ -91,7 +81,6 @@ python -m tensorrt_model_connect build nvidia/segformer-b0-finetuned-ade-512-512 --output /tmp/segformer.bundle trtmc segment /tmp/segformer.bundle \ - --runtime-root "$TRTMC_RUNTIME_ROOT" \ --image families/segformer/tests/data/test_img.jpeg ``` @@ -109,7 +98,6 @@ python -m tensorrt_model_connect build amazon/chronos-bolt-tiny \ --output /tmp/chronos.bundle trtmc forecast /tmp/chronos.bundle \ - --runtime-root "$TRTMC_RUNTIME_ROOT" \ --input /path/to/history.f32 ``` diff --git a/website/docs/getting-started/glossary.md b/website/docs/getting-started/glossary.md index e5471e7484..41483248cb 100644 --- a/website/docs/getting-started/glossary.md +++ b/website/docs/getting-started/glossary.md @@ -26,10 +26,10 @@ title: Glossary | Task | User-visible behavior. | Abstract interfaces such as text generation, transcription, segmentation, embedding, and forecast. | | Backend | Engine implementation. | `trt` or optional `trt_rtx`, selected by the bundle header. | | DSO | Linux shared library loaded at runtime. | Core, loader, one backend, and exactly one `libtrtmc_model_.so`. | -| Runtime root | Explicit DSO directory. | Required by every native execution command; no fallback search exists. | +| Runtime root | One selected DSO directory. | The CLI discovers or explicitly selects it; the Runtime Loader never searches or falls back. | | Precision | Numeric representation. | A build request such as FP32, FP16, or BF16 that the family validates. | | Quantization | Lower-precision graph/weights such as FP8. | Entirely family-owned and qualified per exact checkpoint/path. | -| ABI | Binary compatibility contract. | Runtime DSOs and TensorRT plans must match their software/hardware cohort. | +| ABI | Binary compatibility contract. | Runtime DSOs must share one exact product build and use a supported TensorRT environment. | ## Project building blocks diff --git a/website/docs/getting-started/quick-start.md b/website/docs/getting-started/quick-start.md index 3e5704aa53..febafbab76 100644 --- a/website/docs/getting-started/quick-start.md +++ b/website/docs/getting-started/quick-start.md @@ -53,9 +53,12 @@ The CLI reads the bundle family and backend, then selects the first complete, single-directory plugin root in this order: 1. the directory containing the active `libtrtmc_runtime.so`; -2. the installation belonging to the active `trtmc` selected through `PATH`, - including native CMake and wheel install layouts; -3. colon-separated directories in `TRTMC_RUNTIME_PATH`. +2. colon-separated directories in `TRTMC_RUNTIME_PATH`. + +The wheel console command replaces itself with the native CLI stored beside +Core, Runtime, backend, and family DSOs. Native installs resolve their already +loaded Runtime directory in the same way, so discovery does not inspect Python +installation layouts or scan `PATH` directories. A complete GPT-2 TensorRT plugin root contains root-local `libtrtmc_backend_trt.so` and `libtrtmc_model_gpt2.so` files. Candidates are @@ -65,12 +68,13 @@ requires their descriptors to match the active product build, plugin kinds, and bundle IDs before it calls either factory. A mismatched selected root fails immediately without falling back to another installation. -The Runtime Loader also verifies that its already loaded Core belongs to the -same product build before reading the bundle. +Before bundle inspection crosses a C++ interface, the CLI verifies that its +own product-build identity matches the already loaded Runtime and Core. The +Runtime Loader repeats the Core check and validates selected plugins at load. -The CLI prints the automatically selected directory. If more than one installed -wheel root matches structurally, select one with `--runtime-root DIR`. An -explicit root bypasses discovery but not build and identity validation. The -current directory is not searched implicitly; use `TRTMC_RUNTIME_PATH=.` when -that behavior is intended. `LD_LIBRARY_PATH` remains a platform-loader setting -evaluated before the CLI starts. +The CLI prints the automatically selected directory. Use `--runtime-root DIR` +to override it with one exact root. An explicit root bypasses discovery but not +build and identity validation. The current directory is not searched +implicitly; use `TRTMC_RUNTIME_PATH=.` when that behavior is intended. +`LD_LIBRARY_PATH` remains a platform-loader setting evaluated before the CLI +starts. diff --git a/website/docs/getting-started/troubleshooting.md b/website/docs/getting-started/troubleshooting.md index a8bf1bc807..0ee4217c60 100644 --- a/website/docs/getting-started/troubleshooting.md +++ b/website/docs/getting-started/troubleshooting.md @@ -15,9 +15,9 @@ Identify the first boundary that fails in the [Quick Start](quick-start.md). | Build OOM or disk failure | Requested checkpoint, shape, precision, and cache capacity | Use the exact family manifest/profile or free capacity; retain the first error. | | No family or multiple families match | Root checkpoint identity metadata | Use a supported exact checkpoint; do not add prefix/fallback matching. | | Bundle inspection fails | Partial/corrupt bundle | Rebuild; failed builds must not publish a partial output. | -| Family/backend DSO missing | `--runtime-root` contents | Confirm `libtrtmc_core.so`, `libtrtmc_runtime.so`, selected backend, and exact family DSO are together. | +| Family/backend DSO missing | Selected runtime-root contents | Confirm `libtrtmc_core.so`, `libtrtmc_runtime.so`, selected backend, and exact family DSO are together. | | Task mismatch | Bundle `task` versus CLI command | Use the Task command named by the family manifest/header. | -| TensorRT/DSO ABI error | Mixed software or hardware cohort | Run with a compatible environment and rebuild the bundle when required. | +| TensorRT/DSO ABI error | Mixed native product builds or incompatible hardware/software | Use one product build in a compatible environment and rebuild the bundle when required. | | Output differs | Revision, input framing, precision, sampling, oracle | Reproduce the exact family testcase before changing code or thresholds. | Collect the source revision, model ID/revision, complete build/run commands, diff --git a/website/docs/reference/profiling.md b/website/docs/reference/profiling.md index b94e6fb472..3852311440 100644 --- a/website/docs/reference/profiling.md +++ b/website/docs/reference/profiling.md @@ -14,7 +14,6 @@ First prove that the same bundle and request complete normally: ```bash trtmc run model.bundle \ - --runtime-root /opt/trtmc/lib \ --prompt "The capital of France is" \ --max-new-tokens 20 \ --seed 1234 @@ -39,9 +38,9 @@ Hold these inputs constant across comparisons: - Use family-owned tests and reference comparisons to establish correctness before interpreting a speedup. -Profile loading separately from steady-state task execution. The public CLI -requires `--runtime-root`; include its exact directory in the evidence so the -loaded shared objects are reproducible. +Profile loading separately from steady-state task execution. Record the exact +runtime directory printed by automatic discovery in the evidence so the loaded +shared objects are reproducible. ## Interpreting results diff --git a/website/docs/tutorials/advanced/multi-device-inference.md b/website/docs/tutorials/advanced/multi-device-inference.md index 938f528357..927381cbf4 100644 --- a/website/docs/tutorials/advanced/multi-device-inference.md +++ b/website/docs/tutorials/advanced/multi-device-inference.md @@ -37,7 +37,6 @@ Use the MPI launcher and rank count required by the family-owned manifest: ```bash mpirun -n 4 trtmc run qwen-tp4.bundle \ - --runtime-root /opt/trtmc/lib \ --prompt "Explain tensor parallelism." \ --max-new-tokens 64 \ --seed 1234 diff --git a/website/docs/tutorials/advanced/quantization-and-runtime-knobs.md b/website/docs/tutorials/advanced/quantization-and-runtime-knobs.md index a3dcb89f26..790d81c46c 100644 --- a/website/docs/tutorials/advanced/quantization-and-runtime-knobs.md +++ b/website/docs/tutorials/advanced/quantization-and-runtime-knobs.md @@ -49,7 +49,6 @@ Every execution command requires the installed runtime directory: ```bash trtmc run qwen.bundle \ - --runtime-root /opt/trtmc/lib \ --runtime-cache /tmp/trtmc-cache \ --cuda-graphs \ --kv-cache-size 4096 \ diff --git a/website/docs/tutorials/beginner/inspect-bundles.md b/website/docs/tutorials/beginner/inspect-bundles.md index 5bf49ac16a..a51f35dce9 100644 --- a/website/docs/tutorials/beginner/inspect-bundles.md +++ b/website/docs/tutorials/beginner/inspect-bundles.md @@ -50,8 +50,8 @@ The relevant boundaries are: 1. If inspection cannot parse the header, investigate the build or artifact. 2. If required sections are absent, investigate the family builder. -3. If inspection succeeds but loading fails, verify `--runtime-root` contains - the installed core, backend, and family libraries from a compatible build. +3. If inspection succeeds but loading fails, verify the selected runtime root + contains the installed core, backend, and family libraries from one build. 4. If loading succeeds but a request fails, route the issue to the family task contract and model-owned tests. diff --git a/website/docs/tutorials/beginner/text-generation.md b/website/docs/tutorials/beginner/text-generation.md index 74f81038c5..de7b731f2c 100644 --- a/website/docs/tutorials/beginner/text-generation.md +++ b/website/docs/tutorials/beginner/text-generation.md @@ -13,7 +13,6 @@ reuses `./gpt2.bundle`. trtmc inspect ./gpt2.bundle trtmc run ./gpt2.bundle \ - --runtime-root /opt/trtmc/lib \ --prompt "Explain why KV caches help decoding." \ --max-new-tokens 80 \ --temperature 0.7 \ diff --git a/website/docs/tutorials/intermediate/canary-decoding.md b/website/docs/tutorials/intermediate/canary-decoding.md index 73fdc4d986..f7a2b6c26f 100644 --- a/website/docs/tutorials/intermediate/canary-decoding.md +++ b/website/docs/tutorials/intermediate/canary-decoding.md @@ -24,7 +24,6 @@ metadata, runtime orchestration, and validation. ```bash trtmc transcribe /tmp/canary-1b-v2.bundle \ - --runtime-root /opt/trtmc/lib \ --input /data/input.wav \ --max-output-tokens 80 \ --source-language en \ @@ -40,7 +39,6 @@ segmentation are request options: ```bash trtmc transcribe /tmp/canary-1b-v2.bundle \ - --runtime-root /opt/trtmc/lib \ --input /data/english.wav \ --source-language en \ --target-language fr \ diff --git a/website/docs/tutorials/intermediate/diffusion-and-time-series.md b/website/docs/tutorials/intermediate/diffusion-and-time-series.md index 0ff20ccdcb..a2739fa391 100644 --- a/website/docs/tutorials/intermediate/diffusion-and-time-series.md +++ b/website/docs/tutorials/intermediate/diffusion-and-time-series.md @@ -17,7 +17,6 @@ python -m tensorrt_model_connect build black-forest-labs/FLUX.1-schnell \ --image-width 1024 trtmc generate-image flux.bundle \ - --runtime-root /opt/trtmc/lib \ --prompt "A brass robot reading beside a window" \ --output robot.png \ --height 1024 \ @@ -41,7 +40,6 @@ python -m tensorrt_model_connect build Wan-AI/Wan2.1-T2V-1.3B \ --video-num-frames 81 trtmc generate-video wan.bundle \ - --runtime-root /opt/trtmc/lib \ --prompt "Ocean waves under moonlight" \ --output waves.mp4 \ --height 480 \ @@ -63,7 +61,6 @@ python -m tensorrt_model_connect build amazon/chronos-bolt-tiny \ --precision fp32 trtmc forecast chronos.bundle \ - --runtime-root /opt/trtmc/lib \ --input history.f32 \ --frequency H ``` diff --git a/website/docs/tutorials/intermediate/multimodal-and-speech.md b/website/docs/tutorials/intermediate/multimodal-and-speech.md index c04854e8ce..e772a1a881 100644 --- a/website/docs/tutorials/intermediate/multimodal-and-speech.md +++ b/website/docs/tutorials/intermediate/multimodal-and-speech.md @@ -19,7 +19,6 @@ python -m tensorrt_model_connect build Qwen/Qwen2.5-VL-3B-Instruct \ --max-sequence-length 384 trtmc run qwen-vl.bundle \ - --runtime-root /opt/trtmc/lib \ --image sample.png \ --prompt "Describe the image." \ --max-new-tokens 80 @@ -33,12 +32,10 @@ between vision and decoder engines are family-owned. The shared builder has no ```bash trtmc transcribe whisper.bundle \ - --runtime-root /opt/trtmc/lib \ --input recording.wav \ --max-output-tokens 128 trtmc transcribe-streaming nemotron-asr.bundle \ - --runtime-root /opt/trtmc/lib \ --input recording.wav \ --chunk-samples 16000 \ --max-new-tokens 128 @@ -52,13 +49,11 @@ family supports them. ```bash trtmc generate-audio audio.bundle \ - --runtime-root /opt/trtmc/lib \ --prompt "A calm spoken welcome" \ --output welcome.wav \ --seed 1234 trtmc speech-session voicechat.bundle \ - --runtime-root /opt/trtmc/lib \ --input question.wav \ --output answer.wav \ --system-prompt "Answer concisely." \ diff --git a/website/docs/user-guides/configure-runtime.md b/website/docs/user-guides/configure-runtime.md index 82f3f62dca..f3463ebcad 100644 --- a/website/docs/user-guides/configure-runtime.md +++ b/website/docs/user-guides/configure-runtime.md @@ -18,7 +18,6 @@ configs. Family-only state remains in family sections and implementation code. ```bash trtmc run model.bundle \ - --runtime-root /opt/trtmc/lib \ --kv-cache-size 4GiB \ --prompt "Hello" \ --temperature 0 diff --git a/website/docs/user-guides/image-video-generation.md b/website/docs/user-guides/image-video-generation.md index 0af4dc7ac8..4bb14461ab 100644 --- a/website/docs/user-guides/image-video-generation.md +++ b/website/docs/user-guides/image-video-generation.md @@ -16,7 +16,6 @@ python -m tensorrt_model_connect build MODEL_ID \ --output media.bundle trtmc generate-video media.bundle \ - --runtime-root /opt/trtmc/lib \ --prompt "A sunrise over a mountain lake" \ --output frames \ --num-steps 28 \ @@ -30,17 +29,15 @@ file plus comma-separated seeds for a family that declares the batch Task. ```bash trtmc classify classifier.bundle \ - --runtime-root /opt/trtmc/lib --image input.jpg + --image input.jpg trtmc segment segmenter.bundle \ - --runtime-root /opt/trtmc/lib --image input.jpg + --image input.jpg trtmc segment-prompted prompted.bundle \ - --runtime-root /opt/trtmc/lib \ --image input.jpg --point-x 0.5 --point-y 0.5 --foreground true trtmc geometry moge.bundle \ - --runtime-root /opt/trtmc/lib \ --image input.jpg --output geometry-output ``` diff --git a/website/docs/user-guides/multimodal-speech.md b/website/docs/user-guides/multimodal-speech.md index 95a7f89b71..faedef6e25 100644 --- a/website/docs/user-guides/multimodal-speech.md +++ b/website/docs/user-guides/multimodal-speech.md @@ -7,7 +7,6 @@ description: Vision-language, transcription, audio generation, and speech-sessio ```bash trtmc run vision-language.bundle \ - --runtime-root /opt/trtmc/lib \ --image input.jpg \ --prompt "Describe this image in one sentence." \ --max-new-tokens 48 @@ -20,7 +19,6 @@ request values belong to the selected family. ```bash trtmc transcribe speech-to-text.bundle \ - --runtime-root /opt/trtmc/lib \ --input input.wav \ --beam-size 1 \ --source-language en \ @@ -38,12 +36,10 @@ contracts. ```bash trtmc generate-audio text-to-audio.bundle \ - --runtime-root /opt/trtmc/lib \ --prompt "A clear short test sentence." \ --output output.wav trtmc speak speech-to-speech.bundle \ - --runtime-root /opt/trtmc/lib \ --input input.wav \ --output response.wav ``` @@ -54,7 +50,6 @@ application for a persistent local session: ```bash trtmc speech-session nemotron-voicechat.bundle \ - --runtime-root /opt/trtmc/lib \ --input input.wav \ --output response.wav \ --timeout-ms 30000 diff --git a/website/docs/user-guides/text-generation.md b/website/docs/user-guides/text-generation.md index 226576e7c1..18a185640d 100644 --- a/website/docs/user-guides/text-generation.md +++ b/website/docs/user-guides/text-generation.md @@ -7,7 +7,6 @@ Build and inspect an exact text checkpoint, then call the native Task: ```bash trtmc run model.bundle \ - --runtime-root /opt/trtmc/lib \ --prompt "Summarize TensorRT in one sentence." \ --max-new-tokens 48 \ --temperature 0 \ @@ -18,7 +17,6 @@ For reproducible stochastic sampling, fix every sampling input: ```bash trtmc run model.bundle \ - --runtime-root /opt/trtmc/lib \ --prompt "Write a two-line GPU poem." \ --max-new-tokens 64 \ --temperature 0.8 \ diff --git a/website/docs/user-guides/time-series.md b/website/docs/user-guides/time-series.md index d60b852535..384b8ddf66 100644 --- a/website/docs/user-guides/time-series.md +++ b/website/docs/user-guides/time-series.md @@ -9,7 +9,6 @@ input files. ```bash trtmc forecast forecast.bundle \ - --runtime-root /opt/trtmc/lib \ --input history.f32 ``` @@ -17,7 +16,6 @@ Other model contracts can use: ```bash trtmc solve operator.bundle \ - --runtime-root /opt/trtmc/lib \ --branch branch.f32 \ --trunk trunk.f32 ``` From b2e13b50e05ce1a51fbca1cc77940ec992957295 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Tue, 8 Sep 2026 13:36:18 +0000 Subject: [PATCH 06/13] fix(ci): materialize isolated runtime roots The family E2E harness staged TRTMC libraries as symlinks to the installed wheel, so root canonicalization correctly rejected every isolated runtime as escaping its selected directory. Copy the selected Core, backend, and family DSOs into the temporary root while retaining symlinks only for third-party RUNPATH dependencies. Signed-off-by: chaofengw --- tools/ci/e2e.py | 3 ++- tools/tests/test_new_ci.py | 23 +++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/tools/ci/e2e.py b/tools/ci/e2e.py index 886112a08b..089430c00c 100644 --- a/tools/ci/e2e.py +++ b/tools/ci/e2e.py @@ -8,6 +8,7 @@ import json import re import shlex +import shutil import tempfile import xml.etree.ElementTree as ET from collections import Counter @@ -357,7 +358,7 @@ def _isolated_runtime_root(self, runtime_root: Path, family: str): isolated = root / "tensorrt_model_connect/bin" isolated.mkdir(parents=True) for name in required: - (isolated / name).symlink_to((runtime_root / name).resolve()) + shutil.copy2(runtime_root / name, isolated / name) # Preserve only non-family wheel dependencies expected by RUNPATH. site_packages = runtime_root.parent.parent diff --git a/tools/tests/test_new_ci.py b/tools/tests/test_new_ci.py index 8c78bfda28..87864dd43d 100644 --- a/tools/tests/test_new_ci.py +++ b/tools/tests/test_new_ci.py @@ -222,6 +222,29 @@ def test_selective_e2e_calls_family_tests_directly( ] +def test_isolated_runtime_root_materializes_root_local_trtmc_libraries( + tmp_path: Path, +) -> None: + runtime = tmp_path / "runtime" + runtime.mkdir() + required = ( + "libtrtmc_core.so", + "libtrtmc_backend_trt.so", + "libtrtmc_model_beta.so", + ) + for name in required: + (runtime / name).write_text(name, encoding="utf-8") + + runner = E2ERunner(RecordingContext(tmp_path, {})) + with runner._isolated_runtime_root(runtime, "beta") as isolated: + for name in required: + staged = isolated / name + assert staged.is_file() + assert not staged.is_symlink() + assert staged.resolve().parent == isolated.resolve() + assert staged.read_text(encoding="utf-8") == name + + def test_family_with_only_hardware_tests_accepts_exact_empty_cpu_result( tmp_path: Path, ) -> None: From 38997380c46f9fddf576f64a7b6c2719e3913d37 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Wed, 9 Sep 2026 03:24:09 +0000 Subject: [PATCH 07/13] fix(runtime): complete product descriptor migration Add plugin ABI descriptors to families merged after this branch diverged. Keep shared package and E2E validation model-agnostic so future family additions do not require central CI edits. Signed-off-by: chaofengw --- families/timm_convnext/runtime/plugin.cpp | 2 ++ families/timm_ghostnet/runtime/plugin.cpp | 2 ++ families/timm_hrnet/runtime/plugin.cpp | 2 ++ families/timm_inception_resnet/runtime/plugin.cpp | 2 ++ families/timm_inception_v4/runtime/plugin.cpp | 2 ++ families/timm_regnet/runtime/plugin.cpp | 2 ++ families/timm_senet/runtime/plugin.cpp | 2 ++ families/timm_seresnet/runtime/plugin.cpp | 2 ++ families/timm_xception/runtime/plugin.cpp | 2 ++ tools/ci/package.py | 7 +++++-- tools/tests/test_architecture.py | 1 + tools/tests/test_new_ci.py | 11 +++++++---- 12 files changed, 31 insertions(+), 6 deletions(-) diff --git a/families/timm_convnext/runtime/plugin.cpp b/families/timm_convnext/runtime/plugin.cpp index cb7b8a75a9..0024baf4b4 100644 --- a/families/timm_convnext/runtime/plugin.cpp +++ b/families/timm_convnext/runtime/plugin.cpp @@ -50,6 +50,8 @@ std::unique_ptr load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector None: ) if not version.stdout.startswith("trtmc "): raise CiError("installed trtmc CLI returned an invalid version") + fixture_family = min(expected) with tempfile.TemporaryDirectory(prefix="trtmc-installed-wheel-") as directory: bundle = Path(directory) / "inspect.bundle" subprocess.run( @@ -440,9 +441,11 @@ def validate(self, wheel: Path) -> None: "from pathlib import Path; " "from tensorrt_model_connect.bundle_writer import BundleWriter; " "writer = BundleWriter(Path(__import__('sys').argv[1])); " - "writer.set_header(family='gpt2', task='text_generation', backend='trt'); " + "writer.set_header(family=__import__('sys').argv[2], " + "task='package_validation', backend='trt'); " "writer.finish()", bundle, + fixture_family, ], check=True, cwd=Path("/tmp"), @@ -457,7 +460,7 @@ def validate(self, wheel: Path) -> None: env=environment, ) metadata = json.loads(inspected.stdout) - if metadata.get("family") != "gpt2" or metadata.get("backend") != "trt": + if metadata.get("family") != fixture_family or metadata.get("backend") != "trt": raise CiError("installed trtmc CLI failed bundle inspection") executed = subprocess.run( [executable, "run", bundle], diff --git a/tools/tests/test_architecture.py b/tools/tests/test_architecture.py index 2516ec0c83..c425ce54a3 100644 --- a/tools/tests/test_architecture.py +++ b/tools/tests/test_architecture.py @@ -805,6 +805,7 @@ def dependency_lines(path: Path) -> list[str]: package_validation = (REPO / "tools/ci/package.py").read_text(encoding="utf-8") assert 'import_module(f"families.{family}.model")' not in package_validation + assert re.search(r"set_header\(family=['\"]", package_validation) is None def test_family_reference_consumers_declare_their_source() -> None: diff --git a/tools/tests/test_new_ci.py b/tools/tests/test_new_ci.py index 87864dd43d..d4a5bcfa33 100644 --- a/tools/tests/test_new_ci.py +++ b/tools/tests/test_new_ci.py @@ -414,7 +414,10 @@ def test_e2e_rejects_multiple_family_environments(tmp_path: Path) -> None: def test_e2e_nonexistent_testcase_fails_closed(tmp_path: Path) -> None: - repository = Path(__file__).resolve().parents[2] + family = tmp_path / "families/alpha" + (family / "tests").mkdir(parents=True) + (family / "model.py").write_text("def build(request, writer): pass\n") + (family / "tests/test_e2e.py").write_text("def test_e2e(): pass\n") binary = tmp_path / "trtmc" binary.write_text("") runtime = tmp_path / "runtime" @@ -422,14 +425,14 @@ def test_e2e_nonexistent_testcase_fails_closed(tmp_path: Path) -> None: for name in ( "libtrtmc_core.so", "libtrtmc_backend_trt.so", - "libtrtmc_model_gpt2.so", + "libtrtmc_model_alpha.so", ): (runtime / name).write_text("") native_build = tmp_path / "native-build" native_build.mkdir() (native_build / "CTestTestfile.cmake").write_text("") context = RecordingContext( - repository, + tmp_path, { "TRTMC_BINARY": str(binary), "TRTMC_RUNTIME_ROOT": str(runtime), @@ -439,7 +442,7 @@ def test_e2e_nonexistent_testcase_fails_closed(tmp_path: Path) -> None: context.missing_e2e_testcases.add("does-not-exist") with pytest.raises(CiError, match="missing requested E2E testcase: does-not-exist"): - E2ERunner(context)._run(("gpt2",), ("does-not-exist",)) + E2ERunner(context)._run(("alpha",), ("does-not-exist",)) def test_pipeline_exposes_only_active_stages(tmp_path: Path) -> None: From cd91caae1fe87acad00f9995e79fb8e8f9e3b44b Mon Sep 17 00:00:00 2001 From: chaofengw Date: Wed, 9 Sep 2026 08:34:32 +0000 Subject: [PATCH 08/13] fix(ci): stage complete runtime cohorts Include libtrtmc_runtime.so in isolated E2E roots so source-built qualification helpers cannot mix their Runtime with a wheel-provided Core. Signed-off-by: chaofengw --- tools/ci/e2e.py | 1 + tools/tests/test_new_ci.py | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/tools/ci/e2e.py b/tools/ci/e2e.py index 089430c00c..84620aa206 100644 --- a/tools/ci/e2e.py +++ b/tools/ci/e2e.py @@ -346,6 +346,7 @@ def _family_testcases(self, family: str) -> tuple[str, ...]: def _isolated_runtime_root(self, runtime_root: Path, family: str): required = ( "libtrtmc_core.so", + "libtrtmc_runtime.so", "libtrtmc_backend_trt.so", f"libtrtmc_model_{family}.so", ) diff --git a/tools/tests/test_new_ci.py b/tools/tests/test_new_ci.py index d4a5bcfa33..d338770de8 100644 --- a/tools/tests/test_new_ci.py +++ b/tools/tests/test_new_ci.py @@ -146,6 +146,7 @@ def test_selective_e2e_calls_family_tests_directly( runtime = tmp_path / "runtime" runtime.mkdir() (runtime / "libtrtmc_core.so").write_text("") + (runtime / "libtrtmc_runtime.so").write_text("") (runtime / "libtrtmc_backend_trt.so").write_text("") (runtime / "libtrtmc_model_beta.so").write_text("") native_build = tmp_path / "native-build" @@ -218,7 +219,12 @@ def test_selective_e2e_calls_family_tests_directly( assert options["updates"]["TRTMC_RUNTIME_ROOT"] != str(runtime) assert options["unset"] == ("PYTEST_ADDOPTS",) assert context.runtime_snapshots == [ - ("libtrtmc_backend_trt.so", "libtrtmc_core.so", "libtrtmc_model_beta.so") + ( + "libtrtmc_backend_trt.so", + "libtrtmc_core.so", + "libtrtmc_model_beta.so", + "libtrtmc_runtime.so", + ) ] @@ -229,6 +235,7 @@ def test_isolated_runtime_root_materializes_root_local_trtmc_libraries( runtime.mkdir() required = ( "libtrtmc_core.so", + "libtrtmc_runtime.so", "libtrtmc_backend_trt.so", "libtrtmc_model_beta.so", ) @@ -267,6 +274,7 @@ def test_family_with_only_hardware_tests_accepts_exact_empty_cpu_result( runtime.mkdir() for name in ( "libtrtmc_core.so", + "libtrtmc_runtime.so", "libtrtmc_backend_trt.so", "libtrtmc_model_beta.so", ): @@ -424,6 +432,7 @@ def test_e2e_nonexistent_testcase_fails_closed(tmp_path: Path) -> None: runtime.mkdir() for name in ( "libtrtmc_core.so", + "libtrtmc_runtime.so", "libtrtmc_backend_trt.so", "libtrtmc_model_alpha.so", ): From 936f5bb16e83cdfd718b3051f8e4c0dceaa0e739 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Wed, 9 Sep 2026 08:36:04 +0000 Subject: [PATCH 09/13] test(lerobot-act): identify invalid metrics Preserve the existing positive operational invariants while reporting the exact field and value that violates them. Signed-off-by: chaofengw --- families/lerobot_act/tests/test_e2e.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/families/lerobot_act/tests/test_e2e.py b/families/lerobot_act/tests/test_e2e.py index 0d5f149f8b..ae00ed6d87 100644 --- a/families/lerobot_act/tests/test_e2e.py +++ b/families/lerobot_act/tests/test_e2e.py @@ -244,7 +244,9 @@ def _assert_operational_summary(summary: dict) -> None: "startup_ms", ): value = float(summary[field]) - assert np.isfinite(value) and value > 0.0 + assert np.isfinite(value) and value > 0.0, ( + f"{field} must be finite and positive, got {value}" + ) effective_hz = float(summary["control_effective_hz"]) assert np.isfinite(effective_hz) and effective_hz >= 49.0 jitter = float(summary["control_p99_abs_jitter_ms"]) @@ -305,8 +307,13 @@ def test_operational_summary_rejects_invalid_active_invariants() -> None: "peak_resident_memory_mib": 1.0, "startup_ms": 1.0, } - for field in ("chunk_inference_p50_ms", "chunk_throughput_per_second", "gpu_memory_total_mib"): - with pytest.raises(AssertionError): + for field in ( + "chunk_inference_p50_ms", + "chunk_throughput_per_second", + "gpu_memory_delta_mib", + "gpu_memory_total_mib", + ): + with pytest.raises(AssertionError, match=field): _assert_operational_summary({**summary, field: 0.0}) with pytest.raises(AssertionError): _assert_operational_summary({**summary, "control_frequency_hz": 49.0}) From e95ac1d36f3046667653f4813c47bb65a97fc243 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Thu, 10 Sep 2026 03:32:49 +0000 Subject: [PATCH 10/13] fix(runtime): migrate new family descriptors Main added six families after the runtime-root descriptor migration. Export each family's descriptor from its own plugin entry point so the exact-build loader contract and architecture gate cover the merged product. The existing architecture test reproduces all six missing descriptors before the change and passes afterward. No test criteria or family execution behavior changes. Signed-off-by: chaofengw --- families/k2_horizon_uno/runtime/plugin.cpp | 2 ++ families/openfold3/runtime/plugin.cpp | 2 ++ families/timm_crossvit/runtime/plugin.cpp | 2 ++ families/timm_dpn/runtime/plugin.cpp | 2 ++ families/timm_resnest/runtime/plugin.cpp | 2 ++ families/timm_swin/runtime/plugin.cpp | 2 ++ 6 files changed, 12 insertions(+) diff --git a/families/k2_horizon_uno/runtime/plugin.cpp b/families/k2_horizon_uno/runtime/plugin.cpp index e6a4478a04..c803b60222 100644 --- a/families/k2_horizon_uno/runtime/plugin.cpp +++ b/families/k2_horizon_uno/runtime/plugin.cpp @@ -218,6 +218,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::k2_horizon_uno +TRTMC_DEFINE_FAMILY_PLUGIN_V1("k2_horizon_uno") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("k2_horizon_uno does not support --kv-cache-size"); diff --git a/families/openfold3/runtime/plugin.cpp b/families/openfold3/runtime/plugin.cpp index d210aa28ee..8f2983301b 100644 --- a/families/openfold3/runtime/plugin.cpp +++ b/families/openfold3/runtime/plugin.cpp @@ -108,6 +108,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::openfold3 +TRTMC_DEFINE_FAMILY_PLUGIN_V1("openfold3") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { return trtmc::openfold3::create(context); } diff --git a/families/timm_crossvit/runtime/plugin.cpp b/families/timm_crossvit/runtime/plugin.cpp index 6923cb6771..838514269d 100644 --- a/families/timm_crossvit/runtime/plugin.cpp +++ b/families/timm_crossvit/runtime/plugin.cpp @@ -50,6 +50,8 @@ std::unique_ptr load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector Date: Thu, 10 Sep 2026 12:23:33 +0000 Subject: [PATCH 11/13] fix: repair family validation for runtime discovery Load the selected OpenFold3 product companions, restore Wan shared input embeddings, and request PIL reference frames. Validate every Sana output frame after the official refiner removes its input anchor. Cover the loader mismatch with compiled product fixtures and exercise real Wan checkpoint loading. Add plugin descriptors for the three families merged into main since the previous validation. Signed-off-by: chaofengw --- families/boltz2/runtime/plugin.cpp | 2 + .../tests/cpp/fake_build_identity.cpp | 8 ++ .../tests/cpp/fake_qualification_runtime.cpp | 44 +++++++ families/openfold3/tests/test_e2e.py | 11 ++ families/openfold3/tests/test_e2e_runtime.py | 123 ++++++++++++++++++ families/timm_res2net/runtime/plugin.cpp | 2 + families/yolov10/runtime/plugin.cpp | 2 + 7 files changed, 192 insertions(+) create mode 100644 families/openfold3/tests/cpp/fake_build_identity.cpp create mode 100644 families/openfold3/tests/cpp/fake_qualification_runtime.cpp create mode 100644 families/openfold3/tests/test_e2e_runtime.py diff --git a/families/boltz2/runtime/plugin.cpp b/families/boltz2/runtime/plugin.cpp index 691dfabb11..bf832f6bf0 100644 --- a/families/boltz2/runtime/plugin.cpp +++ b/families/boltz2/runtime/plugin.cpp @@ -90,6 +90,8 @@ boltz2::BundleArtifacts loadArtifacts(const BundleReader& bundle) { } // namespace } // namespace trtmc +TRTMC_DEFINE_FAMILY_PLUGIN_V1("boltz2") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("boltz2 does not support --kv-cache-size"); diff --git a/families/openfold3/tests/cpp/fake_build_identity.cpp b/families/openfold3/tests/cpp/fake_build_identity.cpp new file mode 100644 index 0000000000..3a9b524c02 --- /dev/null +++ b/families/openfold3/tests/cpp/fake_build_identity.cpp @@ -0,0 +1,8 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +extern "C" int openfold3_test_build_id() { + return TEST_BUILD_ID; +} diff --git a/families/openfold3/tests/cpp/fake_qualification_runtime.cpp b/families/openfold3/tests/cpp/fake_qualification_runtime.cpp new file mode 100644 index 0000000000..3d858ebe7c --- /dev/null +++ b/families/openfold3/tests/cpp/fake_qualification_runtime.cpp @@ -0,0 +1,44 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "trtmc/openfold3/structure_prediction.h" +#include "trtmc/runtime/family_loader.h" + +#include +#include +#include + +extern "C" int openfold3_test_build_id(); + +namespace { + +class Prediction final : public trtmc::openfold3::IStructurePrediction { + public: + trtmc::openfold3::StructurePredictionResult predict_structure(const std::string&) override { + return {"data_test\n", "{\"build_id\":" + std::to_string(TEST_BUILD_ID) + "}"}; + } +}; + +} // namespace + +namespace trtmc { + +std::unique_ptr load_task(const std::string&, const std::string& runtime_root, std::uint64_t, + const std::string&, bool) { + const auto backend = std::filesystem::path(runtime_root) / "libtrtmc_backend_trt.so"; + void* library = dlopen(backend.c_str(), RTLD_NOW | RTLD_LOCAL); + if (library == nullptr) + throw std::runtime_error(dlerror()); + const auto backend_build = + reinterpret_cast(dlsym(library, "openfold3_test_build_id")); + const bool matches = backend_build != nullptr && backend_build() == TEST_BUILD_ID && + openfold3_test_build_id() == TEST_BUILD_ID; + dlclose(library); + if (!matches) + throw std::runtime_error("qualification runtime/core/backend product build mismatch"); + return std::make_unique(); +} + +} // namespace trtmc diff --git a/families/openfold3/tests/test_e2e.py b/families/openfold3/tests/test_e2e.py index bf8bff6481..1ac4a31dc9 100644 --- a/families/openfold3/tests/test_e2e.py +++ b/families/openfold3/tests/test_e2e.py @@ -167,6 +167,16 @@ def _run_native( ) -> tuple[str, dict]: structure = output_root / f"prediction-{index}.cif" metadata = output_root / f"prediction-{index}.json" + # The qualification executable is built separately from the selected wheel. + # Resolve the staged core so copied and symlinked roots both select its + # companion loader instead of the executable's source-build RUNPATH. + library_root = (runtime_root / "libtrtmc_core.so").resolve(strict=True).parent + loader = library_root / "libtrtmc_runtime.so" + assert loader.is_file(), f"OpenFold3 runtime is missing its companion loader: {loader}" + environment = os.environ.copy() + environment["LD_LIBRARY_PATH"] = os.pathsep.join( + path for path in (str(library_root), environment.get("LD_LIBRARY_PATH", "")) if path + ) subprocess.run( [ str(qualification), @@ -178,6 +188,7 @@ def _run_native( ], check=True, timeout=timeout, + env=environment, ) record_evidence("native", {"structure": structure, "confidence": metadata}) return structure.read_text(encoding="utf-8"), json.loads(metadata.read_text(encoding="utf-8")) diff --git a/families/openfold3/tests/test_e2e_runtime.py b/families/openfold3/tests/test_e2e_runtime.py new file mode 100644 index 0000000000..54148637d1 --- /dev/null +++ b/families/openfold3/tests/test_e2e_runtime.py @@ -0,0 +1,123 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import os +from pathlib import Path +import shutil +import subprocess + +import pytest + +from . import test_e2e as e2e + + +@pytest.fixture(scope="module") +def qualification_builds(tmp_path_factory): + root = tmp_path_factory.mktemp("openfold3-builds") + repository = Path(__file__).resolve().parents[3] + sources = Path(__file__).with_name("cpp") + compiler = shutil.which("c++") + assert compiler, "OpenFold3 runtime regression requires a C++ compiler" + flags = [ + compiler, + "-std=c++17", + f"-I{repository / 'core/runtime/include'}", + f"-I{repository / 'families/openfold3/include'}", + ] + native, wheel = root / "native-build", root / "wheel" + for build_id, directory in enumerate((native, wheel), start=1): + directory.mkdir() + for name in ("core", "backend_trt"): + library = f"libtrtmc_{name}.so" + subprocess.run( + [ + *flags, + "-shared", + "-fPIC", + f"-DTEST_BUILD_ID={build_id}", + str(sources / "fake_build_identity.cpp"), + f"-Wl,-soname,{library}", + "-o", + str(directory / library), + ], + check=True, + ) + subprocess.run( + [ + *flags, + "-shared", + "-fPIC", + f"-DTEST_BUILD_ID={build_id}", + str(sources / "fake_qualification_runtime.cpp"), + f"-L{directory}", + "-ltrtmc_core", + "-ldl", + "-Wl,-soname,libtrtmc_runtime.so", + "-Wl,-rpath,$ORIGIN", + "-o", + str(directory / "libtrtmc_runtime.so"), + ], + check=True, + ) + qualification = native / "openfold3_qualification" + subprocess.run( + [ + *flags, + str(sources / "qualification.cpp"), + f"-L{native}", + "-ltrtmc_runtime", + f"-Wl,-rpath,{native}", + f"-Wl,-rpath-link,{native}", + "-o", + str(qualification), + ], + check=True, + ) + return qualification, wheel + + +@pytest.mark.parametrize("layout", ("copied", "symlinked")) +def test_qualification_uses_selected_product_build( + qualification_builds, monkeypatch, tmp_path: Path, layout: str +) -> None: + qualification, wheel = qualification_builds + runtime_root = tmp_path / "runtime" + runtime_root.mkdir() + for name in ("libtrtmc_core.so", "libtrtmc_backend_trt.so", "libtrtmc_runtime.so"): + if layout == "copied": + shutil.copy2(wheel / name, runtime_root / name) + elif name != "libtrtmc_runtime.so": + (runtime_root / name).symlink_to(wheel / name) + # The source-built executable's RUNPATH and the inherited environment both + # point at a different product build from the selected wheel libraries. + inherited_path = str(qualification.parent) + monkeypatch.setenv("LD_LIBRARY_PATH", inherited_path) + request = tmp_path / "query.json" + request.write_text("{}", encoding="utf-8") + + structure, metadata = e2e._run_native( + qualification, runtime_root, tmp_path / "model.bundle", request, tmp_path, 0, 30 + ) + + assert structure == "data_test\n" + assert metadata == {"build_id": 2} + assert os.environ["LD_LIBRARY_PATH"] == inherited_path + + +def test_qualification_rejects_a_missing_companion_loader(tmp_path: Path) -> None: + runtime_root = tmp_path / "runtime" + runtime_root.mkdir() + (runtime_root / "libtrtmc_core.so").touch() + + with pytest.raises(AssertionError, match="missing its companion loader"): + e2e._run_native( + tmp_path / "qualification", + runtime_root, + tmp_path / "model.bundle", + tmp_path / "query.json", + tmp_path, + 0, + 30, + ) diff --git a/families/timm_res2net/runtime/plugin.cpp b/families/timm_res2net/runtime/plugin.cpp index 16d8b71ff0..c5dd9ddc73 100644 --- a/families/timm_res2net/runtime/plugin.cpp +++ b/families/timm_res2net/runtime/plugin.cpp @@ -50,6 +50,8 @@ std::unique_ptr load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector Date: Thu, 10 Sep 2026 13:00:20 +0000 Subject: [PATCH 12/13] fix: align family follow-ups with reporting changes Use the same Wan, Sana, and LFM2 adapters and regression controls as #1219 to remove duplicate variants and minimize overlapping edits. Keep its reporting assertions when combining the Sana reference test. Remove the LeRobot diagnostic follow-up: its error-message test conflicts with the observation-based memory contract in #1219. Retain OpenFold3 product-loader selection and all strict runtime build checks. Refs: #1219 Signed-off-by: chaofengw --- families/detr/runtime/plugin.cpp | 2 ++ families/lerobot_act/tests/test_e2e.py | 13 +++---------- families/smollm3/runtime/plugin.cpp | 2 ++ families/timm_mobilenetv4/runtime/plugin.cpp | 2 ++ families/timm_mobilevit/runtime/plugin.cpp | 2 ++ families/timm_nfnet/runtime/plugin.cpp | 2 ++ families/timm_xcit/runtime/plugin.cpp | 2 ++ families/yolo11/runtime/plugin.cpp | 2 ++ families/yolov5/runtime/plugin.cpp | 2 ++ families/yolov8/runtime/plugin.cpp | 2 ++ 10 files changed, 21 insertions(+), 10 deletions(-) diff --git a/families/detr/runtime/plugin.cpp b/families/detr/runtime/plugin.cpp index df352fe6ec..3ef73d40c5 100644 --- a/families/detr/runtime/plugin.cpp +++ b/families/detr/runtime/plugin.cpp @@ -49,6 +49,8 @@ std::unique_ptr load_engine(IBackend& backend, const std::vector None: "startup_ms", ): value = float(summary[field]) - assert np.isfinite(value) and value > 0.0, ( - f"{field} must be finite and positive, got {value}" - ) + assert np.isfinite(value) and value > 0.0 effective_hz = float(summary["control_effective_hz"]) assert np.isfinite(effective_hz) and effective_hz >= 49.0 jitter = float(summary["control_p99_abs_jitter_ms"]) @@ -307,13 +305,8 @@ def test_operational_summary_rejects_invalid_active_invariants() -> None: "peak_resident_memory_mib": 1.0, "startup_ms": 1.0, } - for field in ( - "chunk_inference_p50_ms", - "chunk_throughput_per_second", - "gpu_memory_delta_mib", - "gpu_memory_total_mib", - ): - with pytest.raises(AssertionError, match=field): + for field in ("chunk_inference_p50_ms", "chunk_throughput_per_second", "gpu_memory_total_mib"): + with pytest.raises(AssertionError): _assert_operational_summary({**summary, field: 0.0}) with pytest.raises(AssertionError): _assert_operational_summary({**summary, "control_frequency_hz": 49.0}) diff --git a/families/smollm3/runtime/plugin.cpp b/families/smollm3/runtime/plugin.cpp index 797abd5e09..987ae3475b 100644 --- a/families/smollm3/runtime/plugin.cpp +++ b/families/smollm3/runtime/plugin.cpp @@ -238,6 +238,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::smollm3 +TRTMC_DEFINE_FAMILY_PLUGIN_V1("smollm3") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("smollm3 does not support --kv-cache-size"); diff --git a/families/timm_mobilenetv4/runtime/plugin.cpp b/families/timm_mobilenetv4/runtime/plugin.cpp index 0fc4147383..7842361df4 100644 --- a/families/timm_mobilenetv4/runtime/plugin.cpp +++ b/families/timm_mobilenetv4/runtime/plugin.cpp @@ -51,6 +51,8 @@ std::unique_ptr load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector Date: Mon, 14 Sep 2026 09:52:35 +0000 Subject: [PATCH 13/13] fix(runtime): declare new family plugin identities Signed-off-by: chaofengw --- families/glmasr/runtime/plugin.cpp | 2 ++ families/s1_mini/runtime/plugin.cpp | 2 ++ 2 files changed, 4 insertions(+) diff --git a/families/glmasr/runtime/plugin.cpp b/families/glmasr/runtime/plugin.cpp index 3747edeac0..566ac01187 100644 --- a/families/glmasr/runtime/plugin.cpp +++ b/families/glmasr/runtime/plugin.cpp @@ -20,6 +20,8 @@ std::vector required(const trtmc::BundleReader& b, const char* name) { } } // namespace +TRTMC_DEFINE_FAMILY_PLUGIN_V1("glmasr") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { using namespace trtmc; if (context.kv_cache_size_bytes != 0) diff --git a/families/s1_mini/runtime/plugin.cpp b/families/s1_mini/runtime/plugin.cpp index b03095dc80..5754184479 100644 --- a/families/s1_mini/runtime/plugin.cpp +++ b/families/s1_mini/runtime/plugin.cpp @@ -233,6 +233,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::s1_mini +TRTMC_DEFINE_FAMILY_PLUGIN_V1("s1_mini") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("s1_mini does not support --kv-cache-size");