diff --git a/CMakeLists.txt b/CMakeLists.txt index 1098879680..db1ed031a3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,6 +12,25 @@ set(CMAKE_CUDA_STANDARD_REQUIRED ON) set(CMAKE_POSITION_INDEPENDENT_CODE ON) set(CMAKE_BUILD_RPATH_USE_ORIGIN TRUE) +set(TRTMC_BUILD_ID "" CACHE STRING "Exact product-build identity embedded in runtime plugins") +if(NOT TRTMC_BUILD_ID) + string(RANDOM LENGTH 32 ALPHABET 0123456789abcdef _trtmc_generated_build_id) + set(TRTMC_BUILD_ID "${_trtmc_generated_build_id}" CACHE STRING + "Exact product-build identity embedded in runtime plugins" FORCE + ) +endif() +string(LENGTH "${TRTMC_BUILD_ID}" _trtmc_build_id_length) +if(NOT _trtmc_build_id_length EQUAL 32 OR NOT TRTMC_BUILD_ID MATCHES "^[0-9a-f]+$") + message(FATAL_ERROR "TRTMC_BUILD_ID must contain exactly 32 lowercase hex characters") +endif() +set(_trtmc_generated_include_dir "${CMAKE_BINARY_DIR}/generated/include") +file(MAKE_DIRECTORY "${_trtmc_generated_include_dir}/trtmc/runtime") +configure_file( + cmake/product_build.h.in + "${_trtmc_generated_include_dir}/trtmc/runtime/product_build.h" + @ONLY +) + include(GNUInstallDirs) find_package(CUDAToolkit REQUIRED) find_package(nlohmann_json 3.11 REQUIRED) @@ -79,15 +98,20 @@ endif() set(TRTMC_CUDA_INCLUDE_DIR ${CUDAToolkit_INCLUDE_DIRS}) set(TRTMC_CUDART_LIBRARY CUDA::cudart) -add_library(trtmc_core SHARED +set(_trtmc_core_sources core/runtime/bundle/bundle_format.cpp core/runtime/primitives/cuda_common.cpp core/runtime/primitives/device_tensor.cpp core/runtime/primitives/trt_common.cpp ) +add_library(trtmc_core SHARED + ${_trtmc_core_sources} + core/runtime/primitives/build_identity.cpp +) target_include_directories(trtmc_core PUBLIC $ + $ $ PRIVATE ${PROJECT_SOURCE_DIR}/core @@ -218,6 +242,7 @@ if(TRTMC_HAS_TVM_FFI) $ PRIVATE ${PROJECT_SOURCE_DIR}/core + ${_trtmc_generated_include_dir} ) target_include_directories(trtmc_byok_tvm_ffi SYSTEM PRIVATE ${TRTMC_TRT_INCLUDE_DIR} @@ -244,6 +269,8 @@ endif() option(TRTMC_BUILD_TESTS "Build tests" ON) option(TRTMC_BUILD_EXAMPLES "Build examples" ON) if(TRTMC_BUILD_TESTS) + set(_trtmc_test_runtime_root "${CMAKE_BINARY_DIR}/tests/runtime") + enable_testing() endif() @@ -346,11 +373,15 @@ if(TRTMC_BUILD_TESTS) ) target_link_libraries(test_cli PRIVATE trtmc_cli nlohmann_json::nlohmann_json) target_compile_options(test_cli PRIVATE -Wall -Wextra -Wpedantic) - add_test(NAME cli COMMAND test_cli) + add_test(NAME cli COMMAND test_cli + "${_trtmc_test_runtime_root}" + ) - set(_trtmc_test_runtime_root "${CMAKE_BINARY_DIR}/tests/runtime") add_library(trtmc_test_backend_fake SHARED core/runtime/tests/fake_backend.cpp) - target_include_directories(trtmc_test_backend_fake PRIVATE ${PROJECT_SOURCE_DIR}/core/runtime/include) + target_include_directories(trtmc_test_backend_fake PRIVATE + ${PROJECT_SOURCE_DIR}/core/runtime/include + ${_trtmc_generated_include_dir} + ) target_link_libraries(trtmc_test_backend_fake PRIVATE CUDA::cudart) set_target_properties(trtmc_test_backend_fake PROPERTIES OUTPUT_NAME trtmc_backend_fake @@ -360,6 +391,7 @@ if(TRTMC_BUILD_TESTS) add_library(trtmc_test_backend_fake_rtx SHARED core/runtime/tests/fake_backend.cpp) target_include_directories(trtmc_test_backend_fake_rtx PRIVATE ${PROJECT_SOURCE_DIR}/core/runtime/include + ${_trtmc_generated_include_dir} ) target_link_libraries(trtmc_test_backend_fake_rtx PRIVATE CUDA::cudart) target_compile_definitions(trtmc_test_backend_fake_rtx PRIVATE @@ -370,6 +402,59 @@ if(TRTMC_BUILD_TESTS) LIBRARY_OUTPUT_DIRECTORY "${_trtmc_test_runtime_root}" ) + add_library(trtmc_test_backend_incompatible SHARED core/runtime/tests/fake_backend.cpp) + target_include_directories(trtmc_test_backend_incompatible PRIVATE + ${PROJECT_SOURCE_DIR}/core/runtime/include + ${_trtmc_generated_include_dir} + ) + target_link_libraries(trtmc_test_backend_incompatible PRIVATE CUDA::cudart) + target_compile_definitions(trtmc_test_backend_incompatible PRIVATE + TRTMC_FAKE_BACKEND_NAME="incompatible" + TRTMC_FAKE_INCOMPATIBLE_BUILD=1 + ) + set_target_properties(trtmc_test_backend_incompatible PROPERTIES + OUTPUT_NAME trtmc_backend_incompatible + LIBRARY_OUTPUT_DIRECTORY "${_trtmc_test_runtime_root}" + ) + + add_library(trtmc_test_backend_compatible_incompatible SHARED + core/runtime/tests/fake_backend.cpp + ) + target_include_directories(trtmc_test_backend_compatible_incompatible PRIVATE + ${PROJECT_SOURCE_DIR}/core/runtime/include + ${_trtmc_generated_include_dir} + ) + target_link_libraries(trtmc_test_backend_compatible_incompatible PRIVATE CUDA::cudart) + target_compile_definitions(trtmc_test_backend_compatible_incompatible PRIVATE + TRTMC_FAKE_BACKEND_NAME="incompatible" + ) + set_target_properties(trtmc_test_backend_compatible_incompatible PROPERTIES + OUTPUT_NAME trtmc_backend_incompatible_compatible + LIBRARY_OUTPUT_DIRECTORY "${_trtmc_test_runtime_root}" + ) + + add_library(trtmc_test_core_incompatible SHARED + ${_trtmc_core_sources} + core/runtime/tests/fake_core_build_identity.cpp + ) + target_include_directories(trtmc_test_core_incompatible + PUBLIC ${PROJECT_SOURCE_DIR}/core/runtime/include + PRIVATE ${PROJECT_SOURCE_DIR}/core + ) + target_include_directories(trtmc_test_core_incompatible SYSTEM PRIVATE + ${CUDAToolkit_INCLUDE_DIRS} + ) + target_link_libraries(trtmc_test_core_incompatible + PUBLIC CUDA::cudart + PRIVATE nlohmann_json::nlohmann_json + ) + target_compile_options(trtmc_test_core_incompatible PRIVATE -Wall -Wextra -Wpedantic) + set_target_properties(trtmc_test_core_incompatible PROPERTIES + OUTPUT_NAME trtmc_core + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tests/incompatible-core" + BUILD_RPATH "\$ORIGIN" + ) + add_library(trtmc_test_family_fake SHARED core/runtime/tests/fake_family.cpp) target_include_directories(trtmc_test_family_fake PRIVATE ${PROJECT_SOURCE_DIR}/core/runtime/include) target_link_libraries(trtmc_test_family_fake PRIVATE trtmc_core) @@ -378,6 +463,25 @@ if(TRTMC_BUILD_TESTS) LIBRARY_OUTPUT_DIRECTORY "${_trtmc_test_runtime_root}" BUILD_RPATH "\$ORIGIN/../.." ) + add_dependencies(test_cli + trtmc_test_backend_fake + trtmc_test_backend_incompatible + trtmc_test_backend_compatible_incompatible + trtmc_test_core_incompatible + trtmc_test_family_fake + ) + + add_executable(test_runtime_root core/runtime/tests/test_runtime_root.cpp) + target_include_directories(test_runtime_root PRIVATE ${PROJECT_SOURCE_DIR}/core/runtime/include) + target_link_libraries(test_runtime_root PRIVATE trtmc_runtime) + target_compile_options(test_runtime_root PRIVATE -Wall -Wextra -Wpedantic) + add_dependencies(test_runtime_root trtmc_test_backend_fake trtmc_test_family_fake) + add_test(NAME runtime_root COMMAND test_runtime_root + "${CMAKE_BINARY_DIR}/tests/runtime-root-validator" + $ + $ + $ + ) add_executable(test_family_loader core/runtime/tests/test_family_loader.cpp) target_include_directories(test_family_loader PRIVATE @@ -389,9 +493,28 @@ if(TRTMC_BUILD_TESTS) add_dependencies(test_family_loader trtmc_test_backend_fake trtmc_test_backend_fake_rtx + trtmc_test_backend_incompatible + trtmc_test_core_incompatible trtmc_test_family_fake ) add_test(NAME family_loader COMMAND test_family_loader "${_trtmc_test_runtime_root}") + add_test(NAME family_loader_incompatible_core + COMMAND ${CMAKE_COMMAND} -E env + "LD_LIBRARY_PATH=$" + $ + "${_trtmc_test_runtime_root}" + --expect-core-mismatch + ) + add_test(NAME cli_incompatible_core + COMMAND ${CMAKE_COMMAND} -E env + "LD_LIBRARY_PATH=$" + $ + --expect-core-mismatch + ) + set_tests_properties( + family_loader family_loader_incompatible_core cli cli_incompatible_core runtime_root + PROPERTIES RESOURCE_LOCK runtime-loader-fixtures + ) add_executable(test_trt_module_dynamic_input core/runtime/tests/test_trt_module_dynamic_input.cpp @@ -544,6 +667,8 @@ install(FILES core/runtime/include/trtmc/runtime/device_tensor.h core/runtime/include/trtmc/runtime/family_factory.h core/runtime/include/trtmc/runtime/family_loader.h + core/runtime/include/trtmc/runtime/plugin_abi.h + core/runtime/include/trtmc/runtime/runtime_root.h core/runtime/include/trtmc/runtime/tensor.h core/runtime/include/trtmc/runtime/trt_backend.h core/runtime/include/trtmc/runtime/trt_module.h diff --git a/README.md b/README.md index a68e9a7afb..ed0200c1fb 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,6 @@ python -m tensorrt_model_connect build Qwen/Qwen3-0.6B \ --max-sequence-length 16384 \ --output qwen3-0.6b.bundle trtmc run ./qwen3-0.6b.bundle \ - --runtime-root /opt/trtmc/lib \ --prompt "What is the capital of France? Answer in one word." \ --use-chat-template true \ --enable-thinking false diff --git a/apps/cli/cli.cpp b/apps/cli/cli.cpp index 6aee9dd1df..dbb74a9ae4 100644 --- a/apps/cli/cli.cpp +++ b/apps/cli/cli.cpp @@ -7,6 +7,8 @@ #include "cli/io.h" #include "trtmc/runtime/family_loader.h" +#include "trtmc/runtime/plugin_abi.h" +#include "trtmc/runtime/runtime_root.h" #include #include @@ -14,7 +16,6 @@ #include #include #include -#include #include #include #include @@ -22,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -141,31 +143,59 @@ bool is_byok_option(const std::string& option) { return option == "--byok-library" || option == "--byok-function" || option == "--byok-name"; } -void load_byok_extension(const Command& command) { - using LoadKernelFn = const char* (*)(const char*, const char*, const char*) noexcept; - const fs::path extension = fs::path(command.runtime_root) / "libtrtmc_byok_tvm_ffi.so"; - dlerror(); - void* handle = dlopen(extension.c_str(), RTLD_NOW | RTLD_LOCAL); - if (handle == nullptr) { - const char* error = dlerror(); - throw std::runtime_error("unable to load BYOK extension '" + extension.string() + - "': " + (error != nullptr ? error : "unknown dlopen error")); - } - static auto* handles = new std::vector; - handles->push_back(handle); - dlerror(); - auto load = reinterpret_cast(dlsym(handle, "trtmc_load_byok_kernel")); - if (const char* error = dlerror(); error != nullptr || load == nullptr) { - throw std::runtime_error("BYOK extension is missing trtmc_load_byok_kernel"); - } - if (const char* error = load(command.options.at("--byok-library").c_str(), - command.options.at("--byok-function").c_str(), - command.options.at("--byok-name").c_str())) { - const std::string message = error; - throw std::runtime_error(message); +void append_candidate(std::vector& candidates, std::set& seen, + const fs::path& candidate) { + if (candidate.empty()) + return; + std::error_code error; + fs::path absolute = fs::absolute(candidate, error); + if (error) + return; + fs::path normalized = fs::weakly_canonical(absolute, error); + if (error) + normalized = absolute.lexically_normal(); + const std::string key = normalized.string(); + if (seen.insert(key).second) + candidates.push_back(std::move(normalized)); +} + +void append_path_list(std::vector& candidates, std::set& seen, + const std::string& paths) { + std::size_t begin = 0; + while (begin <= paths.size()) { + const std::size_t end = paths.find(':', begin); + const std::string path = + paths.substr(begin, end == std::string::npos ? std::string::npos : end - begin); + if (!path.empty()) + append_candidate(candidates, seen, path); + if (end == std::string::npos) + break; + begin = end + 1; } } +RuntimeRootSearchContext runtime_root_search_context() { + RuntimeRootSearchContext context; + context.loaded_runtime_root = loaded_runtime_root(); + if (const char* value = std::getenv("TRTMC_RUNTIME_PATH")) + context.runtime_path = value; + return context; +} + +void require_matching_product_build() { + const std::string expected = trtmc::kPluginBuildId; + const auto require_module = [&](const char* module, const char* actual) { + if (actual == nullptr || expected != actual) { + throw std::runtime_error( + "TRTMC product build mismatch: CLI requires '" + expected + "' but active " + + module + " reports '" + + (actual != nullptr ? std::string(actual) : std::string("")) + "'"); + } + }; + require_module("Runtime", trtmc_runtime_build_id()); + require_module("Core", trtmc_core_build_id()); +} + std::string take_value(int argc, char** argv, int& index, const std::string& option) { if (index + 1 >= argc) throw std::invalid_argument(option + " requires a value"); @@ -661,6 +691,35 @@ int dispatch_run(const Command& command, ITask& task, std::ostream& output) { } // namespace +std::string resolve_runtime_root(const BundleInfo& bundle, const std::string& explicit_root, + bool require_byok, const RuntimeRootSearchContext& context, + const RuntimeRootMatcher& matches) { + if (!explicit_root.empty()) + return explicit_root; + if (!matches) + throw std::logic_error("runtime-root discovery requires a candidate matcher"); + + std::vector candidates; + std::set seen; + append_candidate(candidates, seen, context.loaded_runtime_root); + append_path_list(candidates, seen, context.runtime_path); + + std::vector searched; + for (const auto& candidate : candidates) { + searched.push_back(candidate); + if (matches(bundle, candidate, require_byok)) + return candidate.string(); + } + + std::ostringstream message; + message << "Unable to discover a complete TRTMC runtime for bundle '" << bundle.family << "/" + << bundle.backend << "'. Searched:"; + for (const auto& candidate : searched) + message << " " << candidate.string(); + message << ". Pass --runtime-root DIR or add a directory to TRTMC_RUNTIME_PATH."; + throw std::runtime_error(message.str()); +} + Command parse_args(int argc, char** argv) { if (argc < 2) throw std::invalid_argument("a command is required"); @@ -728,8 +787,6 @@ Command parse_args(int argc, char** argv) { throw std::invalid_argument(option + " may be specified only once"); command.options.emplace(option, take_value(argc, argv, index, option)); } - if (command.runtime_root.empty()) - throw std::invalid_argument("--runtime-root is required for " + name); const int byok_option_count = static_cast(command.options.count("--byok-library") + command.options.count("--byok-function") + command.options.count("--byok-name")); @@ -1280,7 +1337,7 @@ void print_usage(std::ostream& output) { output << "Usage:\n" " trtmc version\n" " trtmc inspect BUNDLE\n" - " trtmc COMMAND BUNDLE --runtime-root DIR [OPTIONS]\n\n" + " trtmc COMMAND BUNDLE [--runtime-root DIR] [OPTIONS]\n\n" "Execution commands:\n" " run, encode, embed, rerank, classify, detect, extract-features,\n" " predict-structure, disparity, geometry,\n" @@ -1308,12 +1365,14 @@ void print_usage(std::ostream& output) { " [--kv-cache-size BYTES|GB|GiB]\n\n" "TensorRT-RTX runtime options:\n" " [--runtime-cache PATH] [--cuda-graphs]\n\n" - "Execution never searches for runtimes; --runtime-root is always required.\n"; + "Runtime discovery: the active Runtime directory, then TRTMC_RUNTIME_PATH.\n" + "The current directory is not searched; use TRTMC_RUNTIME_PATH=. explicitly.\n" + "--runtime-root selects one exact root without fallback.\n"; } int run(int argc, char** argv, std::ostream& output, std::ostream& error) { try { - const Command command = parse_args(argc, argv); + Command command = parse_args(argc, argv); if (command.kind == CommandKind::kHelp) { print_usage(output); return EXIT_SUCCESS; @@ -1322,6 +1381,7 @@ int run(int argc, char** argv, std::ostream& output, std::ostream& error) { output << "trtmc " << TRTMC_VERSION_STRING << '\n'; return EXIT_SUCCESS; } + require_matching_product_build(); if (command.kind == CommandKind::kInspect) { const BundleInfo bundle = InspectBundle(command.bundle); nlohmann::json sections = nlohmann::json::object(); @@ -1342,8 +1402,23 @@ int run(int argc, char** argv, std::ostream& output, std::ostream& error) { throw std::invalid_argument( "--byok-library, --byok-function, and --byok-name must be used together"); } - load_byok_extension(command); } + const bool discover_runtime = command.runtime_root.empty(); + if (discover_runtime) { + const BundleInfo bundle = InspectBundle(command.bundle); + command.runtime_root = + resolve_runtime_root(bundle, {}, has_byok_library, runtime_root_search_context(), + [](const BundleInfo& candidate_bundle, + const fs::path& candidate, bool require_byok) { + return runtime_root_contains_bundle( + candidate_bundle, candidate.string(), require_byok); + }); + error << "Using TRTMC runtime: " << command.runtime_root << '\n'; + } + if (has_byok_library) + load_byok_kernel_from_runtime( + command.runtime_root, command.options.at("--byok-library"), + command.options.at("--byok-function"), command.options.at("--byok-name")); std::unique_ptr task = load_task(command.bundle, command.runtime_root, command.kv_cache_size_bytes, command.runtime_cache_path, command.cuda_graphs); diff --git a/apps/cli/cli.h b/apps/cli/cli.h index 9fe114557f..350db11c17 100644 --- a/apps/cli/cli.h +++ b/apps/cli/cli.h @@ -9,6 +9,8 @@ #include "trtmc/task.h" #include +#include +#include #include #include #include @@ -61,7 +63,18 @@ struct Command { bool cuda_graphs{false}; }; +struct RuntimeRootSearchContext { + std::filesystem::path loaded_runtime_root; + std::string runtime_path; +}; + +using RuntimeRootMatcher = + std::function; + Command parse_args(int argc, char** argv); +std::string resolve_runtime_root(const BundleInfo& bundle, const std::string& explicit_root, + bool require_byok, const RuntimeRootSearchContext& context, + const RuntimeRootMatcher& matches); int dispatch(const Command& command, ITask& task, std::ostream& output); void print_usage(std::ostream& output); int run(int argc, char** argv, std::ostream& output, std::ostream& error); diff --git a/apps/cli/tests/test_cli.cpp b/apps/cli/tests/test_cli.cpp index 2023b7ce0d..33d523f1e1 100644 --- a/apps/cli/tests/test_cli.cpp +++ b/apps/cli/tests/test_cli.cpp @@ -7,11 +7,13 @@ #include "cli/io.h" #include +#include #include #include #include #include #include +#include #include #include #include @@ -46,6 +48,137 @@ bool parse_throws(std::vector arguments) { } } +std::string path_key(const std::filesystem::path& path) { + std::error_code error; + const auto absolute = std::filesystem::absolute(path, error); + if (error) + return path.lexically_normal().string(); + const auto canonical = std::filesystem::weakly_canonical(absolute, error); + return (error ? absolute.lexically_normal() : canonical).string(); +} + +void write_fake_bundle(const std::filesystem::path& path, const std::string& backend = "fake") { + static constexpr unsigned char magic[8] = {'B', 'U', 'N', 'D', 'L', 'E', '\x01', '\0'}; + const std::string header = + "{\"format\":1,\"family\":\"fake\",\"task\":\"time_series_forecast\"," + "\"backend\":\"" + + backend + + "\",\"sections\":{\"runtime.json\":{\"offset\":0,\"length\":2}," + "\"engine.plan\":{\"offset\":2,\"length\":4}}}"; + std::ofstream output(path, std::ios::binary); + output.write(reinterpret_cast(magic), 8); + const std::uint64_t length = header.size(); + for (int shift = 0; shift < 64; shift += 8) + output.put(static_cast((length >> shift) & 0xffU)); + output.write(header.data(), static_cast(header.size())); + output.write("{}PLAN", 6); +} + +void check_cli_runtime_discovery(const std::filesystem::path& runtime_root) { + const auto discovered_root = runtime_root.parent_path() / "cli-e2e-runtime"; + std::filesystem::remove_all(discovered_root); + std::filesystem::create_directories(discovered_root); + std::filesystem::copy_file(runtime_root / "libtrtmc_backend_fake.so", + discovered_root / "libtrtmc_backend_fake.so"); + std::filesystem::copy_file(runtime_root / "libtrtmc_model_fake.so", + discovered_root / "libtrtmc_model_fake.so"); + const auto bundle = discovered_root / "cli-fake.bundle"; + const auto input = discovered_root / "cli-fake-input.f32"; + write_fake_bundle(bundle); + { + const float values[] = {1.0F, 2.0F, 3.0F}; + std::ofstream output(input, std::ios::binary); + output.write(reinterpret_cast(values), sizeof(values)); + } + + const char* previous_runtime_path_value = std::getenv("TRTMC_RUNTIME_PATH"); + const bool had_runtime_path = previous_runtime_path_value != nullptr; + const std::string previous_runtime_path = had_runtime_path ? previous_runtime_path_value : ""; + setenv("TRTMC_RUNTIME_PATH", discovered_root.c_str(), 1); + std::vector arguments{"trtmc", "forecast", bundle.string(), "--input", + input.string()}; + std::vector argv; + for (auto& argument : arguments) + argv.push_back(argument.data()); + std::ostringstream output; + std::ostringstream error; + const int result = trtmc::cli::run(static_cast(argv.size()), argv.data(), output, error); + if (had_runtime_path) + setenv("TRTMC_RUNTIME_PATH", previous_runtime_path.c_str(), 1); + else + unsetenv("TRTMC_RUNTIME_PATH"); + check(result == 0 && error.str() == "Using TRTMC runtime: " + discovered_root.string() + "\n", + "CLI executes with a runtime discovered from TRTMC_RUNTIME_PATH"); + check(output.str().find("\"shape\":[1,3]") != std::string::npos, + "automatically discovered runtime dispatches the bundle task"); + + std::filesystem::remove_all(discovered_root); +} + +void check_selected_root_never_falls_back(const std::filesystem::path& runtime_root) { + const auto base = runtime_root.parent_path() / "cli-no-fallback"; + const auto incompatible_root = base / "incompatible"; + const auto compatible_root = base / "compatible"; + std::filesystem::remove_all(base); + std::filesystem::create_directories(incompatible_root); + std::filesystem::create_directories(compatible_root); + for (const auto& root : {incompatible_root, compatible_root}) { + std::filesystem::copy_file(runtime_root / "libtrtmc_model_fake.so", + root / "libtrtmc_model_fake.so"); + } + std::filesystem::copy_file(runtime_root / "libtrtmc_backend_incompatible.so", + incompatible_root / "libtrtmc_backend_incompatible.so"); + std::filesystem::copy_file(runtime_root / "libtrtmc_backend_incompatible_compatible.so", + compatible_root / "libtrtmc_backend_incompatible.so"); + const auto bundle = base / "incompatible.bundle"; + const auto input = base / "input.f32"; + write_fake_bundle(bundle, "incompatible"); + { + const float values[] = {1.0F}; + std::ofstream output(input, std::ios::binary); + output.write(reinterpret_cast(values), sizeof(values)); + } + + const char* previous_runtime_path_value = std::getenv("TRTMC_RUNTIME_PATH"); + const bool had_runtime_path = previous_runtime_path_value != nullptr; + const std::string previous_runtime_path = had_runtime_path ? previous_runtime_path_value : ""; + const std::string configured_roots = + incompatible_root.string() + ":" + compatible_root.string(); + setenv("TRTMC_RUNTIME_PATH", configured_roots.c_str(), 1); + std::vector arguments{"trtmc", "forecast", bundle.string(), "--input", + input.string()}; + std::vector argv; + for (auto& argument : arguments) + argv.push_back(argument.data()); + std::ostringstream output; + std::ostringstream error; + const int result = trtmc::cli::run(static_cast(argv.size()), argv.data(), output, error); + if (had_runtime_path) + setenv("TRTMC_RUNTIME_PATH", previous_runtime_path.c_str(), 1); + else + unsetenv("TRTMC_RUNTIME_PATH"); + + check(result != 0 && error.str().find("belongs to product build") != std::string::npos, + "selected structurally complete root fails without fallback"); + check(error.str().find("Using TRTMC runtime: " + incompatible_root.string()) != + std::string::npos && + output.str().empty(), + "runtime path order cannot hide an incompatible selected product"); + std::filesystem::remove_all(base); +} + +bool resolve_throws(const trtmc::BundleInfo& bundle, + const trtmc::cli::RuntimeRootSearchContext& context, + const trtmc::cli::RuntimeRootMatcher& matches, std::string& message) { + try { + (void)trtmc::cli::resolve_runtime_root(bundle, {}, false, context, matches); + return false; + } catch (const std::runtime_error& error) { + message = error.what(); + return true; + } +} + class FakeText final : public trtmc::ITextGeneration, public trtmc::IEmbedding, public trtmc::ILoraAdapterManager { @@ -269,7 +402,23 @@ bool dispatch_throws(const trtmc::cli::Command& command, trtmc::ITask& task) { } // namespace -int main() { +int main(int argc, char** argv) { + if (argc == 2 && std::string(argv[1]) == "--expect-core-mismatch") { + std::vector arguments{"trtmc", "inspect", "not-opened.bundle"}; + std::vector pointers; + for (auto& argument : arguments) + pointers.push_back(argument.data()); + std::ostringstream output; + std::ostringstream error; + const int result = + trtmc::cli::run(static_cast(pointers.size()), pointers.data(), output, error); + check(result != 0 && error.str().find("active Core reports") != std::string::npos, + "CLI rejects an incompatible Core before bundle inspection"); + check(error.str().find("not-opened.bundle") == std::string::npos, + "CLI does not cross the Core C++ seam before identity validation"); + return failures == 0 ? 0 : 1; + } + const std::vector execution_commands{ "run", "encode", @@ -303,11 +452,104 @@ int main() { check(command.runtime_root == "lib", "runtime root is retained"); } - check(parse_throws({"trtmc", "run", "model.bundle"}), - "execution command requires runtime root"); + check(parse({"trtmc", "run", "model.bundle"}).runtime_root.empty(), + "execution command permits automatic runtime discovery"); check(parse_throws( {"trtmc", "run", "model.bundle", "--runtime-root", "a", "--runtime-root", "b"}), "duplicate runtime root rejected"); + + const std::filesystem::path runtime_test_root = + argc == 2 ? std::filesystem::path(argv[1]).parent_path() / "cli-runtime-root-test" + : std::filesystem::temp_directory_path() / "trtmc-cli-runtime-root-test"; + std::filesystem::remove_all(runtime_test_root); + const trtmc::BundleInfo runtime_bundle{1, "gpt2", "text_generation", "trt", {}}; + const auto loaded_runtime_root = runtime_test_root / "loaded-runtime"; + const auto optional_root = runtime_test_root / "optional"; + std::filesystem::create_directories(loaded_runtime_root); + std::filesystem::create_directories(optional_root); + std::set complete_roots{ + path_key(loaded_runtime_root), + path_key(optional_root), + }; + std::set byok_roots; + std::vector observed_candidates; + const trtmc::cli::RuntimeRootMatcher matches = + [&](const trtmc::BundleInfo&, const std::filesystem::path& candidate, bool require_byok) { + const std::string key = path_key(candidate); + observed_candidates.push_back(key); + return complete_roots.count(key) != 0 && (!require_byok || byok_roots.count(key) != 0); + }; + + trtmc::cli::RuntimeRootSearchContext runtime_context; + runtime_context.loaded_runtime_root = loaded_runtime_root; + runtime_context.runtime_path = optional_root.string(); + check(trtmc::cli::resolve_runtime_root(runtime_bundle, {}, false, runtime_context, matches) == + loaded_runtime_root.string(), + "the active runtime installation wins automatic discovery"); + + complete_roots.erase(path_key(loaded_runtime_root)); + check(trtmc::cli::resolve_runtime_root(runtime_bundle, {}, false, runtime_context, matches) == + optional_root.string(), + "the configured runtime path is used when the active installation is incomplete"); + complete_roots.insert(path_key(loaded_runtime_root)); + + const auto first_optional = runtime_test_root / "first-optional"; + const auto second_optional = runtime_test_root / "second-optional"; + const auto loaded_libraries = runtime_test_root / "loaded-libraries"; + std::filesystem::create_directories(first_optional); + std::filesystem::create_directories(second_optional); + std::filesystem::create_directories(loaded_libraries); + complete_roots.insert(path_key(second_optional)); + runtime_context = {}; + runtime_context.loaded_runtime_root = loaded_libraries; + runtime_context.runtime_path = first_optional.string() + ":" + second_optional.string(); + check(trtmc::cli::resolve_runtime_root(runtime_bundle, {}, false, runtime_context, matches) == + second_optional.string(), + "dedicated runtime path preserves directory order"); + + const auto current_directory_root = runtime_test_root / "current-directory"; + std::filesystem::create_directories(current_directory_root); + complete_roots.insert(path_key(current_directory_root)); + runtime_context.runtime_path.clear(); + const auto original_current_directory = std::filesystem::current_path(); + std::filesystem::current_path(current_directory_root); + observed_candidates.clear(); + std::string discovery_error; + check(resolve_throws(runtime_bundle, runtime_context, matches, discovery_error), + "runtime discovery does not search an unconfigured current directory"); + std::filesystem::current_path(original_current_directory); + check(std::find(observed_candidates.begin(), observed_candidates.end(), + path_key(current_directory_root)) == observed_candidates.end(), + "current directory participates only through explicit configuration"); + + const auto byok_root = runtime_test_root / "byok"; + std::filesystem::create_directories(byok_root); + complete_roots.insert(path_key(byok_root)); + byok_roots.insert(path_key(byok_root)); + runtime_context.runtime_path = second_optional.string() + ":" + byok_root.string(); + check(trtmc::cli::resolve_runtime_root(runtime_bundle, {}, true, runtime_context, matches) == + byok_root.string(), + "BYOK discovery delegates its extra requirement to the runtime matcher"); + + const auto explicit_root = runtime_test_root / "explicit-missing"; + check(trtmc::cli::resolve_runtime_root(runtime_bundle, explicit_root.string(), false, + runtime_context, {}) == explicit_root.string(), + "explicit runtime root bypasses automatic fallback"); + + runtime_context = {}; + runtime_context.loaded_runtime_root = loaded_libraries; + runtime_context.runtime_path = (runtime_test_root / "family-only").string(); + std::filesystem::create_directories(runtime_test_root / "family-only"); + check(resolve_throws(runtime_bundle, runtime_context, matches, discovery_error), + "runtime discovery never combines partial roots"); + check(discovery_error.find("bundle 'gpt2/trt'") != std::string::npos && + discovery_error.find("--runtime-root") != std::string::npos, + "runtime discovery failure identifies the bundle and explicit override"); + std::filesystem::remove_all(runtime_test_root); + if (argc == 2) { + check_cli_runtime_discovery(argv[1]); + check_selected_root_never_falls_back(argv[1]); + } const auto dynamic_kv = parse({"trtmc", "run", "model.bundle", "--runtime-root", "lib", "--kv-cache-size", "1GiB"}); check(dynamic_kv.kv_cache_size_bytes == 1024ULL * 1024ULL * 1024ULL, diff --git a/cmake/product_build.h.in b/cmake/product_build.h.in new file mode 100644 index 0000000000..236021f601 --- /dev/null +++ b/cmake/product_build.h.in @@ -0,0 +1,8 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#define TRTMC_BUILD_ID "@TRTMC_BUILD_ID@" diff --git a/conanfile.py b/conanfile.py index 4b69898a08..c3f22b7172 100644 --- a/conanfile.py +++ b/conanfile.py @@ -75,19 +75,16 @@ def package(self) -> None: build = Path(self.build_folder) package = Path(self.package_folder) module_bin = package / "tensorrt_model_connect" / "bin" - script_bin = package / f"{self.name.replace('-', '_')}-{self.version}.data" / "scripts" copy(self, "trtmc", src=str(build), dst=str(module_bin), keep_path=False) - copy(self, "trtmc", src=str(build), dst=str(script_bin), keep_path=False) - for destination in (module_bin, script_bin): - for library in ("libtrtmc_core.so", "libtrtmc_runtime.so"): - copy( - self, - library, - src=str(build), - dst=str(destination), - keep_path=False, - ) + for library in ("libtrtmc_core.so", "libtrtmc_runtime.so"): + copy( + self, + library, + src=str(build), + dst=str(module_bin), + keep_path=False, + ) copy( self, "libtrtmc_backend_trt*.so", @@ -146,11 +143,8 @@ def package(self) -> None: ) native = module_bin / "trtmc" - installed = script_bin / "trtmc" shared_runtime = [ - destination / library - for destination in (module_bin, script_bin) - for library in ("libtrtmc_core.so", "libtrtmc_runtime.so") + module_bin / library for library in ("libtrtmc_core.so", "libtrtmc_runtime.so") ] backend = module_bin / "libtrtmc_backend_trt.so" backends = sorted(module_bin.glob("libtrtmc_backend_trt*.so")) @@ -159,7 +153,6 @@ def package(self) -> None: dataset_benchmark = module_bin / "trtmc_dataset_benchmark" if ( not native.is_file() - or not installed.is_file() or not all(library.is_file() for library in shared_runtime) or not backend.is_file() or not byok.is_file() @@ -168,7 +161,7 @@ def package(self) -> None: ): raise ConanException("native runtime package is incomplete") - for executable in (native, installed, benchmark_worker, dataset_benchmark): + for executable in (native, benchmark_worker, dataset_benchmark): _make_executable(executable) _set_runpath(executable, "$ORIGIN") for library in shared_runtime: diff --git a/core/builder/tensorrt_model_connect/native_cli.py b/core/builder/tensorrt_model_connect/native_cli.py new file mode 100644 index 0000000000..83b3f6d617 --- /dev/null +++ b/core/builder/tensorrt_model_connect/native_cli.py @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import os +from pathlib import Path +import sys + + +def main() -> None: + """Replace the console adapter with its co-located native product CLI.""" + executable = Path(__file__).resolve().parent / "bin" / "trtmc" + if not executable.is_file() or not os.access(executable, os.X_OK): + raise RuntimeError(f"native trtmc executable is missing or not executable: {executable}") + os.execv(executable, [str(executable), *sys.argv[1:]]) diff --git a/core/builder/tests/test_native_cli.py b/core/builder/tests/test_native_cli.py new file mode 100644 index 0000000000..0284656532 --- /dev/null +++ b/core/builder/tests/test_native_cli.py @@ -0,0 +1,45 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from pathlib import Path +import sys + +import pytest + +from tensorrt_model_connect import native_cli + + +def test_main_executes_the_native_cli_in_the_package_bin( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + package = tmp_path / "tensorrt_model_connect" + executable = package / "bin" / "trtmc" + executable.parent.mkdir(parents=True) + executable.touch() + executable.chmod(0o755) + monkeypatch.setattr(native_cli, "__file__", str(package / "native_cli.py")) + monkeypatch.setattr(sys, "argv", ["trtmc", "version"]) + + called: tuple[Path, list[str]] | None = None + + def capture_execv(path: Path, arguments: list[str]) -> None: + nonlocal called + called = (path, arguments) + + monkeypatch.setattr(native_cli.os, "execv", capture_execv) + native_cli.main() + + assert called == (executable, [str(executable), "version"]) + + +def test_main_rejects_an_incomplete_native_product( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + package = tmp_path / "tensorrt_model_connect" + package.mkdir() + monkeypatch.setattr(native_cli, "__file__", str(package / "native_cli.py")) + + with pytest.raises(RuntimeError, match="native trtmc executable is missing"): + native_cli.main() diff --git a/core/runtime/byok/byok.cpp b/core/runtime/byok/byok.cpp index 1277c06bcb..fe0fcd033e 100644 --- a/core/runtime/byok/byok.cpp +++ b/core/runtime/byok/byok.cpp @@ -121,6 +121,8 @@ void load_kernel(const std::string& library, const std::string& function, } // namespace trtmc +TRTMC_DEFINE_PLUGIN_DESCRIPTOR_V1(::trtmc::PluginKind::kRuntimeExtension, "tvm_ffi") + extern "C" const char* trtmc_load_byok_kernel(const char* library, const char* function, const char* kernel_name) noexcept { static thread_local std::string error; diff --git a/core/runtime/include/trtmc/byok.h b/core/runtime/include/trtmc/byok.h index 8ac679800e..63c8f60fd0 100644 --- a/core/runtime/include/trtmc/byok.h +++ b/core/runtime/include/trtmc/byok.h @@ -5,6 +5,8 @@ #pragma once +#include "trtmc/runtime/plugin_abi.h" + #include #include diff --git a/core/runtime/include/trtmc/runtime/family_factory.h b/core/runtime/include/trtmc/runtime/family_factory.h index c0927b50b0..1a0d707cd8 100644 --- a/core/runtime/include/trtmc/runtime/family_factory.h +++ b/core/runtime/include/trtmc/runtime/family_factory.h @@ -6,6 +6,7 @@ #pragma once #include "trtmc/bundle.h" +#include "trtmc/runtime/plugin_abi.h" #include "trtmc/task.h" #include @@ -27,3 +28,6 @@ inline constexpr const char* kCreateFamilySymbol = "trtmc_create_family"; } // namespace trtmc extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context); + +#define TRTMC_DEFINE_FAMILY_PLUGIN_V1(family_id) \ + TRTMC_DEFINE_PLUGIN_DESCRIPTOR_V1(::trtmc::PluginKind::kFamily, family_id) diff --git a/core/runtime/include/trtmc/runtime/family_loader.h b/core/runtime/include/trtmc/runtime/family_loader.h index 78ccba3f3b..5f044a2b81 100644 --- a/core/runtime/include/trtmc/runtime/family_loader.h +++ b/core/runtime/include/trtmc/runtime/family_loader.h @@ -23,4 +23,9 @@ std::unique_ptr load_task(const std::string& bundle_path, const std::stri const std::string& runtime_cache_path = {}, bool cuda_graphs = false); +// Load the exact-build-checked TVM-FFI runtime extension from runtime_root, then +// publish one BYOK kernel. The extension remains resident for process lifetime. +void load_byok_kernel_from_runtime(const std::string& runtime_root, const std::string& library, + const std::string& function, const std::string& kernel_name); + } // namespace trtmc diff --git a/core/runtime/include/trtmc/runtime/plugin_abi.h b/core/runtime/include/trtmc/runtime/plugin_abi.h new file mode 100644 index 0000000000..109d01d614 --- /dev/null +++ b/core/runtime/include/trtmc/runtime/plugin_abi.h @@ -0,0 +1,57 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#if __has_include("trtmc/runtime/product_build.h") +#include "trtmc/runtime/product_build.h" +#endif + +#include + +namespace trtmc { + +inline constexpr std::uint32_t kPluginDescriptorVersion = 1; +inline constexpr const char* kPluginDescriptorSymbol = "trtmc_plugin_descriptor_v1"; + +#ifdef TRTMC_BUILD_ID +inline constexpr const char* kPluginBuildId = TRTMC_BUILD_ID; +#else +// Installed headers remain consumable, but plugins built outside a coordinated +// TRTMC product build cannot accidentally match a configured runtime. +inline constexpr const char* kPluginBuildId = "unconfigured"; +#endif + +enum class PluginKind : std::uint32_t { + kBackend = 1, + kFamily = 2, + kRuntimeExtension = 3, +}; + +// Model-agnostic identity returned by every backend, family, and extension DSO. +// V1 is an exact descriptor layout, not C++ ABI compatibility negotiation. +struct PluginDescriptorV1 { + std::uint32_t struct_size; + std::uint32_t descriptor_version; + PluginKind kind; + const char* id; + const char* build_id; +}; + +using PluginDescriptorFn = const PluginDescriptorV1* (*)() noexcept; + +} // namespace trtmc + +extern "C" const trtmc::PluginDescriptorV1* trtmc_plugin_descriptor_v1() noexcept; +extern "C" const char* trtmc_core_build_id() noexcept; +extern "C" const char* trtmc_runtime_build_id() noexcept; + +#define TRTMC_DEFINE_PLUGIN_DESCRIPTOR_V1(plugin_kind, plugin_id) \ + extern "C" const trtmc::PluginDescriptorV1* trtmc_plugin_descriptor_v1() noexcept { \ + static const trtmc::PluginDescriptorV1 descriptor{ \ + sizeof(trtmc::PluginDescriptorV1), trtmc::kPluginDescriptorVersion, plugin_kind, \ + plugin_id, trtmc::kPluginBuildId}; \ + return &descriptor; \ + } diff --git a/core/runtime/include/trtmc/runtime/runtime_root.h b/core/runtime/include/trtmc/runtime/runtime_root.h new file mode 100644 index 0000000000..a66212c0db --- /dev/null +++ b/core/runtime/include/trtmc/runtime/runtime_root.h @@ -0,0 +1,26 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include "trtmc/bundle.h" + +#include + +namespace trtmc { + +// Return the directory containing the runtime loader used by this process. +// Applications may use this as a discovery candidate; load_task remains the +// only operation that loads a family and backend. +std::string loaded_runtime_root(); + +// Return whether runtime_root contains the root-local backend and family DSOs +// named by bundle, plus BYOK when requested. This structural check never +// searches, loads, or falls back. Exact product-build and plugin identities are +// validated when the selected DSOs are loaded. +bool runtime_root_contains_bundle(const BundleInfo& bundle, const std::string& runtime_root, + bool require_byok = false); + +} // namespace trtmc diff --git a/core/runtime/include/trtmc/runtime/trt_backend.h b/core/runtime/include/trtmc/runtime/trt_backend.h index ace743553e..4200a9a49f 100644 --- a/core/runtime/include/trtmc/runtime/trt_backend.h +++ b/core/runtime/include/trtmc/runtime/trt_backend.h @@ -5,6 +5,8 @@ #pragma once +#include "trtmc/runtime/plugin_abi.h" + // IBackend: the narrow interface implemented by the TensorRT backend DSO. #include "trtmc/runtime/trt_module.h" @@ -70,3 +72,6 @@ extern "C" { trtmc::IBackend* trtmc_create_backend(); void trtmc_destroy_backend(trtmc::IBackend* backend); } + +#define TRTMC_DEFINE_BACKEND_PLUGIN_V1(backend_id) \ + TRTMC_DEFINE_PLUGIN_DESCRIPTOR_V1(::trtmc::PluginKind::kBackend, backend_id) diff --git a/core/runtime/loader/family_loader.cpp b/core/runtime/loader/family_loader.cpp index 76c419e633..1bfcfe4f72 100644 --- a/core/runtime/loader/family_loader.cpp +++ b/core/runtime/loader/family_loader.cpp @@ -7,8 +7,13 @@ #include "runtime/bundle/bundle_format.h" #include "trtmc/runtime/family_factory.h" +#include "trtmc/runtime/plugin_abi.h" +#include "trtmc/runtime/runtime_root.h" #include "trtmc/runtime/trt_backend.h" +#include +#include +#include #include #include #include @@ -18,6 +23,7 @@ #include #include #include +#include namespace trtmc { @@ -28,6 +34,30 @@ namespace fs = std::filesystem; using CreateBackendFn = IBackend* (*)(); using DestroyBackendFn = void (*)(IBackend*); +std::string backend_library_name(const std::string& backend_id) { + return "libtrtmc_backend_" + backend_id + ".so"; +} + +std::string family_library_name(const std::string& family_id) { + return "libtrtmc_model_" + family_id + ".so"; +} + +std::string runtime_extension_library_name(const std::string& extension_id) { + return "libtrtmc_byok_" + extension_id + ".so"; +} + +const char* plugin_kind_name(PluginKind kind) { + switch (kind) { + case PluginKind::kBackend: + return "backend"; + case PluginKind::kFamily: + return "family"; + case PluginKind::kRuntimeExtension: + return "runtime extension"; + } + return "unknown"; +} + bool is_safe_id(const std::string& value) { if (value.empty() || value.front() < 'a' || value.front() > 'z') return false; @@ -52,18 +82,74 @@ fs::path explicit_runtime_root(const std::string& runtime_root) { if (runtime_root.empty()) throw std::invalid_argument("runtime_root must be explicit and non-empty"); std::error_code error; - fs::path root = fs::absolute(fs::path(runtime_root), error); + const fs::path absolute = fs::absolute(fs::path(runtime_root), error); if (error) throw std::runtime_error("Unable to resolve runtime_root '" + runtime_root + "': " + error.message()); - return root.lexically_normal(); + fs::path root = fs::canonical(absolute, error); + if (error) + throw std::runtime_error("Unable to resolve runtime_root '" + runtime_root + + "': " + error.message()); + if (!fs::is_directory(root, error) || error) + throw std::runtime_error("runtime_root is not a directory: '" + runtime_root + "'"); + return root; +} + +fs::path loaded_library_path(const void* symbol) { + Dl_info info{}; + if (dladdr(symbol, &info) == 0 || info.dli_fname == nullptr) + return {}; + std::error_code error; + fs::path path = fs::absolute(info.dli_fname, error); + if (error) + return info.dli_fname; + fs::path normalized = fs::weakly_canonical(path, error); + return error ? path.lexically_normal() : normalized; +} + +bool contains_root_local_library(const fs::path& root, const std::string& library) { + std::error_code error; + const fs::path resolved = fs::canonical(root / library, error); + return !error && resolved.parent_path() == root && fs::is_regular_file(resolved, error) && + !error; +} + +fs::path require_root_local_library(const fs::path& root, const std::string& library) { + std::error_code error; + const fs::path resolved = fs::canonical(root / library, error); + if (error || !fs::is_regular_file(resolved, error) || error) { + throw std::runtime_error("Runtime root '" + root.string() + "' does not contain '" + + library + "'"); + } + if (resolved.parent_path() != root) { + throw std::runtime_error("Library '" + (root / library).string() + + "' escapes the selected runtime root"); + } + return resolved; +} + +void require_matching_build(const std::string& path, const PluginDescriptorV1& descriptor) { + if (descriptor.build_id != nullptr && std::string(descriptor.build_id) == kPluginBuildId) + return; + const std::string actual = descriptor.build_id != nullptr ? descriptor.build_id : ""; + throw std::runtime_error("Library '" + path + "' belongs to product build '" + actual + + "'; active runtime requires '" + kPluginBuildId + "'"); +} + +void require_matching_core_build() { + const char* core_build_id = trtmc_core_build_id(); + if (core_build_id != nullptr && std::string(core_build_id) == kPluginBuildId) + return; + const std::string actual = core_build_id != nullptr ? core_build_id : ""; + throw std::runtime_error("Active libtrtmc_core.so belongs to product build '" + actual + + "'; libtrtmc_runtime.so requires '" + kPluginBuildId + "'"); } class SharedLibrary { public: explicit SharedLibrary(const fs::path& path) : path_(path.string()) { dlerror(); - handle_ = dlopen(path_.c_str(), RTLD_NOW | RTLD_LOCAL); + handle_ = dlopen(path_.c_str(), RTLD_NOW | RTLD_LOCAL | RTLD_NODELETE); if (handle_ == nullptr) { const char* error = dlerror(); throw std::runtime_error("Unable to load '" + path_ + @@ -90,6 +176,37 @@ class SharedLibrary { return symbol; } + void require_plugin(PluginKind expected_kind, const std::string& expected_id) const { + const auto descriptor_function = + reinterpret_cast(require_symbol(kPluginDescriptorSymbol)); + const PluginDescriptorV1* descriptor = descriptor_function(); + if (descriptor == nullptr) { + throw std::runtime_error("Library '" + path_ + "' returned a null plugin descriptor"); + } + if (descriptor->struct_size != sizeof(PluginDescriptorV1)) { + throw std::runtime_error("Library '" + path_ + "' declares descriptor size " + + std::to_string(descriptor->struct_size) + "; expected " + + std::to_string(sizeof(PluginDescriptorV1))); + } + if (descriptor->descriptor_version != kPluginDescriptorVersion) { + throw std::runtime_error("Library '" + path_ + "' declares plugin descriptor version " + + std::to_string(descriptor->descriptor_version) + + "; expected " + std::to_string(kPluginDescriptorVersion)); + } + if (descriptor->kind != expected_kind) { + throw std::runtime_error("Library '" + path_ + "' declares a " + + plugin_kind_name(descriptor->kind) + " plugin; expected " + + plugin_kind_name(expected_kind)); + } + if (descriptor->id == nullptr || expected_id != descriptor->id) { + const std::string actual = descriptor->id != nullptr ? descriptor->id : ""; + throw std::runtime_error("Library '" + path_ + "' declares " + + plugin_kind_name(expected_kind) + " '" + actual + + "'; expected '" + expected_id + "'"); + } + require_matching_build(path_, *descriptor); + } + private: std::string path_; void* handle_{nullptr}; @@ -98,7 +215,8 @@ class SharedLibrary { class BackendLibrary { public: BackendLibrary(const fs::path& runtime_root, const std::string& backend_id) - : library_(runtime_root / ("libtrtmc_backend_" + backend_id + ".so")) { + : library_(require_root_local_library(runtime_root, backend_library_name(backend_id))) { + library_.require_plugin(PluginKind::kBackend, backend_id); const auto create = reinterpret_cast(library_.require_symbol("trtmc_create_backend")); destroy_ = @@ -136,8 +254,10 @@ class BackendLibrary { class FamilyLibrary { public: FamilyLibrary(const fs::path& runtime_root, const std::string& family_id) - : library_(runtime_root / ("libtrtmc_model_" + family_id + ".so")), - create_(reinterpret_cast(library_.require_symbol(kCreateFamilySymbol))) {} + : library_(require_root_local_library(runtime_root, family_library_name(family_id))) { + library_.require_plugin(PluginKind::kFamily, family_id); + create_ = reinterpret_cast(library_.require_symbol(kCreateFamilySymbol)); + } FamilyLibrary(const FamilyLibrary&) = delete; FamilyLibrary& operator=(const FamilyLibrary&) = delete; @@ -149,6 +269,28 @@ class FamilyLibrary { CreateFamilyFn create_{nullptr}; }; +class RuntimeExtensionLibrary { + public: + explicit RuntimeExtensionLibrary(const fs::path& runtime_root) + : library_( + require_root_local_library(runtime_root, runtime_extension_library_name("tvm_ffi"))) { + library_.require_plugin(PluginKind::kRuntimeExtension, "tvm_ffi"); + load_ = reinterpret_cast(library_.require_symbol("trtmc_load_byok_kernel")); + } + + void load(const std::string& library, const std::string& function, + const std::string& kernel_name) const { + if (const char* error = load_(library.c_str(), function.c_str(), kernel_name.c_str())) + throw std::runtime_error(error); + } + + private: + using LoadKernelFn = const char* (*)(const char*, const char*, const char*) noexcept; + + SharedLibrary library_; + LoadKernelFn load_{nullptr}; +}; + class RuntimeOptionsBackend final : public IBackend { public: RuntimeOptionsBackend(IBackend& backend, std::string runtime_cache_path, bool cuda_graphs) @@ -214,6 +356,7 @@ struct RuntimeLibraryCache { std::mutex mutex; std::unordered_map> backends; std::unordered_map> families; + std::unordered_map> extensions; std::unordered_map, ConfiguredBackendKeyHash> configured_backends; @@ -228,7 +371,7 @@ RuntimeLibraryCache& runtime_library_cache() { } IBackend& cached_backend(const fs::path& runtime_root, const std::string& backend_id) { - const std::string path = (runtime_root / ("libtrtmc_backend_" + backend_id + ".so")).string(); + const std::string path = (runtime_root / backend_library_name(backend_id)).string(); auto& cache = runtime_library_cache(); std::lock_guard lock(cache.mutex); const auto found = cache.backends.find(path); @@ -258,7 +401,7 @@ IBackend& cached_configured_backend(IBackend& backend, const std::string& runtim } FamilyLibrary& cached_family(const fs::path& runtime_root, const std::string& family_id) { - const std::string path = (runtime_root / ("libtrtmc_model_" + family_id + ".so")).string(); + const std::string path = (runtime_root / family_library_name(family_id)).string(); auto& cache = runtime_library_cache(); std::lock_guard lock(cache.mutex); const auto found = cache.families.find(path); @@ -271,6 +414,20 @@ FamilyLibrary& cached_family(const fs::path& runtime_root, const std::string& fa return family; } +RuntimeExtensionLibrary& cached_runtime_extension(const fs::path& runtime_root) { + const std::string path = (runtime_root / runtime_extension_library_name("tvm_ffi")).string(); + auto& cache = runtime_library_cache(); + std::lock_guard lock(cache.mutex); + const auto found = cache.extensions.find(path); + if (found != cache.extensions.end()) + return *found->second; + + auto library = std::make_unique(runtime_root); + RuntimeExtensionLibrary& extension = *library; + cache.extensions.emplace(path, std::move(library)); + return extension; +} + void require_matching_task(const BundleInfo& info, const ITask& task) { const char* actual = task.task(); if (actual == nullptr || info.task != actual) { @@ -283,9 +440,54 @@ void require_matching_task(const BundleInfo& info, const ITask& task) { } // namespace +std::string loaded_runtime_root() { + using LoadTaskFn = std::unique_ptr (*)(const std::string&, const std::string&, + std::uint64_t, const std::string&, bool); + const auto load_task_function = static_cast(&load_task); + const fs::path runtime_library = + loaded_library_path(reinterpret_cast(load_task_function)); + if (runtime_library.empty()) + throw std::runtime_error("Unable to locate the active TRTMC runtime loader"); + return runtime_library.parent_path().string(); +} + +bool runtime_root_contains_bundle(const BundleInfo& bundle, const std::string& runtime_root, + bool require_byok) { + require_matching_core_build(); + require_safe_id("family", bundle.family); + require_safe_id("backend", bundle.backend); + if (runtime_root.empty()) + return false; + + std::error_code error; + fs::path root = fs::canonical(fs::absolute(runtime_root, error), error); + if (error || !fs::is_directory(root, error) || error) + return false; + + std::vector required{ + backend_library_name(bundle.backend), + family_library_name(bundle.family), + }; + if (require_byok) + required.emplace_back(runtime_extension_library_name("tvm_ffi")); + + return std::all_of(required.begin(), required.end(), [&](const std::string& library) { + return contains_root_local_library(root, library); + }); +} + +void load_byok_kernel_from_runtime(const std::string& runtime_root, const std::string& library, + const std::string& function, const std::string& kernel_name) { + require_matching_core_build(); + RuntimeExtensionLibrary& extension = + cached_runtime_extension(explicit_runtime_root(runtime_root)); + extension.load(library, function, kernel_name); +} + std::unique_ptr load_task(const std::string& bundle_path, const std::string& runtime_root, std::uint64_t kv_cache_size_bytes, const std::string& runtime_cache_path, bool cuda_graphs) { + require_matching_core_build(); const BundleReader reader(bundle_path); const BundleInfo& info = reader.info(); require_safe_id("family", info.family); @@ -310,3 +512,7 @@ std::unique_ptr load_task(const std::string& bundle_path, const std::stri } } // namespace trtmc + +extern "C" const char* trtmc_runtime_build_id() noexcept { + return trtmc::kPluginBuildId; +} diff --git a/core/runtime/primitives/build_identity.cpp b/core/runtime/primitives/build_identity.cpp new file mode 100644 index 0000000000..7c2788401a --- /dev/null +++ b/core/runtime/primitives/build_identity.cpp @@ -0,0 +1,10 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "trtmc/runtime/plugin_abi.h" + +extern "C" const char* trtmc_core_build_id() noexcept { + return trtmc::kPluginBuildId; +} diff --git a/core/runtime/tensorrt/rtx_backend.cpp b/core/runtime/tensorrt/rtx_backend.cpp index 5bb004cd14..ec0a41bd02 100644 --- a/core/runtime/tensorrt/rtx_backend.cpp +++ b/core/runtime/tensorrt/rtx_backend.cpp @@ -217,6 +217,8 @@ class RtxBackend final : public IBackend { } // namespace trtmc +TRTMC_DEFINE_BACKEND_PLUGIN_V1("trt_rtx") + extern "C" trtmc::IBackend* trtmc_create_backend() { try { return new trtmc::RtxBackend(); diff --git a/core/runtime/tensorrt/trt_backend.cpp b/core/runtime/tensorrt/trt_backend.cpp index caf20f03a0..6259602e7f 100644 --- a/core/runtime/tensorrt/trt_backend.cpp +++ b/core/runtime/tensorrt/trt_backend.cpp @@ -143,6 +143,8 @@ class TrtBackend final : public IBackend { } // namespace trtmc +TRTMC_DEFINE_BACKEND_PLUGIN_V1("trt") + extern "C" trtmc::IBackend* trtmc_create_backend() { try { return new trtmc::TrtBackend(); diff --git a/core/runtime/tests/fake_backend.cpp b/core/runtime/tests/fake_backend.cpp index f97107743f..f2750b25e8 100644 --- a/core/runtime/tests/fake_backend.cpp +++ b/core/runtime/tests/fake_backend.cpp @@ -47,6 +47,17 @@ class FakeBackend final : public trtmc::IBackend { } // namespace +#ifdef TRTMC_FAKE_INCOMPATIBLE_BUILD +extern "C" const trtmc::PluginDescriptorV1* trtmc_plugin_descriptor_v1() noexcept { + static const trtmc::PluginDescriptorV1 descriptor{ + sizeof(trtmc::PluginDescriptorV1), trtmc::kPluginDescriptorVersion, + trtmc::PluginKind::kBackend, TRTMC_FAKE_BACKEND_NAME, "00000000000000000000000000000000"}; + return &descriptor; +} +#else +TRTMC_DEFINE_BACKEND_PLUGIN_V1(TRTMC_FAKE_BACKEND_NAME) +#endif + extern "C" trtmc::IBackend* trtmc_create_backend() { ++create_count; if (create_count != 1) diff --git a/core/runtime/tests/fake_core_build_identity.cpp b/core/runtime/tests/fake_core_build_identity.cpp new file mode 100644 index 0000000000..aea43e98c3 --- /dev/null +++ b/core/runtime/tests/fake_core_build_identity.cpp @@ -0,0 +1,8 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +extern "C" const char* trtmc_core_build_id() noexcept { + return "00000000000000000000000000000000"; +} diff --git a/core/runtime/tests/fake_family.cpp b/core/runtime/tests/fake_family.cpp index 289084c219..0c5e60d13e 100644 --- a/core/runtime/tests/fake_family.cpp +++ b/core/runtime/tests/fake_family.cpp @@ -44,6 +44,8 @@ class FakeForecast final : public trtmc::ITimeSeriesForecast { } // namespace +TRTMC_DEFINE_FAMILY_PLUGIN_V1("fake") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.reader.info().family != "fake") throw std::runtime_error("unexpected family"); diff --git a/core/runtime/tests/test_family_loader.cpp b/core/runtime/tests/test_family_loader.cpp index 1f77d55db2..bb3f9c96c9 100644 --- a/core/runtime/tests/test_family_loader.cpp +++ b/core/runtime/tests/test_family_loader.cpp @@ -42,15 +42,19 @@ void write_bundle(const std::filesystem::path& path, const std::string& family, output.write("{}PLAN", 6); } -bool load_throws(const std::filesystem::path& bundle, const std::string& runtime_root) { +std::string load_error(const std::filesystem::path& bundle, const std::string& runtime_root) { try { (void)trtmc::load_task(bundle.string(), runtime_root); - return false; - } catch (const std::exception&) { - return true; + return {}; + } catch (const std::exception& error) { + return error.what(); } } +bool load_throws(const std::filesystem::path& bundle, const std::string& runtime_root) { + return !load_error(bundle, runtime_root).empty(); +} + bool rtx_options_throw(const std::filesystem::path& bundle, const std::string& runtime_root) { try { (void)trtmc::load_task(bundle.string(), runtime_root, 0, "runtime.cache", true); @@ -87,14 +91,24 @@ void check_rtx_options(const std::filesystem::path& runtime_root, } // namespace int main(int argc, char** argv) { - if (argc != 2) { - std::cerr << "usage: test_family_loader RUNTIME_ROOT\n"; + if (argc != 2 && argc != 3) { + std::cerr << "usage: test_family_loader RUNTIME_ROOT [--expect-core-mismatch]\n"; return 2; } const std::filesystem::path runtime_root(argv[1]); const auto bundle_path = runtime_root / "fake.bundle"; write_bundle(bundle_path, "fake"); + if (argc == 3) { + const std::string core_error = load_error(bundle_path, runtime_root.string()); + check(std::string(argv[2]) == "--expect-core-mismatch" && + core_error.find("Active libtrtmc_core.so belongs to product build") != + std::string::npos, + "runtime rejects a core DSO from a different product build"); + std::filesystem::remove(bundle_path); + return failures; + } + auto task = trtmc::load_task(bundle_path.string(), runtime_root.string()); auto* forecast = dynamic_cast(task.get()); check(forecast != nullptr, "load returns forecast interface"); @@ -147,6 +161,47 @@ int main(int argc, char** argv) { check(load_throws(bundle_path, (runtime_root / "missing").string()), "loader does not search outside explicit root"); + const auto wrong_backend_library = runtime_root / "libtrtmc_backend_other.so"; + const auto wrong_backend_bundle = runtime_root / "wrong-backend.bundle"; + std::filesystem::copy_file(runtime_root / "libtrtmc_backend_fake.so", wrong_backend_library, + std::filesystem::copy_options::overwrite_existing); + write_bundle(wrong_backend_bundle, "fake", "time_series_forecast", "other"); + const std::string backend_descriptor_error = + load_error(wrong_backend_bundle, runtime_root.string()); + check(backend_descriptor_error.find("declares backend 'fake'; expected 'other'") != + std::string::npos, + "explicit root validates backend plugin identity before its factory"); + + const auto incompatible_bundle = runtime_root / "incompatible.bundle"; + write_bundle(incompatible_bundle, "fake", "time_series_forecast", "incompatible"); + const std::string incompatible_build_error = + load_error(incompatible_bundle, runtime_root.string()); + check(incompatible_build_error.find("belongs to product build") != std::string::npos, + "loader rejects a plugin from a different product build before its factory"); + + const auto escaped_root = runtime_root.parent_path() / "escaped-runtime-root"; + std::filesystem::remove_all(escaped_root); + std::filesystem::create_directories(escaped_root); + std::filesystem::create_symlink(runtime_root / "libtrtmc_backend_fake.so", + escaped_root / "libtrtmc_backend_fake.so"); + std::filesystem::create_symlink(runtime_root / "libtrtmc_model_fake.so", + escaped_root / "libtrtmc_model_fake.so"); + const std::string escaped_root_error = load_error(bundle_path, escaped_root.string()); + check(escaped_root_error.find("escapes the selected runtime root") != std::string::npos, + "explicit loading rejects a DSO symlink that escapes the selected root"); + std::filesystem::remove_all(escaped_root); + + const auto wrong_family_library = runtime_root / "libtrtmc_model_other.so"; + const auto wrong_family_bundle = runtime_root / "wrong-family.bundle"; + std::filesystem::copy_file(runtime_root / "libtrtmc_model_fake.so", wrong_family_library, + std::filesystem::copy_options::overwrite_existing); + write_bundle(wrong_family_bundle, "other"); + const std::string family_descriptor_error = + load_error(wrong_family_bundle, runtime_root.string()); + check(family_descriptor_error.find("declares family 'fake'; expected 'other'") != + std::string::npos, + "explicit root validates family plugin identity before its factory"); + const auto unsafe_bundle = runtime_root / "unsafe.bundle"; write_bundle(unsafe_bundle, "../fake"); check(load_throws(unsafe_bundle, runtime_root.string()), "unsafe family id rejected"); @@ -160,6 +215,11 @@ int main(int argc, char** argv) { std::filesystem::remove(unsafe_bundle); std::filesystem::remove(mismatch_bundle); std::filesystem::remove(rtx_bundle); + std::filesystem::remove(wrong_backend_bundle); + std::filesystem::remove(wrong_backend_library); + std::filesystem::remove(incompatible_bundle); + std::filesystem::remove(wrong_family_bundle); + std::filesystem::remove(wrong_family_library); std::cerr << (failures == 0 ? "ALL PASSED\n" : "SOME FAILED\n"); return failures; } diff --git a/core/runtime/tests/test_runtime_root.cpp b/core/runtime/tests/test_runtime_root.cpp new file mode 100644 index 0000000000..0bf7060c6c --- /dev/null +++ b/core/runtime/tests/test_runtime_root.cpp @@ -0,0 +1,87 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "trtmc/runtime/runtime_root.h" + +#include +#include +#include +#include + +namespace { + +namespace fs = std::filesystem; + +int failures = 0; + +void check(bool condition, const char* name) { + if (!condition) { + std::cerr << "FAIL: " << name << '\n'; + ++failures; + } +} + +void copy_library(const fs::path& source, const fs::path& root, const char* name) { + fs::copy_file(source, root / name, fs::copy_options::overwrite_existing); +} + +} // namespace + +int main(int argc, char** argv) { + if (argc != 5) { + std::cerr << "usage: test_runtime_root ROOT RUNTIME BACKEND FAMILY\n"; + return 2; + } + + const fs::path root = argv[1]; + const fs::path runtime_library = argv[2]; + const fs::path backend_library = argv[3]; + const fs::path family_library = argv[4]; + fs::remove_all(root); + fs::create_directories(root); + + copy_library(backend_library, root, "libtrtmc_backend_fake.so"); + copy_library(family_library, root, "libtrtmc_model_fake.so"); + const trtmc::BundleInfo bundle{1, "fake", "time_series_forecast", "fake", {}}; + + std::error_code error; + check(fs::equivalent(trtmc::loaded_runtime_root(), runtime_library.parent_path(), error), + "runtime loader reports its active installation root"); + check(trtmc::runtime_root_contains_bundle(bundle, root.string()), + "loader contract accepts one complete plugin root"); + + fs::remove(root / "libtrtmc_backend_fake.so"); + check(!trtmc::runtime_root_contains_bundle(bundle, root.string()), + "loader contract rejects an incomplete root"); + copy_library(backend_library, root, "libtrtmc_backend_fake.so"); + + check(!trtmc::runtime_root_contains_bundle(bundle, root.string(), true), + "loader contract requires the BYOK DSO when requested"); + copy_library(runtime_library, root, "libtrtmc_byok_tvm_ffi.so"); + check(trtmc::runtime_root_contains_bundle(bundle, root.string(), true), + "loader contract accepts a root-local BYOK DSO"); + + fs::remove(root / "libtrtmc_model_fake.so"); + fs::create_symlink(family_library, root / "libtrtmc_model_fake.so"); + check(!trtmc::runtime_root_contains_bundle(bundle, root.string()), + "loader contract rejects a DSO symlink that escapes the selected root"); + + bool unsafe_rejected = false; + try { + const trtmc::BundleInfo unsafe{1, "../fake", "time_series_forecast", "fake", {}}; + (void)trtmc::runtime_root_contains_bundle(unsafe, root.string()); + } catch (const std::runtime_error&) { + unsafe_rejected = true; + } + check(unsafe_rejected, "loader contract owns safe family and backend validation"); + + fs::remove_all(root); + if (failures != 0) { + std::cerr << failures << " runtime-root test(s) failed\n"; + return 1; + } + std::cout << "runtime-root tests passed\n"; + return 0; +} diff --git a/families/albert/runtime/plugin.cpp b/families/albert/runtime/plugin.cpp index 70548ff353..df98ccb9e0 100644 --- a/families/albert/runtime/plugin.cpp +++ b/families/albert/runtime/plugin.cpp @@ -55,6 +55,8 @@ std::string require_task(const BundleInfo& info) { } // namespace } // namespace trtmc::albert_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("albert") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("albert does not support --kv-cache-size"); diff --git a/families/bark/runtime/plugin.cpp b/families/bark/runtime/plugin.cpp index 02d489d468..2f3414a86b 100644 --- a/families/bark/runtime/plugin.cpp +++ b/families/bark/runtime/plugin.cpp @@ -70,6 +70,8 @@ BarkConfig parse_config(const nlohmann::json& json) { } // namespace } // namespace trtmc::bark_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("bark") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("bark does not support --kv-cache-size"); diff --git a/families/bart/runtime/plugin.cpp b/families/bart/runtime/plugin.cpp index c0639033c0..7946d359dd 100644 --- a/families/bart/runtime/plugin.cpp +++ b/families/bart/runtime/plugin.cpp @@ -358,6 +358,8 @@ ITask* create_bart(const FamilyContext& context) { } // namespace trtmc +TRTMC_DEFINE_FAMILY_PLUGIN_V1("bart") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("bart does not support --kv-cache-size"); diff --git a/families/bert/runtime/plugin.cpp b/families/bert/runtime/plugin.cpp index 4d20f2374a..17751f7224 100644 --- a/families/bert/runtime/plugin.cpp +++ b/families/bert/runtime/plugin.cpp @@ -55,6 +55,8 @@ std::string require_task(const BundleInfo& info) { } // namespace } // namespace trtmc::bert_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("bert") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("bert does not support --kv-cache-size"); diff --git a/families/bloom/runtime/plugin.cpp b/families/bloom/runtime/plugin.cpp index 49cbc07eee..79acca59ff 100644 --- a/families/bloom/runtime/plugin.cpp +++ b/families/bloom/runtime/plugin.cpp @@ -208,6 +208,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::bloom +TRTMC_DEFINE_FAMILY_PLUGIN_V1("bloom") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("bloom does not support --kv-cache-size"); diff --git a/families/boltz2/runtime/plugin.cpp b/families/boltz2/runtime/plugin.cpp index 691dfabb11..bf832f6bf0 100644 --- a/families/boltz2/runtime/plugin.cpp +++ b/families/boltz2/runtime/plugin.cpp @@ -90,6 +90,8 @@ boltz2::BundleArtifacts loadArtifacts(const BundleReader& bundle) { } // namespace } // namespace trtmc +TRTMC_DEFINE_FAMILY_PLUGIN_V1("boltz2") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("boltz2 does not support --kv-cache-size"); diff --git a/families/canary/runtime/plugin.cpp b/families/canary/runtime/plugin.cpp index 85cad3b8dc..5ccf8fe8ca 100644 --- a/families/canary/runtime/plugin.cpp +++ b/families/canary/runtime/plugin.cpp @@ -28,6 +28,8 @@ std::vector require_section(const BundleReader& bundle, const char* name) } // namespace } // namespace trtmc::canary_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("canary") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("canary does not support --kv-cache-size"); diff --git a/families/chronos_bolt/runtime/plugin.cpp b/families/chronos_bolt/runtime/plugin.cpp index d6def3a036..428de4d21d 100644 --- a/families/chronos_bolt/runtime/plugin.cpp +++ b/families/chronos_bolt/runtime/plugin.cpp @@ -69,6 +69,8 @@ std::unique_ptr load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_module(IBackend& backend, const std::vector required(const trtmc::BundleReader& b, const char* name) { } } // namespace +TRTMC_DEFINE_FAMILY_PLUGIN_V1("glmasr") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { using namespace trtmc; if (context.kv_cache_size_bytes != 0) diff --git a/families/gpt2/runtime/plugin.cpp b/families/gpt2/runtime/plugin.cpp index 30fb55467d..87ae3a892a 100644 --- a/families/gpt2/runtime/plugin.cpp +++ b/families/gpt2/runtime/plugin.cpp @@ -200,6 +200,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::gpt2 +TRTMC_DEFINE_FAMILY_PLUGIN_V1("gpt2") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("gpt2 does not support --kv-cache-size"); diff --git a/families/gpt_neo/runtime/plugin.cpp b/families/gpt_neo/runtime/plugin.cpp index b49f3cd412..10fd4448a9 100644 --- a/families/gpt_neo/runtime/plugin.cpp +++ b/families/gpt_neo/runtime/plugin.cpp @@ -208,6 +208,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::gpt_neo +TRTMC_DEFINE_FAMILY_PLUGIN_V1("gpt_neo") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("gpt_neo does not support --kv-cache-size"); diff --git a/families/gpt_neox/runtime/plugin.cpp b/families/gpt_neox/runtime/plugin.cpp index 75c29ac4c0..21d67ebd4d 100644 --- a/families/gpt_neox/runtime/plugin.cpp +++ b/families/gpt_neox/runtime/plugin.cpp @@ -208,6 +208,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::gpt_neox +TRTMC_DEFINE_FAMILY_PLUGIN_V1("gpt_neox") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("gpt_neox does not support --kv-cache-size"); diff --git a/families/gpt_oss/runtime/plugin.cpp b/families/gpt_oss/runtime/plugin.cpp index e72ab778bc..5fb824d725 100644 --- a/families/gpt_oss/runtime/plugin.cpp +++ b/families/gpt_oss/runtime/plugin.cpp @@ -214,6 +214,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::gpt_oss +TRTMC_DEFINE_FAMILY_PLUGIN_V1("gpt_oss") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("gpt_oss does not support --kv-cache-size"); diff --git a/families/granite/runtime/plugin.cpp b/families/granite/runtime/plugin.cpp index ffa5b633ed..68b666052a 100644 --- a/families/granite/runtime/plugin.cpp +++ b/families/granite/runtime/plugin.cpp @@ -208,6 +208,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::granite +TRTMC_DEFINE_FAMILY_PLUGIN_V1("granite") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("granite does not support --kv-cache-size"); diff --git a/families/internlm/runtime/plugin.cpp b/families/internlm/runtime/plugin.cpp index 6ac0011ebd..17ce6a76ed 100644 --- a/families/internlm/runtime/plugin.cpp +++ b/families/internlm/runtime/plugin.cpp @@ -216,6 +216,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::internlm +TRTMC_DEFINE_FAMILY_PLUGIN_V1("internlm") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("internlm does not support --kv-cache-size"); diff --git a/families/internvl/runtime/plugin.cpp b/families/internvl/runtime/plugin.cpp index 15a328d5b2..dd67feac29 100644 --- a/families/internvl/runtime/plugin.cpp +++ b/families/internvl/runtime/plugin.cpp @@ -63,6 +63,8 @@ InternvlKvCacheNames build_kv_names(const nlohmann::json& config, std::int32_t n } // namespace } // namespace trtmc::internvl_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("internvl") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("internvl does not support --kv-cache-size"); diff --git a/families/k2_horizon/runtime/plugin.cpp b/families/k2_horizon/runtime/plugin.cpp index cc8a316aab..da1da41b83 100644 --- a/families/k2_horizon/runtime/plugin.cpp +++ b/families/k2_horizon/runtime/plugin.cpp @@ -211,6 +211,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::k2_horizon +TRTMC_DEFINE_FAMILY_PLUGIN_V1("k2_horizon") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("k2_horizon does not support --kv-cache-size"); diff --git a/families/k2_horizon_uno/runtime/plugin.cpp b/families/k2_horizon_uno/runtime/plugin.cpp index e6a4478a04..c803b60222 100644 --- a/families/k2_horizon_uno/runtime/plugin.cpp +++ b/families/k2_horizon_uno/runtime/plugin.cpp @@ -218,6 +218,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::k2_horizon_uno +TRTMC_DEFINE_FAMILY_PLUGIN_V1("k2_horizon_uno") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("k2_horizon_uno does not support --kv-cache-size"); diff --git a/families/lance/runtime/plugin.cpp b/families/lance/runtime/plugin.cpp index ff8e047891..acbd73d476 100644 --- a/families/lance/runtime/plugin.cpp +++ b/families/lance/runtime/plugin.cpp @@ -76,6 +76,8 @@ LanceKvCacheNames build_kv_names(const nlohmann::json& config, std::int32_t num_ } // namespace } // namespace trtmc::lance_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("lance") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("lance does not support --kv-cache-size"); diff --git a/families/lerobot_act/runtime/plugin.cpp b/families/lerobot_act/runtime/plugin.cpp index 087bd11ee9..c3adebdf26 100644 --- a/families/lerobot_act/runtime/plugin.cpp +++ b/families/lerobot_act/runtime/plugin.cpp @@ -82,6 +82,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::lerobot_act +TRTMC_DEFINE_FAMILY_PLUGIN_V1("lerobot_act") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("lerobot_act does not support --kv-cache-size"); diff --git a/families/lfm2/runtime/plugin.cpp b/families/lfm2/runtime/plugin.cpp index 693e9968d4..ffe027d5e8 100644 --- a/families/lfm2/runtime/plugin.cpp +++ b/families/lfm2/runtime/plugin.cpp @@ -144,6 +144,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::lfm2 +TRTMC_DEFINE_FAMILY_PLUGIN_V1("lfm2") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("lfm2 does not support --kv-cache-size"); diff --git a/families/llama/runtime/plugin.cpp b/families/llama/runtime/plugin.cpp index 9a4d3f4408..4359642c9e 100644 --- a/families/llama/runtime/plugin.cpp +++ b/families/llama/runtime/plugin.cpp @@ -238,6 +238,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::llama +TRTMC_DEFINE_FAMILY_PLUGIN_V1("llama") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { return trtmc::llama::create(context); } diff --git a/families/locateanything/runtime/plugin.cpp b/families/locateanything/runtime/plugin.cpp index d3a7045d30..a1cd935b4a 100644 --- a/families/locateanything/runtime/plugin.cpp +++ b/families/locateanything/runtime/plugin.cpp @@ -64,6 +64,8 @@ LocateanythingKvCacheNames build_kv_names(const nlohmann::json& config, std::int } // namespace } // namespace trtmc::locateanything_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("locateanything") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("locateanything does not support --kv-cache-size"); diff --git a/families/ltx_video/runtime/plugin.cpp b/families/ltx_video/runtime/plugin.cpp index 93d18ab25d..73fe3468ba 100644 --- a/families/ltx_video/runtime/plugin.cpp +++ b/families/ltx_video/runtime/plugin.cpp @@ -32,6 +32,8 @@ std::string section_text(const BundleReader& bundle, const char* name) { } // namespace } // namespace trtmc::ltx_video_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("ltx_video") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("ltx_video does not support --kv-cache-size"); diff --git a/families/m2m_100/runtime/plugin.cpp b/families/m2m_100/runtime/plugin.cpp index 0e51a7cbb1..1ff8104097 100644 --- a/families/m2m_100/runtime/plugin.cpp +++ b/families/m2m_100/runtime/plugin.cpp @@ -358,6 +358,8 @@ ITask* create_m2m_100(const FamilyContext& context) { } // namespace trtmc +TRTMC_DEFINE_FAMILY_PLUGIN_V1("m2m_100") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("m2m_100 does not support --kv-cache-size"); diff --git a/families/magpie_tts/runtime/plugin.cpp b/families/magpie_tts/runtime/plugin.cpp index 1d1623b4f0..9266aedd75 100644 --- a/families/magpie_tts/runtime/plugin.cpp +++ b/families/magpie_tts/runtime/plugin.cpp @@ -27,6 +27,8 @@ std::vector require_section(const BundleReader& bundle, const char* name) } // namespace } // namespace trtmc::magpie_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("magpie_tts") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("magpie_tts does not support --kv-cache-size"); diff --git a/families/mamba/runtime/plugin.cpp b/families/mamba/runtime/plugin.cpp index 1fafc43e39..178e05f81b 100644 --- a/families/mamba/runtime/plugin.cpp +++ b/families/mamba/runtime/plugin.cpp @@ -119,6 +119,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::mamba +TRTMC_DEFINE_FAMILY_PLUGIN_V1("mamba") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("mamba does not support --kv-cache-size"); diff --git a/families/marian/runtime/plugin.cpp b/families/marian/runtime/plugin.cpp index e2b75c30e6..f5cef4d54c 100644 --- a/families/marian/runtime/plugin.cpp +++ b/families/marian/runtime/plugin.cpp @@ -411,6 +411,8 @@ ITask* create_marian(const FamilyContext& context) { } // namespace trtmc +TRTMC_DEFINE_FAMILY_PLUGIN_V1("marian") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("marian does not support --kv-cache-size"); diff --git a/families/minimax_h3/runtime/plugin.cpp b/families/minimax_h3/runtime/plugin.cpp index bef6a2fc4e..3724a44a8f 100644 --- a/families/minimax_h3/runtime/plugin.cpp +++ b/families/minimax_h3/runtime/plugin.cpp @@ -71,6 +71,8 @@ MiniMaxH3ModuleLoader make_loader(IBackend& backend, PlanMap plans) { } // namespace } // namespace trtmc::minimax_h3_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("minimax_h3") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("minimax_h3 does not support --kv-cache-size"); diff --git a/families/mistral/runtime/plugin.cpp b/families/mistral/runtime/plugin.cpp index a9598bd6d5..c6e2e7d89d 100644 --- a/families/mistral/runtime/plugin.cpp +++ b/families/mistral/runtime/plugin.cpp @@ -171,6 +171,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::mistral +TRTMC_DEFINE_FAMILY_PLUGIN_V1("mistral") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("mistral does not support --kv-cache-size"); diff --git a/families/mixtral/runtime/plugin.cpp b/families/mixtral/runtime/plugin.cpp index 0a0ff108e4..16946dd6cb 100644 --- a/families/mixtral/runtime/plugin.cpp +++ b/families/mixtral/runtime/plugin.cpp @@ -214,6 +214,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::mixtral +TRTMC_DEFINE_FAMILY_PLUGIN_V1("mixtral") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("mixtral does not support --kv-cache-size"); diff --git a/families/modernbert/runtime/plugin.cpp b/families/modernbert/runtime/plugin.cpp index d221cb2be3..68bc1d7b5e 100644 --- a/families/modernbert/runtime/plugin.cpp +++ b/families/modernbert/runtime/plugin.cpp @@ -55,6 +55,8 @@ std::string require_task(const BundleInfo& info) { } // namespace } // namespace trtmc::modernbert_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("modernbert") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("modernbert does not support --kv-cache-size"); diff --git a/families/moge/runtime/plugin.cpp b/families/moge/runtime/plugin.cpp index 946489a569..5ba09e207e 100644 --- a/families/moge/runtime/plugin.cpp +++ b/families/moge/runtime/plugin.cpp @@ -40,6 +40,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::moge +TRTMC_DEFINE_FAMILY_PLUGIN_V1("moge") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("moge does not support --kv-cache-size"); diff --git a/families/mpnet/runtime/plugin.cpp b/families/mpnet/runtime/plugin.cpp index 6fb5b30048..6066325951 100644 --- a/families/mpnet/runtime/plugin.cpp +++ b/families/mpnet/runtime/plugin.cpp @@ -55,6 +55,8 @@ std::string require_task(const BundleInfo& info) { } // namespace } // namespace trtmc::mpnet_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("mpnet") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("mpnet does not support --kv-cache-size"); diff --git a/families/nemotron/runtime/plugin.cpp b/families/nemotron/runtime/plugin.cpp index 436cd825e5..ce9cc0a6a8 100644 --- a/families/nemotron/runtime/plugin.cpp +++ b/families/nemotron/runtime/plugin.cpp @@ -171,6 +171,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::nemotron +TRTMC_DEFINE_FAMILY_PLUGIN_V1("nemotron") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("nemotron does not support --kv-cache-size"); diff --git a/families/nemotron_h/runtime/plugin.cpp b/families/nemotron_h/runtime/plugin.cpp index 7ab2faaff9..a986a8e083 100644 --- a/families/nemotron_h/runtime/plugin.cpp +++ b/families/nemotron_h/runtime/plugin.cpp @@ -212,6 +212,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::nemotron_h +TRTMC_DEFINE_FAMILY_PLUGIN_V1("nemotron_h") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("nemotron_h does not support --kv-cache-size"); diff --git a/families/nemotron_labs_diffusion/runtime/plugin.cpp b/families/nemotron_labs_diffusion/runtime/plugin.cpp index 89a0098fbc..a2b451604e 100644 --- a/families/nemotron_labs_diffusion/runtime/plugin.cpp +++ b/families/nemotron_labs_diffusion/runtime/plugin.cpp @@ -187,6 +187,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::nemotron_labs_diffusion +TRTMC_DEFINE_FAMILY_PLUGIN_V1("nemotron_labs_diffusion") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("nemotron_labs_diffusion does not support --kv-cache-size"); diff --git a/families/nemotron_speech_streaming/runtime/plugin.cpp b/families/nemotron_speech_streaming/runtime/plugin.cpp index 74e4119241..057f95119d 100644 --- a/families/nemotron_speech_streaming/runtime/plugin.cpp +++ b/families/nemotron_speech_streaming/runtime/plugin.cpp @@ -70,6 +70,8 @@ RnntConfig parse_config(const nlohmann::json& json) { } // namespace } // namespace trtmc::nemotron_streaming_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("nemotron_speech_streaming") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("nemotron_speech_streaming does not support --kv-cache-size"); diff --git a/families/nemotron_voicechat/runtime/plugin.cpp b/families/nemotron_voicechat/runtime/plugin.cpp index 1a0cb1fd86..44c658f346 100644 --- a/families/nemotron_voicechat/runtime/plugin.cpp +++ b/families/nemotron_voicechat/runtime/plugin.cpp @@ -144,6 +144,8 @@ VoiceChatTtsPrompt load_tts_prompt(const BundleReader& bundle, } // namespace } // namespace trtmc::voicechat_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("nemotron_voicechat") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("nemotron_voicechat does not support --kv-cache-size"); diff --git a/families/olmo/runtime/plugin.cpp b/families/olmo/runtime/plugin.cpp index 2ce4280186..bf5cfeae1c 100644 --- a/families/olmo/runtime/plugin.cpp +++ b/families/olmo/runtime/plugin.cpp @@ -208,6 +208,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::olmo +TRTMC_DEFINE_FAMILY_PLUGIN_V1("olmo") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("olmo does not support --kv-cache-size"); diff --git a/families/olmo2/runtime/plugin.cpp b/families/olmo2/runtime/plugin.cpp index 813ed600b3..d64b2ffc26 100644 --- a/families/olmo2/runtime/plugin.cpp +++ b/families/olmo2/runtime/plugin.cpp @@ -208,6 +208,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::olmo2 +TRTMC_DEFINE_FAMILY_PLUGIN_V1("olmo2") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("olmo2 does not support --kv-cache-size"); diff --git a/families/openfold3/runtime/plugin.cpp b/families/openfold3/runtime/plugin.cpp index d210aa28ee..8f2983301b 100644 --- a/families/openfold3/runtime/plugin.cpp +++ b/families/openfold3/runtime/plugin.cpp @@ -108,6 +108,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::openfold3 +TRTMC_DEFINE_FAMILY_PLUGIN_V1("openfold3") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { return trtmc::openfold3::create(context); } diff --git a/families/openfold3/tests/cpp/fake_build_identity.cpp b/families/openfold3/tests/cpp/fake_build_identity.cpp new file mode 100644 index 0000000000..3a9b524c02 --- /dev/null +++ b/families/openfold3/tests/cpp/fake_build_identity.cpp @@ -0,0 +1,8 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +extern "C" int openfold3_test_build_id() { + return TEST_BUILD_ID; +} diff --git a/families/openfold3/tests/cpp/fake_qualification_runtime.cpp b/families/openfold3/tests/cpp/fake_qualification_runtime.cpp new file mode 100644 index 0000000000..3d858ebe7c --- /dev/null +++ b/families/openfold3/tests/cpp/fake_qualification_runtime.cpp @@ -0,0 +1,44 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "trtmc/openfold3/structure_prediction.h" +#include "trtmc/runtime/family_loader.h" + +#include +#include +#include + +extern "C" int openfold3_test_build_id(); + +namespace { + +class Prediction final : public trtmc::openfold3::IStructurePrediction { + public: + trtmc::openfold3::StructurePredictionResult predict_structure(const std::string&) override { + return {"data_test\n", "{\"build_id\":" + std::to_string(TEST_BUILD_ID) + "}"}; + } +}; + +} // namespace + +namespace trtmc { + +std::unique_ptr load_task(const std::string&, const std::string& runtime_root, std::uint64_t, + const std::string&, bool) { + const auto backend = std::filesystem::path(runtime_root) / "libtrtmc_backend_trt.so"; + void* library = dlopen(backend.c_str(), RTLD_NOW | RTLD_LOCAL); + if (library == nullptr) + throw std::runtime_error(dlerror()); + const auto backend_build = + reinterpret_cast(dlsym(library, "openfold3_test_build_id")); + const bool matches = backend_build != nullptr && backend_build() == TEST_BUILD_ID && + openfold3_test_build_id() == TEST_BUILD_ID; + dlclose(library); + if (!matches) + throw std::runtime_error("qualification runtime/core/backend product build mismatch"); + return std::make_unique(); +} + +} // namespace trtmc diff --git a/families/openfold3/tests/test_e2e.py b/families/openfold3/tests/test_e2e.py index bf8bff6481..1ac4a31dc9 100644 --- a/families/openfold3/tests/test_e2e.py +++ b/families/openfold3/tests/test_e2e.py @@ -167,6 +167,16 @@ def _run_native( ) -> tuple[str, dict]: structure = output_root / f"prediction-{index}.cif" metadata = output_root / f"prediction-{index}.json" + # The qualification executable is built separately from the selected wheel. + # Resolve the staged core so copied and symlinked roots both select its + # companion loader instead of the executable's source-build RUNPATH. + library_root = (runtime_root / "libtrtmc_core.so").resolve(strict=True).parent + loader = library_root / "libtrtmc_runtime.so" + assert loader.is_file(), f"OpenFold3 runtime is missing its companion loader: {loader}" + environment = os.environ.copy() + environment["LD_LIBRARY_PATH"] = os.pathsep.join( + path for path in (str(library_root), environment.get("LD_LIBRARY_PATH", "")) if path + ) subprocess.run( [ str(qualification), @@ -178,6 +188,7 @@ def _run_native( ], check=True, timeout=timeout, + env=environment, ) record_evidence("native", {"structure": structure, "confidence": metadata}) return structure.read_text(encoding="utf-8"), json.loads(metadata.read_text(encoding="utf-8")) diff --git a/families/openfold3/tests/test_e2e_runtime.py b/families/openfold3/tests/test_e2e_runtime.py new file mode 100644 index 0000000000..54148637d1 --- /dev/null +++ b/families/openfold3/tests/test_e2e_runtime.py @@ -0,0 +1,123 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import os +from pathlib import Path +import shutil +import subprocess + +import pytest + +from . import test_e2e as e2e + + +@pytest.fixture(scope="module") +def qualification_builds(tmp_path_factory): + root = tmp_path_factory.mktemp("openfold3-builds") + repository = Path(__file__).resolve().parents[3] + sources = Path(__file__).with_name("cpp") + compiler = shutil.which("c++") + assert compiler, "OpenFold3 runtime regression requires a C++ compiler" + flags = [ + compiler, + "-std=c++17", + f"-I{repository / 'core/runtime/include'}", + f"-I{repository / 'families/openfold3/include'}", + ] + native, wheel = root / "native-build", root / "wheel" + for build_id, directory in enumerate((native, wheel), start=1): + directory.mkdir() + for name in ("core", "backend_trt"): + library = f"libtrtmc_{name}.so" + subprocess.run( + [ + *flags, + "-shared", + "-fPIC", + f"-DTEST_BUILD_ID={build_id}", + str(sources / "fake_build_identity.cpp"), + f"-Wl,-soname,{library}", + "-o", + str(directory / library), + ], + check=True, + ) + subprocess.run( + [ + *flags, + "-shared", + "-fPIC", + f"-DTEST_BUILD_ID={build_id}", + str(sources / "fake_qualification_runtime.cpp"), + f"-L{directory}", + "-ltrtmc_core", + "-ldl", + "-Wl,-soname,libtrtmc_runtime.so", + "-Wl,-rpath,$ORIGIN", + "-o", + str(directory / "libtrtmc_runtime.so"), + ], + check=True, + ) + qualification = native / "openfold3_qualification" + subprocess.run( + [ + *flags, + str(sources / "qualification.cpp"), + f"-L{native}", + "-ltrtmc_runtime", + f"-Wl,-rpath,{native}", + f"-Wl,-rpath-link,{native}", + "-o", + str(qualification), + ], + check=True, + ) + return qualification, wheel + + +@pytest.mark.parametrize("layout", ("copied", "symlinked")) +def test_qualification_uses_selected_product_build( + qualification_builds, monkeypatch, tmp_path: Path, layout: str +) -> None: + qualification, wheel = qualification_builds + runtime_root = tmp_path / "runtime" + runtime_root.mkdir() + for name in ("libtrtmc_core.so", "libtrtmc_backend_trt.so", "libtrtmc_runtime.so"): + if layout == "copied": + shutil.copy2(wheel / name, runtime_root / name) + elif name != "libtrtmc_runtime.so": + (runtime_root / name).symlink_to(wheel / name) + # The source-built executable's RUNPATH and the inherited environment both + # point at a different product build from the selected wheel libraries. + inherited_path = str(qualification.parent) + monkeypatch.setenv("LD_LIBRARY_PATH", inherited_path) + request = tmp_path / "query.json" + request.write_text("{}", encoding="utf-8") + + structure, metadata = e2e._run_native( + qualification, runtime_root, tmp_path / "model.bundle", request, tmp_path, 0, 30 + ) + + assert structure == "data_test\n" + assert metadata == {"build_id": 2} + assert os.environ["LD_LIBRARY_PATH"] == inherited_path + + +def test_qualification_rejects_a_missing_companion_loader(tmp_path: Path) -> None: + runtime_root = tmp_path / "runtime" + runtime_root.mkdir() + (runtime_root / "libtrtmc_core.so").touch() + + with pytest.raises(AssertionError, match="missing its companion loader"): + e2e._run_native( + tmp_path / "qualification", + runtime_root, + tmp_path / "model.bundle", + tmp_path / "query.json", + tmp_path, + 0, + 30, + ) diff --git a/families/opt/runtime/plugin.cpp b/families/opt/runtime/plugin.cpp index 21b4558207..c61bedea2d 100644 --- a/families/opt/runtime/plugin.cpp +++ b/families/opt/runtime/plugin.cpp @@ -208,6 +208,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::opt +TRTMC_DEFINE_FAMILY_PLUGIN_V1("opt") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("opt does not support --kv-cache-size"); diff --git a/families/patchtsmixer/runtime/plugin.cpp b/families/patchtsmixer/runtime/plugin.cpp index 1e7aa3aac9..9f7ccafc0a 100644 --- a/families/patchtsmixer/runtime/plugin.cpp +++ b/families/patchtsmixer/runtime/plugin.cpp @@ -11,6 +11,8 @@ #include #include +TRTMC_DEFINE_FAMILY_PLUGIN_V1("patchtsmixer") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("patchtsmixer does not support --kv-cache-size"); diff --git a/families/patchtst/runtime/plugin.cpp b/families/patchtst/runtime/plugin.cpp index 200fa8035d..e4a44d52f4 100644 --- a/families/patchtst/runtime/plugin.cpp +++ b/families/patchtst/runtime/plugin.cpp @@ -76,6 +76,8 @@ std::unique_ptr load_module(IBackend& backend, const std::vector require_section(const BundleReader& bundle, const char* name) } // namespace } // namespace trtmc::pixart_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("pixart") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("pixart does not support --kv-cache-size"); diff --git a/families/qwen/runtime/plugin.cpp b/families/qwen/runtime/plugin.cpp index fe8d8a30bb..a314c7b977 100644 --- a/families/qwen/runtime/plugin.cpp +++ b/families/qwen/runtime/plugin.cpp @@ -233,6 +233,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::qwen +TRTMC_DEFINE_FAMILY_PLUGIN_V1("qwen") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("qwen does not support --kv-cache-size"); diff --git a/families/qwen3_5/runtime/plugin.cpp b/families/qwen3_5/runtime/plugin.cpp index 7786e34b43..2119cb307c 100644 --- a/families/qwen3_5/runtime/plugin.cpp +++ b/families/qwen3_5/runtime/plugin.cpp @@ -160,6 +160,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::qwen3_5 +TRTMC_DEFINE_FAMILY_PLUGIN_V1("qwen3_5") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("qwen3_5 does not support --kv-cache-size"); diff --git a/families/qwen3_8/runtime/plugin.cpp b/families/qwen3_8/runtime/plugin.cpp index 49a8b4c57f..66349b8629 100644 --- a/families/qwen3_8/runtime/plugin.cpp +++ b/families/qwen3_8/runtime/plugin.cpp @@ -160,6 +160,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::qwen3_8 +TRTMC_DEFINE_FAMILY_PLUGIN_V1("qwen3_8") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("qwen3_8 does not support --kv-cache-size"); diff --git a/families/qwen3_omni/runtime/plugin.cpp b/families/qwen3_omni/runtime/plugin.cpp index 15fd0bb118..b7da0bd2af 100644 --- a/families/qwen3_omni/runtime/plugin.cpp +++ b/families/qwen3_omni/runtime/plugin.cpp @@ -101,6 +101,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::qwen3_omni +TRTMC_DEFINE_FAMILY_PLUGIN_V1("qwen3_omni") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("qwen3_omni does not support --kv-cache-size"); diff --git a/families/qwen_image/runtime/plugin.cpp b/families/qwen_image/runtime/plugin.cpp index 5b3196b08d..a32e856f0a 100644 --- a/families/qwen_image/runtime/plugin.cpp +++ b/families/qwen_image/runtime/plugin.cpp @@ -33,6 +33,8 @@ std::unique_ptr load(IBackend& backend, const BundleReader& bundle, } // namespace } // namespace trtmc::qwen_image_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("qwen_image") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("qwen_image does not support --kv-cache-size"); diff --git a/families/qwen_moe/runtime/plugin.cpp b/families/qwen_moe/runtime/plugin.cpp index 6488b8263f..c35c7090cf 100644 --- a/families/qwen_moe/runtime/plugin.cpp +++ b/families/qwen_moe/runtime/plugin.cpp @@ -216,6 +216,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::qwen_moe +TRTMC_DEFINE_FAMILY_PLUGIN_V1("qwen_moe") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("qwen_moe does not support --kv-cache-size"); diff --git a/families/qwen_vl/runtime/plugin.cpp b/families/qwen_vl/runtime/plugin.cpp index 295189918a..8b1fe23ea4 100644 --- a/families/qwen_vl/runtime/plugin.cpp +++ b/families/qwen_vl/runtime/plugin.cpp @@ -62,6 +62,8 @@ std::vector lora_contract(const ITrtModule& module) { } // namespace } // namespace trtmc::qwen_vl_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("qwen_vl") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("qwen_vl does not support --kv-cache-size"); diff --git a/families/roberta/runtime/plugin.cpp b/families/roberta/runtime/plugin.cpp index 0eec0266b1..2bde4f0923 100644 --- a/families/roberta/runtime/plugin.cpp +++ b/families/roberta/runtime/plugin.cpp @@ -55,6 +55,8 @@ std::string require_task(const BundleInfo& info) { } // namespace } // namespace trtmc::roberta_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("roberta") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("roberta does not support --kv-cache-size"); diff --git a/families/rwkv/runtime/plugin.cpp b/families/rwkv/runtime/plugin.cpp index b231eacf1a..40451ced97 100644 --- a/families/rwkv/runtime/plugin.cpp +++ b/families/rwkv/runtime/plugin.cpp @@ -114,6 +114,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::rwkv +TRTMC_DEFINE_FAMILY_PLUGIN_V1("rwkv") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("rwkv does not support --kv-cache-size"); diff --git a/families/s1_mini/runtime/plugin.cpp b/families/s1_mini/runtime/plugin.cpp index b03095dc80..5754184479 100644 --- a/families/s1_mini/runtime/plugin.cpp +++ b/families/s1_mini/runtime/plugin.cpp @@ -233,6 +233,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::s1_mini +TRTMC_DEFINE_FAMILY_PLUGIN_V1("s1_mini") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("s1_mini does not support --kv-cache-size"); diff --git a/families/sam/runtime/plugin.cpp b/families/sam/runtime/plugin.cpp index 98adea4216..3903757829 100644 --- a/families/sam/runtime/plugin.cpp +++ b/families/sam/runtime/plugin.cpp @@ -54,6 +54,8 @@ SamConfig parse_config(const std::vector& data, std::int32_t& tp_size) { } // namespace } // namespace trtmc::sam_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("sam") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("sam does not support --kv-cache-size"); diff --git a/families/sam2/runtime/plugin.cpp b/families/sam2/runtime/plugin.cpp index 1aedd5fa2a..ac8215f795 100644 --- a/families/sam2/runtime/plugin.cpp +++ b/families/sam2/runtime/plugin.cpp @@ -40,6 +40,8 @@ NativePlanModuleFactory makeModuleFactory(IBackend& backend) { } // namespace } // namespace trtmc::sam2 +TRTMC_DEFINE_FAMILY_PLUGIN_V1("sam2") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("sam2 does not support --kv-cache-size"); diff --git a/families/sam3/runtime/plugin.cpp b/families/sam3/runtime/plugin.cpp index 4ab33f0428..fe854d4606 100644 --- a/families/sam3/runtime/plugin.cpp +++ b/families/sam3/runtime/plugin.cpp @@ -82,6 +82,8 @@ std::unique_ptr load(IBackend& backend, const BundleReader& bundle, } // namespace } // namespace trtmc::sam3_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("sam3") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("sam3 does not support --kv-cache-size"); diff --git a/families/sana_wm/runtime/plugin.cpp b/families/sana_wm/runtime/plugin.cpp index 3a669a1ec5..79c1ea7261 100644 --- a/families/sana_wm/runtime/plugin.cpp +++ b/families/sana_wm/runtime/plugin.cpp @@ -35,6 +35,8 @@ std::shared_ptr load_tokenizer(const BundleReader& bundle, const cha } // namespace } // namespace trtmc::sana_wm_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("sana_wm") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("sana_wm does not support --kv-cache-size"); diff --git a/families/segformer/runtime/plugin.cpp b/families/segformer/runtime/plugin.cpp index 07dc71fe5b..3f3b9c6b6b 100644 --- a/families/segformer/runtime/plugin.cpp +++ b/families/segformer/runtime/plugin.cpp @@ -46,6 +46,8 @@ SegformerPreprocessConfig parse_config(const std::vector& data, std::int32 } // namespace } // namespace trtmc::segformer +TRTMC_DEFINE_FAMILY_PLUGIN_V1("segformer") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("segformer does not support --kv-cache-size"); diff --git a/families/smollm3/runtime/plugin.cpp b/families/smollm3/runtime/plugin.cpp index 797abd5e09..987ae3475b 100644 --- a/families/smollm3/runtime/plugin.cpp +++ b/families/smollm3/runtime/plugin.cpp @@ -238,6 +238,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::smollm3 +TRTMC_DEFINE_FAMILY_PLUGIN_V1("smollm3") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("smollm3 does not support --kv-cache-size"); diff --git a/families/stablelm/runtime/plugin.cpp b/families/stablelm/runtime/plugin.cpp index d098b83688..2199f858d7 100644 --- a/families/stablelm/runtime/plugin.cpp +++ b/families/stablelm/runtime/plugin.cpp @@ -208,6 +208,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::stablelm +TRTMC_DEFINE_FAMILY_PLUGIN_V1("stablelm") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("stablelm does not support --kv-cache-size"); diff --git a/families/starcoder2/runtime/plugin.cpp b/families/starcoder2/runtime/plugin.cpp index 7762bd3144..80542183d4 100644 --- a/families/starcoder2/runtime/plugin.cpp +++ b/families/starcoder2/runtime/plugin.cpp @@ -208,6 +208,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::starcoder2 +TRTMC_DEFINE_FAMILY_PLUGIN_V1("starcoder2") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("starcoder2 does not support --kv-cache-size"); diff --git a/families/t5/runtime/plugin.cpp b/families/t5/runtime/plugin.cpp index 8e3e617b4d..f2127b2e19 100644 --- a/families/t5/runtime/plugin.cpp +++ b/families/t5/runtime/plugin.cpp @@ -430,6 +430,8 @@ ITask* create_t5(const FamilyContext& context) { } // namespace trtmc +TRTMC_DEFINE_FAMILY_PLUGIN_V1("t5") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("t5 does not support --kv-cache-size"); diff --git a/families/timesfm/runtime/plugin.cpp b/families/timesfm/runtime/plugin.cpp index 23764aadae..c7e8acb5bc 100644 --- a/families/timesfm/runtime/plugin.cpp +++ b/families/timesfm/runtime/plugin.cpp @@ -67,6 +67,8 @@ std::unique_ptr load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_tokenizer(const BundleReader& bundle) { } // namespace } // namespace trtmc::wan22_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("wan2_2_ti2v") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("wan2_2_ti2v does not support --kv-cache-size"); diff --git a/families/wan_t2v/runtime/plugin.cpp b/families/wan_t2v/runtime/plugin.cpp index bcdb7636ad..790a7a469c 100644 --- a/families/wan_t2v/runtime/plugin.cpp +++ b/families/wan_t2v/runtime/plugin.cpp @@ -27,6 +27,8 @@ std::vector require_section(const BundleReader& bundle, const char* name) } // namespace } // namespace trtmc::wan_t2v +TRTMC_DEFINE_FAMILY_PLUGIN_V1("wan_t2v") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("wan_t2v does not support --kv-cache-size"); diff --git a/families/whisper/runtime/plugin.cpp b/families/whisper/runtime/plugin.cpp index 38789139e9..6c0c5d183d 100644 --- a/families/whisper/runtime/plugin.cpp +++ b/families/whisper/runtime/plugin.cpp @@ -27,6 +27,8 @@ std::vector require_section(const BundleReader& bundle, const char* name) } // namespace } // namespace trtmc::whisper_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("whisper") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("whisper does not support --kv-cache-size"); diff --git a/families/xglm/runtime/plugin.cpp b/families/xglm/runtime/plugin.cpp index 3a6c72ecd4..1382b0435c 100644 --- a/families/xglm/runtime/plugin.cpp +++ b/families/xglm/runtime/plugin.cpp @@ -208,6 +208,8 @@ ITask* create(const FamilyContext& context) { } // namespace trtmc::xglm +TRTMC_DEFINE_FAMILY_PLUGIN_V1("xglm") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("xglm does not support --kv-cache-size"); diff --git a/families/xlnet/runtime/plugin.cpp b/families/xlnet/runtime/plugin.cpp index 693cc0c50b..94bc51a575 100644 --- a/families/xlnet/runtime/plugin.cpp +++ b/families/xlnet/runtime/plugin.cpp @@ -55,6 +55,8 @@ std::string require_task(const BundleInfo& info) { } // namespace } // namespace trtmc::xlnet_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("xlnet") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("xlnet does not support --kv-cache-size"); diff --git a/families/yolo11/runtime/plugin.cpp b/families/yolo11/runtime/plugin.cpp index 188a22b01f..29b5ca9b71 100644 --- a/families/yolo11/runtime/plugin.cpp +++ b/families/yolo11/runtime/plugin.cpp @@ -53,6 +53,8 @@ std::unique_ptr load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector load_engine(IBackend& backend, const std::vector& data) { } // namespace } // namespace trtmc::z_image_factory +TRTMC_DEFINE_FAMILY_PLUGIN_V1("z_image") + extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { if (context.kv_cache_size_bytes != 0) throw std::invalid_argument("z_image does not support --kv-cache-size"); diff --git a/pyproject.toml b/pyproject.toml index c7435458d9..aa581ad519 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,7 @@ dependencies = [ ] [project.scripts] +trtmc = "tensorrt_model_connect.native_cli:main" trtmc-bench = "trtmc_benchmark.cli:main" [project.optional-dependencies] diff --git a/tools/ci/e2e.py b/tools/ci/e2e.py index 886112a08b..84620aa206 100644 --- a/tools/ci/e2e.py +++ b/tools/ci/e2e.py @@ -8,6 +8,7 @@ import json import re import shlex +import shutil import tempfile import xml.etree.ElementTree as ET from collections import Counter @@ -345,6 +346,7 @@ def _family_testcases(self, family: str) -> tuple[str, ...]: def _isolated_runtime_root(self, runtime_root: Path, family: str): required = ( "libtrtmc_core.so", + "libtrtmc_runtime.so", "libtrtmc_backend_trt.so", f"libtrtmc_model_{family}.so", ) @@ -357,7 +359,7 @@ def _isolated_runtime_root(self, runtime_root: Path, family: str): isolated = root / "tensorrt_model_connect/bin" isolated.mkdir(parents=True) for name in required: - (isolated / name).symlink_to((runtime_root / name).resolve()) + shutil.copy2(runtime_root / name, isolated / name) # Preserve only non-family wheel dependencies expected by RUNPATH. site_packages = runtime_root.parent.parent diff --git a/tools/ci/package.py b/tools/ci/package.py index d1b097d187..d8b0a9f149 100644 --- a/tools/ci/package.py +++ b/tools/ci/package.py @@ -74,11 +74,67 @@ def load_native_libraries(bin_dir: Path, families: tuple[str, ...]) -> None: script = """ import ctypes import os +from pathlib import Path import sys -ctypes.CDLL(sys.argv[1], mode=os.RTLD_NOW | ctypes.RTLD_GLOBAL) -for path in sys.argv[2:]: - ctypes.CDLL(path, mode=os.RTLD_NOW | ctypes.RTLD_LOCAL) +core = ctypes.CDLL(sys.argv[1], mode=os.RTLD_NOW | ctypes.RTLD_GLOBAL) +runtime = ctypes.CDLL(sys.argv[2], mode=os.RTLD_NOW | ctypes.RTLD_LOCAL) + +def build_identity(library, symbol): + function = getattr(library, symbol) + function.restype = ctypes.c_char_p + raw_identity = function() + identity = raw_identity.decode() if raw_identity is not None else "" + if len(identity) != 32 or any(character not in "0123456789abcdef" for character in identity): + raise RuntimeError(f"invalid TRTMC product-build identity from {symbol}: {identity}") + return identity + +core_build_id = build_identity(core, "trtmc_core_build_id") +runtime_build_id = build_identity(runtime, "trtmc_runtime_build_id") +if core_build_id != runtime_build_id: + raise RuntimeError( + f"installed TRTMC core/runtime build mismatch: core={core_build_id} " + f"runtime={runtime_build_id}" + ) + +class PluginDescriptorV1(ctypes.Structure): + _fields_ = [ + ("struct_size", ctypes.c_uint32), + ("descriptor_version", ctypes.c_uint32), + ("kind", ctypes.c_uint32), + ("id", ctypes.c_char_p), + ("build_id", ctypes.c_char_p), + ] + +for raw_path in sys.argv[3:]: + path = Path(raw_path) + library = ctypes.CDLL(path, mode=os.RTLD_NOW | ctypes.RTLD_LOCAL) + descriptor_function = library.trtmc_plugin_descriptor_v1 + descriptor_function.restype = ctypes.POINTER(PluginDescriptorV1) + descriptor = descriptor_function().contents + name = path.name + if name.startswith("libtrtmc_backend_"): + expected_kind, expected_id = 1, name.removeprefix("libtrtmc_backend_").removesuffix(".so") + elif name.startswith("libtrtmc_model_"): + expected_kind, expected_id = 2, name.removeprefix("libtrtmc_model_").removesuffix(".so") + elif name == "libtrtmc_byok_tvm_ffi.so": + expected_kind, expected_id = 3, "tvm_ffi" + else: + raise RuntimeError(f"unknown TRTMC plugin library: {path}") + actual_id = descriptor.id.decode() if descriptor.id is not None else "" + actual_build_id = descriptor.build_id.decode() if descriptor.build_id is not None else "" + if ( + descriptor.struct_size != ctypes.sizeof(PluginDescriptorV1) + or descriptor.descriptor_version != 1 + or descriptor.kind != expected_kind + or actual_id != expected_id + or actual_build_id != runtime_build_id + ): + raise RuntimeError( + f"invalid TRTMC plugin descriptor: {path}: " + f"size={descriptor.struct_size} version={descriptor.descriptor_version} " + f"kind={descriptor.kind} id={actual_id} build={actual_build_id}" + ) """ environment = os.environ.copy() environment["LD_LIBRARY_PATH"] = ":".join( @@ -117,6 +173,8 @@ def validate(self, wheels: list[Path]) -> None: raise CiError(f"{wheel}: generated Python cache files are packaged") if "tensorrt_model_connect/__init__.py" not in names: raise CiError(f"{wheel}: Python core package is missing") + if "tensorrt_model_connect/native_cli.py" not in names: + raise CiError(f"{wheel}: native CLI console adapter is missing") if "trtmc_benchmark/__init__.py" not in names: raise CiError(f"{wheel}: Python benchmark application is missing") source_suffixes = { @@ -262,20 +320,24 @@ def validate(self, wheels: list[Path]) -> None: raise CiError( f"{wheel}: expected only unaliased TensorRT backend DSOs, found {backend_dsos}" ) - scripts = [name for name in names if name.endswith(".data/scripts/trtmc")] - script_cores = [ - name for name in names if name.endswith(".data/scripts/libtrtmc_core.so") - ] - script_runtimes = [ - name for name in names if name.endswith(".data/scripts/libtrtmc_runtime.so") + duplicate_native_payload = [ + name + for name in names + if ".data/scripts/" in name + and Path(name).name in {"trtmc", "libtrtmc_core.so", "libtrtmc_runtime.so"} ] - if len(scripts) != 1 or len(script_cores) != 1 or len(script_runtimes) != 1: - raise CiError(f"{wheel}: installed CLI payload is incomplete") + if duplicate_native_payload: + raise CiError(f"{wheel}: native product payload is duplicated in wheel scripts") entry_points = [name for name in names if name.endswith(".dist-info/entry_points.txt")] - if len(entry_points) != 1 or "trtmc-bench" not in archive.read(entry_points[0]).decode( - "utf-8" - ): - raise CiError(f"{wheel}: trtmc-bench console entrypoint is missing") + if len(entry_points) != 1: + raise CiError(f"{wheel}: console entrypoints are missing") + entrypoint_text = archive.read(entry_points[0]).decode("utf-8") + required_entrypoints = ( + "trtmc = tensorrt_model_connect.native_cli:main", + "trtmc-bench = trtmc_benchmark.cli:main", + ) + if not all(entrypoint in entrypoint_text for entrypoint in required_entrypoints): + raise CiError(f"{wheel}: required console entrypoints are missing") print(f"validated wheel={wheel} families={len(expected_families)}") @@ -369,6 +431,7 @@ def validate(self, wheel: Path) -> None: ) if not version.stdout.startswith("trtmc "): raise CiError("installed trtmc CLI returned an invalid version") + fixture_family = min(expected) with tempfile.TemporaryDirectory(prefix="trtmc-installed-wheel-") as directory: bundle = Path(directory) / "inspect.bundle" subprocess.run( @@ -378,9 +441,11 @@ def validate(self, wheel: Path) -> None: "from pathlib import Path; " "from tensorrt_model_connect.bundle_writer import BundleWriter; " "writer = BundleWriter(Path(__import__('sys').argv[1])); " - "writer.set_header(family='inspect', task='text_generation', backend='trt'); " + "writer.set_header(family=__import__('sys').argv[2], " + "task='package_validation', backend='trt'); " "writer.finish()", bundle, + fixture_family, ], check=True, cwd=Path("/tmp"), @@ -395,8 +460,18 @@ def validate(self, wheel: Path) -> None: env=environment, ) metadata = json.loads(inspected.stdout) - if metadata.get("family") != "inspect" or metadata.get("backend") != "trt": + if metadata.get("family") != fixture_family or metadata.get("backend") != "trt": raise CiError("installed trtmc CLI failed bundle inspection") + executed = subprocess.run( + [executable, "run", bundle], + capture_output=True, + text=True, + cwd=Path("/tmp"), + env=environment, + ) + selected = f"Using TRTMC runtime: {bin_dir}\n" + if executed.returncode == 0 or selected not in executed.stderr: + raise CiError("installed trtmc CLI failed automatic wheel runtime discovery") print(f"installed wheel={wheel} trtmc={executable} families={len(packaged)}") diff --git a/tools/tests/test_architecture.py b/tools/tests/test_architecture.py index f3969e24db..c425ce54a3 100644 --- a/tools/tests/test_architecture.py +++ b/tools/tests/test_architecture.py @@ -268,6 +268,7 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "core/builder/tensorrt_model_connect/bundle_writer.py", "core/builder/tensorrt_model_connect/graph_transform.py", "core/builder/tensorrt_model_connect/model_support.py", + "core/builder/tensorrt_model_connect/native_cli.py", "core/builder/tests/__init__.py", "core/builder/tests/test_build.py", "core/builder/tests/test_build_cli.py", @@ -275,6 +276,7 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "core/builder/tests/test_byok.py", "core/builder/tests/test_graph_transform.py", "core/builder/tests/test_model_support.py", + "core/builder/tests/test_native_cli.py", } expected_native = { "core/runtime/bundle/bundle_format.cpp", @@ -293,6 +295,7 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "core/runtime/byok/tvm_ffi_kernel_plugin.h", "core/runtime/primitives/cuda_common.cpp", "core/runtime/primitives/cuda_common.h", + "core/runtime/primitives/build_identity.cpp", "core/runtime/primitives/device_tensor.cpp", "core/runtime/primitives/trt_common.cpp", "core/runtime/primitives/trt_common.h", @@ -303,14 +306,18 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "core/runtime/include/trtmc/runtime/device_tensor.h", "core/runtime/include/trtmc/runtime/family_factory.h", "core/runtime/include/trtmc/runtime/family_loader.h", + "core/runtime/include/trtmc/runtime/plugin_abi.h", + "core/runtime/include/trtmc/runtime/runtime_root.h", "core/runtime/include/trtmc/runtime/tensor.h", "core/runtime/include/trtmc/runtime/trt_backend.h", "core/runtime/include/trtmc/runtime/trt_module.h", "core/runtime/tests/fake_backend.cpp", + "core/runtime/tests/fake_core_build_identity.cpp", "core/runtime/tests/fake_family.cpp", "core/runtime/tests/test_bundle_format_v1.cpp", "core/runtime/tests/test_byok_shape_spec.cpp", "core/runtime/tests/test_family_loader.cpp", + "core/runtime/tests/test_runtime_root.cpp", "core/runtime/tests/test_task_api.cpp", "core/runtime/tests/test_trt_module_dynamic_input.cpp", } @@ -355,7 +362,7 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "tools/tests/test_pr_metadata.py", "tools/tests/test_public_source_hygiene.py", } - expected_cmake = {"cmake/trtmcConfig.cmake.in"} + expected_cmake = {"cmake/product_build.h.in", "cmake/trtmcConfig.cmake.in"} expected_third_party = { "third_party/stb/stb_image.h", "third_party/stb/stb_image_resize2.h", @@ -798,6 +805,7 @@ def dependency_lines(path: Path) -> list[str]: package_validation = (REPO / "tools/ci/package.py").read_text(encoding="utf-8") assert 'import_module(f"families.{family}.model")' not in package_validation + assert re.search(r"set_header\(family=['\"]", package_validation) is None def test_family_reference_consumers_declare_their_source() -> None: @@ -1121,7 +1129,7 @@ def test_rtx_backend_is_an_explicit_optional_dso() -> None: assert "runtime cache and whole-graph capture require a TensorRT-RTX bundle" in loader -def test_every_runtime_exports_only_the_task_factory_contract() -> None: +def test_every_runtime_exports_task_factory_and_abi_descriptor() -> None: forbidden = ( "IPipeline", "PipelineContext", @@ -1146,6 +1154,8 @@ def test_every_runtime_exports_only_the_task_factory_contract() -> None: source = factory.read_text(encoding="utf-8", errors="ignore") if "trtmc_create_family" not in source or "trtmc::ITask*" not in source: violations.append(f"{family.name}:factory") + if f'TRTMC_DEFINE_FAMILY_PLUGIN_V1("{family.name}")' not in source: + violations.append(f"{family.name}:plugin-descriptor") cmake = (runtime / "CMakeLists.txt").read_text(encoding="utf-8") if f"trtmc_model_{family.name}" not in cmake: violations.append(f"{family.name}:target") @@ -1164,6 +1174,40 @@ def test_every_runtime_exports_only_the_task_factory_contract() -> None: assert violations == [] +def test_runtime_plugins_publish_one_exact_build_descriptor() -> None: + descriptor = (REPO / "core/runtime/include/trtmc/runtime/plugin_abi.h").read_text( + encoding="utf-8" + ) + loader = (REPO / "core/runtime/loader/family_loader.cpp").read_text(encoding="utf-8") + cmake = (REPO / "CMakeLists.txt").read_text(encoding="utf-8") + + assert "struct PluginDescriptorV1" in descriptor + assert 'kPluginDescriptorSymbol = "trtmc_plugin_descriptor_v1"' in descriptor + assert "kPluginBuildId = TRTMC_BUILD_ID" in descriptor + assert "trtmc_core_build_id" in descriptor + assert "trtmc_runtime_build_id" in descriptor + assert "require_plugin(PluginKind expected_kind" in loader + assert "require_matching_core_build();" in loader + assert "active runtime requires" in loader + assert "" not in loader + assert "build_cohort" not in loader + assert "TRTMC_BUILD_COHORT_ID" not in cmake + assert "TRTMC_BUILD_ID" in cmake + assert "cmake/product_build.h.in" in cmake + assert "add_compile_definitions(TRTMC_BUILD_ID" not in cmake + + backends = { + "core/runtime/tensorrt/trt_backend.cpp": "trt", + "core/runtime/tensorrt/rtx_backend.cpp": "trt_rtx", + } + for path, backend in backends.items(): + source = (REPO / path).read_text(encoding="utf-8") + assert f'TRTMC_DEFINE_BACKEND_PLUGIN_V1("{backend}")' in source + + byok = (REPO / "core/runtime/byok/byok.cpp").read_text(encoding="utf-8") + assert "PluginKind::kRuntimeExtension" in byok + + def test_family_factory_receives_only_direct_runtime_inputs() -> None: factory_header = (REPO / "core/runtime/include/trtmc/runtime/family_factory.h").read_text( encoding="utf-8" diff --git a/tools/tests/test_new_ci.py b/tools/tests/test_new_ci.py index cab2bc05d3..d338770de8 100644 --- a/tools/tests/test_new_ci.py +++ b/tools/tests/test_new_ci.py @@ -146,6 +146,7 @@ def test_selective_e2e_calls_family_tests_directly( runtime = tmp_path / "runtime" runtime.mkdir() (runtime / "libtrtmc_core.so").write_text("") + (runtime / "libtrtmc_runtime.so").write_text("") (runtime / "libtrtmc_backend_trt.so").write_text("") (runtime / "libtrtmc_model_beta.so").write_text("") native_build = tmp_path / "native-build" @@ -218,10 +219,39 @@ def test_selective_e2e_calls_family_tests_directly( assert options["updates"]["TRTMC_RUNTIME_ROOT"] != str(runtime) assert options["unset"] == ("PYTEST_ADDOPTS",) assert context.runtime_snapshots == [ - ("libtrtmc_backend_trt.so", "libtrtmc_core.so", "libtrtmc_model_beta.so") + ( + "libtrtmc_backend_trt.so", + "libtrtmc_core.so", + "libtrtmc_model_beta.so", + "libtrtmc_runtime.so", + ) ] +def test_isolated_runtime_root_materializes_root_local_trtmc_libraries( + tmp_path: Path, +) -> None: + runtime = tmp_path / "runtime" + runtime.mkdir() + required = ( + "libtrtmc_core.so", + "libtrtmc_runtime.so", + "libtrtmc_backend_trt.so", + "libtrtmc_model_beta.so", + ) + for name in required: + (runtime / name).write_text(name, encoding="utf-8") + + runner = E2ERunner(RecordingContext(tmp_path, {})) + with runner._isolated_runtime_root(runtime, "beta") as isolated: + for name in required: + staged = isolated / name + assert staged.is_file() + assert not staged.is_symlink() + assert staged.resolve().parent == isolated.resolve() + assert staged.read_text(encoding="utf-8") == name + + def test_family_with_only_hardware_tests_accepts_exact_empty_cpu_result( tmp_path: Path, ) -> None: @@ -244,6 +274,7 @@ def test_family_with_only_hardware_tests_accepts_exact_empty_cpu_result( runtime.mkdir() for name in ( "libtrtmc_core.so", + "libtrtmc_runtime.so", "libtrtmc_backend_trt.so", "libtrtmc_model_beta.so", ): @@ -391,22 +422,26 @@ def test_e2e_rejects_multiple_family_environments(tmp_path: Path) -> None: def test_e2e_nonexistent_testcase_fails_closed(tmp_path: Path) -> None: - repository = Path(__file__).resolve().parents[2] + family = tmp_path / "families/alpha" + (family / "tests").mkdir(parents=True) + (family / "model.py").write_text("def build(request, writer): pass\n") + (family / "tests/test_e2e.py").write_text("def test_e2e(): pass\n") binary = tmp_path / "trtmc" binary.write_text("") runtime = tmp_path / "runtime" runtime.mkdir() for name in ( "libtrtmc_core.so", + "libtrtmc_runtime.so", "libtrtmc_backend_trt.so", - "libtrtmc_model_gpt2.so", + "libtrtmc_model_alpha.so", ): (runtime / name).write_text("") native_build = tmp_path / "native-build" native_build.mkdir() (native_build / "CTestTestfile.cmake").write_text("") context = RecordingContext( - repository, + tmp_path, { "TRTMC_BINARY": str(binary), "TRTMC_RUNTIME_ROOT": str(runtime), @@ -416,7 +451,7 @@ def test_e2e_nonexistent_testcase_fails_closed(tmp_path: Path) -> None: context.missing_e2e_testcases.add("does-not-exist") with pytest.raises(CiError, match="missing requested E2E testcase: does-not-exist"): - E2ERunner(context)._run(("gpt2",), ("does-not-exist",)) + E2ERunner(context)._run(("alpha",), ("does-not-exist",)) def test_pipeline_exposes_only_active_stages(tmp_path: Path) -> None: @@ -803,6 +838,7 @@ def test_wheel_validation_requires_exact_new_payload(tmp_path: Path) -> None: wheel = tmp_path / "package.whl" with zipfile.ZipFile(wheel, "w") as archive: archive.writestr("tensorrt_model_connect/__init__.py", "") + archive.writestr("tensorrt_model_connect/native_cli.py", "") archive.writestr("trtmc_benchmark/__init__.py", "") archive.writestr("families/__init__.py", "") archive.writestr("tensorrt_model_connect/bin/trtmc", "") @@ -814,7 +850,9 @@ def test_wheel_validation_requires_exact_new_payload(tmp_path: Path) -> None: archive.writestr("tensorrt_model_connect/bin/libtrtmc_byok_tvm_ffi.so", "") archive.writestr( "package-0.1.dist-info/entry_points.txt", - "[console_scripts]\ntrtmc-bench = trtmc_benchmark.cli:main\n", + "[console_scripts]\n" + "trtmc = tensorrt_model_connect.native_cli:main\n" + "trtmc-bench = trtmc_benchmark.cli:main\n", ) archive.writestr( "package-0.1.dist-info/METADATA", @@ -824,9 +862,6 @@ def test_wheel_validation_requires_exact_new_payload(tmp_path: Path) -> None: "Provides-Extra: cutedsl\n" "Provides-Extra: test\n", ) - archive.writestr("package-0.1.data/scripts/trtmc", "") - archive.writestr("package-0.1.data/scripts/libtrtmc_core.so", "") - archive.writestr("package-0.1.data/scripts/libtrtmc_runtime.so", "") for family in family_names: archive.writestr( f"families/{family}/model.py", @@ -836,6 +871,22 @@ def test_wheel_validation_requires_exact_new_payload(tmp_path: Path) -> None: WheelArchiveValidator(CiContext(tmp_path, {})).validate([wheel]) + without_launcher = tmp_path / "without-launcher.whl" + with zipfile.ZipFile(wheel) as source, zipfile.ZipFile(without_launcher, "w") as output: + for entry in source.infolist(): + if entry.filename != "tensorrt_model_connect/native_cli.py": + output.writestr(entry, source.read(entry.filename)) + with pytest.raises(CiError, match="console adapter is missing"): + WheelArchiveValidator(CiContext(tmp_path, {})).validate([without_launcher]) + + duplicated = tmp_path / "duplicated.whl" + with zipfile.ZipFile(wheel) as source, zipfile.ZipFile(duplicated, "w") as output: + for entry in source.infolist(): + output.writestr(entry, source.read(entry.filename)) + output.writestr("package-0.1.data/scripts/trtmc", "") + with pytest.raises(CiError, match="duplicated in wheel scripts"): + WheelArchiveValidator(CiContext(tmp_path, {})).validate([duplicated]) + corrupt = tmp_path / "corrupt.whl" with zipfile.ZipFile(wheel) as source, zipfile.ZipFile(corrupt, "w") as output: for entry in source.infolist(): @@ -885,17 +936,78 @@ def compile_library(name: str, source: str) -> None: check=True, ) - compile_library("libtrtmc_core.so", "void core_symbol(void) {}\n") - compile_library("libtrtmc_runtime.so", "void runtime_symbol(void) {}\n") - compile_library("libtrtmc_backend_trt.so", "void backend_symbol(void) {}\n") - compile_library("libtrtmc_byok_tvm_ffi.so", "void byok_symbol(void) {}\n") - compile_library("libtrtmc_model_alpha.so", "void alpha_symbol(void) {}\n") - compile_library("libtrtmc_model_beta.so", "void beta_symbol(void) {}\n") + def plugin_source( + kind: int, + plugin_id: str, + implementation: str, + build_id: str = "1234567890abcdef1234567890abcdef", + struct_size: str = "sizeof(struct PluginDescriptorV1)", + descriptor_version: int = 1, + ) -> str: + return f""" +#include +struct PluginDescriptorV1 {{ + uint32_t struct_size; + uint32_t descriptor_version; + uint32_t kind; + const char *id; + const char *build_id; +}}; +static const struct PluginDescriptorV1 descriptor = {{ + {struct_size}, {descriptor_version}, {kind}, "{plugin_id}", "{build_id}" +}}; +const struct PluginDescriptorV1 *trtmc_plugin_descriptor_v1(void) {{ return &descriptor; }} +{implementation} +""" + + build_id = "1234567890abcdef1234567890abcdef" + compile_library( + "libtrtmc_core.so", + f'const char *trtmc_core_build_id(void) {{ return "{build_id}"; }}\n', + ) + compile_library( + "libtrtmc_runtime.so", + f'const char *trtmc_runtime_build_id(void) {{ return "{build_id}"; }}\n', + ) + compile_library( + "libtrtmc_backend_trt.so", plugin_source(1, "trt", "void backend_symbol(void) {}") + ) + compile_library( + "libtrtmc_byok_tvm_ffi.so", + plugin_source(3, "tvm_ffi", "void byok_symbol(void) {}"), + ) + compile_library( + "libtrtmc_model_alpha.so", plugin_source(2, "alpha", "void alpha_symbol(void) {}") + ) + compile_library("libtrtmc_model_beta.so", plugin_source(2, "beta", "void beta_symbol(void) {}")) load_native_libraries(tmp_path, ("alpha", "beta")) + invalid_descriptors = ( + ("size=0", plugin_source(2, "beta", "", struct_size="0")), + ("version=2", plugin_source(2, "beta", "", descriptor_version=2)), + ("kind=1", plugin_source(1, "beta", "")), + ("id=gamma", plugin_source(2, "gamma", "")), + ( + "build=00000000000000000000000000000000", + plugin_source(2, "beta", "", build_id="00000000000000000000000000000000"), + ), + ) + for expected_error, source in invalid_descriptors: + compile_library("libtrtmc_model_beta.so", source) + with pytest.raises(CiError, match=expected_error): + load_native_libraries(tmp_path, ("alpha", "beta")) + + compile_library("libtrtmc_model_beta.so", "void beta_symbol(void) {}") + with pytest.raises(CiError, match="trtmc_plugin_descriptor_v1"): + load_native_libraries(tmp_path, ("alpha", "beta")) + compile_library( "libtrtmc_model_beta.so", - "extern void missing_symbol(void); void beta_symbol(void) { missing_symbol(); }\n", + plugin_source( + 2, + "beta", + "extern void missing_symbol(void); void beta_symbol(void) { missing_symbol(); }", + ), ) with pytest.raises(CiError, match="undefined symbol: missing_symbol"): load_native_libraries(tmp_path, ("alpha", "beta")) diff --git a/website/docs/api/cli-reference.md b/website/docs/api/cli-reference.md index 062f946e57..0a41768042 100644 --- a/website/docs/api/cli-reference.md +++ b/website/docs/api/cli-reference.md @@ -56,17 +56,21 @@ backend, and section bounds. Family-owned section payloads are not decoded. Every execution command has this shape: ```bash -trtmc COMMAND MODEL.bundle --runtime-root DIR [OPTIONS] +trtmc COMMAND MODEL.bundle [--runtime-root DIR] [OPTIONS] ``` -`--runtime-root` is always required. It must contain the matching -`libtrtmc_core.so`, `libtrtmc_runtime.so`, backend DSO, and selected family DSO. -The CLI never searches the current directory, environment variables, or an -installed fallback. Common load options are: +When `--runtime-root` is omitted, the CLI checks the active Runtime directory, +followed by colon-separated `TRTMC_RUNTIME_PATH` entries. A wheel console +command replaces itself with the native CLI in the wheel's single native root. +The CLI does not scan `PATH`, Python installation layouts, or the current +directory unless `.` is explicitly present in `TRTMC_RUNTIME_PATH`. One +selected root must contain the requested backend and family DSOs; their +descriptors must declare the active product-build identity, expected kind, and +bundle ID before either factory is called. Common load options are: | Option | Contract | | --- | --- | -| `--runtime-root DIR` | Required exact DSO root. | +| `--runtime-root DIR` | Select one exact plugin root without search fallback; exact-build validation still applies. | | `--kv-cache-size BYTES\|GB\|GiB` | Runtime-sized KV capacity for a compatible bundle. | | `--runtime-cache PATH` | TensorRT-RTX cache path; rejected by the standard TensorRT backend. | | `--cuda-graphs` | Enable TensorRT-RTX CUDA graphs; rejected by the standard backend. | @@ -101,7 +105,6 @@ text-diffusion replay inputs, and a paired `--lora-adapter` / ```bash trtmc run qwen.bundle \ - --runtime-root /opt/trtmc/lib \ --prompt "Hello" \ --max-new-tokens 32 \ --temperature 0 \ diff --git a/website/docs/api/overview.md b/website/docs/api/overview.md index 6a6aa7d032..4121d1a73d 100644 --- a/website/docs/api/overview.md +++ b/website/docs/api/overview.md @@ -20,16 +20,16 @@ The build and runtime entry points are intentionally separate. The Python builder resolves exactly one `families//support.py`, imports only that family's `model.py`, and writes a bundle. The native loader reads the bundle's `family`, `task`, and `backend`, then loads exactly one family DSO and one -backend DSO from the explicit runtime root. +backend DSO from one selected plugin root. ```text Hugging Face model ID or local snapshot -> python -m tensorrt_model_connect build -> model.bundle - -> trtmc::load_task() or trtmc TASK --runtime-root DIR + -> trtmc::load_task() with an explicit root, or trtmc TASK with CLI discovery -> task-specific output ``` There is no Python runtime wrapper, central model registry, runtime-strategy -switch, backend search path, or fallback runtime discovery in the current -architecture. +switch, sibling-family probe, or load-time fallback. CLI discovery selects one +root before the Runtime Loader performs an exact load. diff --git a/website/docs/architecture/ai-native-horizontal-scaling.md b/website/docs/architecture/ai-native-horizontal-scaling.md index f975ff1c79..af52a80beb 100644 --- a/website/docs/architecture/ai-native-horizontal-scaling.md +++ b/website/docs/architecture/ai-native-horizontal-scaling.md @@ -120,7 +120,7 @@ After each transfer, core no longer participates in model behavior. | Component | Owns | Explicitly does not own | | --- | --- | --- | | Native Core (`libtrtmc_core.so`) | bounded bundle reads, device tensors, stable engine primitives | `dlopen`, model config, weight mapping, preprocessing, request loops | -| Runtime Loader (`libtrtmc_runtime.so`) | safe family/backend names, explicit runtime root, exact `dlopen`, one control transfer | model pipelines, preprocessing, policy dispatch, family fallback | +| Runtime Loader (`libtrtmc_runtime.so`) | safe family/backend names, explicit runtime root, plugin-root validation, exact product-build and plugin identity, exact `dlopen`, one control transfer | path search policy, model pipelines, preprocessing, policy dispatch, family fallback | | Family | checkpoint identity, tasks/default, graph build, weights, section semantics, native pipeline, dispatch, bindings, pre/postprocessing | sibling families, shared model policy | | Bundle | header and named byte sections with bounded streaming I/O | model schema, section semantics, content hashes | | Task API | user behavior such as text, image, audio, embedding, and segmentation | family names, TensorRT objects, backend details | @@ -506,8 +506,8 @@ Runtime dispatch occurs once: 1. The user calls `load(bundle)`. 2. Runtime reads the fixed header and bounded section table. -3. Runtime derives and loads `libtrtmc_model_.so` from the explicit - runtime root. +3. Runtime loads the named backend and family DSOs from the explicit runtime + root and validates their product-build identities, kinds, and IDs. 4. Runtime finds the fixed family-factory symbol and passes BundleReader, Engine API, and the direct runtime KV budget. Backend-only RTX cache and whole-graph options are applied through the Engine API wrapper. @@ -515,8 +515,34 @@ Runtime dispatch occurs once: 6. Runtime returns the abstract Task interface. Later requests call the family pipeline directly. -Core, family, and backend DSOs are produced by one product build. There is no -ABI negotiation, version translation, old-symbol alias, or compatibility shim. +Core, runtime, family, backend, and extension DSOs are produced by one product +build. Core and runtime expose the same exact build identity; each plugin +descriptor carries it alongside the plugin kind and ID. The runtime verifies +core before crossing their C++ seam, then verifies every selected plugin. There +is no ABI negotiation, compatible-version selection, translation, old-symbol +alias, or fallback. Source work remains family-local, but native artifacts from +separate product builds are never mixed. + +The human-facing `trtmc` CLI may discover one plugin root before this control +transfer. A product installation owns one plugin root containing Core, Runtime, +backend, extension, and family artifacts. A native CMake install may keep the +CLI in its conventional sibling `bin` directory; the already loaded Runtime +still identifies the `lib` plugin root. A wheel console adapter instead +replaces itself with the native CLI inside its single native product directory. +The CLI therefore owns only candidate enumeration and search order: the active +Runtime directory, then explicitly configured `TRTMC_RUNTIME_PATH` entries. +The current directory participates only when a user explicitly adds `.` to +that variable. For each candidate, the CLI asks the model-agnostic Runtime +Loader contract whether the requested root-local backend, family, and optional +BYOK files exist. It does not derive DSO names, inspect native metadata, scan +installation layouts, or load discovery candidates. The public C++ load +interface still receives one explicit root, and a selected-root load failure +never triggers another search. + +Before calling bundle or loader C++ interfaces, the CLI compares its embedded +product-build identity with the C identity symbols exported by the active +Runtime and Core. This keeps executable validation at the application seam; +the Runtime Loader continues to own Core and plugin validation. ### Task API diff --git a/website/docs/architecture/build-system.md b/website/docs/architecture/build-system.md index 0e88a85da3..22904a2a23 100644 --- a/website/docs/architecture/build-system.md +++ b/website/docs/architecture/build-system.md @@ -23,11 +23,19 @@ private dependencies, warnings, tests, output name, and install rule. Adding a normal family never changes a central model source list. The wheel packages `core/builder/tensorrt_model_connect`, the top-level -`families` package, benchmark Python code, and installed native binaries/DSOs. +`families` package, benchmark Python code, and one native product directory at +`tensorrt_model_connect/bin`. The `trtmc` console entry point is a Python +adapter that uses `exec` to replace itself with the native CLI in that +directory; it does not duplicate native files into the wheel scripts area. Optional family dependencies remain in each `families//requirements.txt`; package validation does not import every family implementation. +CMake generates one private product-build header for all native targets in a +build tree. It is not installed. A release build may set the 32-character +`TRTMC_BUILD_ID` explicitly for coordinated reproducibility; never reuse that +identity across independently compiled native artifact sets. + ## Typical source build ```bash diff --git a/website/docs/architecture/runtime-lifecycle.md b/website/docs/architecture/runtime-lifecycle.md index b82b68eb40..af41d99f96 100644 --- a/website/docs/architecture/runtime-lifecycle.md +++ b/website/docs/architecture/runtime-lifecycle.md @@ -14,12 +14,16 @@ auto task = trtmc::load_task("model.bundle", "/opt/trtmc/lib"); 2. The loader validates the `family` and `backend` names as safe DSO tokens. 3. It loads `libtrtmc_backend_.so` from the explicit runtime root. 4. It loads `libtrtmc_model_.so` from that same root. -5. It resolves the single `trtmc_create_family` factory and passes a +5. It requires both plugin descriptors to declare the active product build, + expected kind, and identity. +6. It resolves the single `trtmc_create_family` factory and passes a `FamilyContext` containing the read-only bundle reader and abstract backend. -6. It verifies that the returned `ITask::task()` matches the bundle header. +7. It verifies that the backend name and returned `ITask::task()` match the + bundle header. -There is no current-directory search, environment fallback, registry lookup, -strategy switch, sibling-family probe, or load retry. +The Runtime Loader performs no current-directory search, environment fallback, +registry lookup, strategy switch, sibling-family probe, or load retry. The CLI +may select one explicit root before entering this sequence. ## Ownership after transfer diff --git a/website/docs/architecture/runtime-plugins.md b/website/docs/architecture/runtime-plugins.md index 874f8fb874..327cbab1e6 100644 --- a/website/docs/architecture/runtime-plugins.md +++ b/website/docs/architecture/runtime-plugins.md @@ -5,13 +5,43 @@ title: Family Runtime DSOs The pre-#1093 runtime-plugin registry no longer exists. The stable load unit is one family DSO named `libtrtmc_model_.so`. -The bundle header names exactly one family and backend. The loader opens that -family DSO from the explicit runtime root, resolves `trtmc_create_family`, and -receives an implementation of an abstract Task interface. A family owns its -factory, pipeline, preprocessing, postprocessing, dispatch, bindings, state, -samplers, and any genuinely model-specific TensorRT plugin. - -There is no registrar macro, runtime-strategy map, central manifest, sibling -fallback, or hot plugin marketplace. To extend an existing family, edit only -its `families//runtime/` implementation and family-owned tests. To add -a new family, follow [Add a Model Family](../extend/add-model-family.md). +The bundle header names exactly one family and backend. Each DSO publishes the +model-agnostic `trtmc_plugin_descriptor_v1` interface with its product-build +identity, kind, and ID. The loader opens the exact backend and family paths +from one selected plugin root, validates both descriptors, resolves +`trtmc_create_family`, and receives an implementation of an abstract Task +interface. + +A family owns its descriptor declaration, factory, pipeline, preprocessing, +postprocessing, dispatch, bindings, state, samplers, and any genuinely +model-specific TensorRT plugin. Source ownership remains family-local, while +the resulting DSO is released as part of one coordinated product build. + +Descriptor v1 is an exact metadata layout, not ABI negotiation. The factory, +context, backend, module, and Task seams contain C++ interfaces, so plugins from +another product build are rejected even when they expose the same descriptor +version. Core and runtime also expose and compare their product-build identities +before the runtime calls a core C++ interface. + +The native CLI also embeds that identity and compares it with the Runtime and +Core C symbols before calling bundle or loader C++ interfaces. This check is +application assembly, not family selection, so it remains outside the generic +Runtime Loader. + +Plugin roots are trusted native-code inputs. ELF constructors run during +`dlopen`, before the descriptor can be called. The loader therefore never opens +discovery candidates speculatively, never falls back after a selected load +fails, and keeps opened DSOs resident so TensorRT registrars cannot outlive +their defining code. + +There is no runtime-strategy map, central manifest, sibling fallback, or hot +plugin marketplace. To extend an existing family, edit only its +`families//runtime/` implementation and family-owned tests. To add a +new family, follow [Add a Model Family](../extend/add-model-family.md). + +Every family factory declares +`TRTMC_DEFINE_FAMILY_PLUGIN_V1("")`; every backend declares +`TRTMC_DEFINE_BACKEND_PLUGIN_V1("")`. Runtime extensions use the +generic descriptor macro with `PluginKind::kRuntimeExtension`. These macros +only publish the fixed descriptor symbol and build identity. They are required +because calling a factory first would cross an unvalidated C++ interface. diff --git a/website/docs/extend/add-model-family.md b/website/docs/extend/add-model-family.md index 16e8c92cd9..743cbb52f1 100644 --- a/website/docs/extend/add-model-family.md +++ b/website/docs/extend/add-model-family.md @@ -76,6 +76,17 @@ concrete implementation of an abstract interface in `trtmc/task.h`. The family pipeline depends on and implements `trtmc/task.h`; `trtmc/task.h` never includes or links a family. +The same `plugin.cpp` must publish the model-agnostic descriptor next to its +factory: + +```cpp +TRTMC_DEFINE_FAMILY_PLUGIN_V1("my_family") +``` + +This declaration supplies the family ID, plugin kind, and coordinated product +build identity used before the factory crosses a C++ interface. It does not +register the family centrally or move any family behavior into shared code. + If the family graph contains distributed collectives, that same family owns its communicator setup and NCCL loading. A replicated plan that only selects a rank-specific section must not load NCCL. diff --git a/website/docs/extend/add-optimized-runtime.md b/website/docs/extend/add-optimized-runtime.md index 92fe8a32af..680aac4c31 100644 --- a/website/docs/extend/add-optimized-runtime.md +++ b/website/docs/extend/add-optimized-runtime.md @@ -38,7 +38,6 @@ python -m tensorrt_model_connect build MODEL \ -o model-rtx.bundle trtmc run model-rtx.bundle \ - --runtime-root /opt/trtmc/lib \ --runtime-cache /tmp/trtmc-rtx.cache \ --cuda-graphs \ --prompt "Hello" diff --git a/website/docs/features/config-and-backends.md b/website/docs/features/config-and-backends.md index f36bc18449..9c4041e01e 100644 --- a/website/docs/features/config-and-backends.md +++ b/website/docs/features/config-and-backends.md @@ -35,7 +35,6 @@ Both are loaded from the same required runtime root as the family DSO: ```bash trtmc run model.bundle \ - --runtime-root /opt/trtmc/lib \ --prompt "Hello" ``` diff --git a/website/docs/features/multi-device.md b/website/docs/features/multi-device.md index 35a3c996f7..0db5d30145 100644 --- a/website/docs/features/multi-device.md +++ b/website/docs/features/multi-device.md @@ -45,7 +45,6 @@ mpirun --tag-output -np 4 \ -x CUDA_VISIBLE_DEVICES \ -x TRTMC_NCCL_RENDEZVOUS \ trtmc run model-tp4.bundle \ - --runtime-root /opt/trtmc/lib \ --prompt "Hello" ``` diff --git a/website/docs/features/sampling.md b/website/docs/features/sampling.md index eb5c3b2afb..d1632840a2 100644 --- a/website/docs/features/sampling.md +++ b/website/docs/features/sampling.md @@ -47,7 +47,7 @@ to disabled top-p behavior. ### CLI ```bash -$TRTMC run bundle.bundle --runtime-root /opt/trtmc/lib \ +$TRTMC run bundle.bundle \ --prompt "Once upon a time" \ --temperature 0.7 --top-p 0.9 --min-p 0.05 --top-k 50 \ --repetition-penalty 1.05 --seed 42 diff --git a/website/docs/getting-started/build-and-run.md b/website/docs/getting-started/build-and-run.md index be3bfbdd0e..ed7d5b6f2f 100644 --- a/website/docs/getting-started/build-and-run.md +++ b/website/docs/getting-started/build-and-run.md @@ -7,12 +7,6 @@ checkpoint, task, precision, topology, dependency, and validation support is generated on [Models & Recipes](../models-recipes/overview.md) from the current family-owned manifests. -Set the runtime root once for the examples: - -```bash -export TRTMC_RUNTIME_ROOT=/opt/trtmc/lib -``` - ## Text generation ```bash @@ -22,7 +16,6 @@ python -m tensorrt_model_connect build Qwen/Qwen3-0.6B \ --output /tmp/qwen.bundle trtmc run /tmp/qwen.bundle \ - --runtime-root "$TRTMC_RUNTIME_ROOT" \ --prompt "What is the capital of France? Answer in one word." \ --max-new-tokens 10 \ --temperature 0 \ @@ -42,7 +35,6 @@ python -m tensorrt_model_connect build Qwen/Qwen2.5-VL-3B-Instruct \ --output /tmp/qwen25vl.bundle trtmc run /tmp/qwen25vl.bundle \ - --runtime-root "$TRTMC_RUNTIME_ROOT" \ --prompt "Describe this image." \ --image families/qwen_vl/tests/data/test_img.jpeg \ --max-new-tokens 48 @@ -59,7 +51,6 @@ python -m tensorrt_model_connect build openai/whisper-large-v3-turbo \ --output /tmp/whisper.bundle trtmc transcribe /tmp/whisper.bundle \ - --runtime-root "$TRTMC_RUNTIME_ROOT" \ --input families/whisper/tests/data/Recording.wav \ --max-output-tokens 224 ``` @@ -70,7 +61,6 @@ python -m tensorrt_model_connect build nvidia/magpie_tts_multilingual_357m \ --output /tmp/magpie.bundle trtmc generate-audio /tmp/magpie.bundle \ - --runtime-root "$TRTMC_RUNTIME_ROOT" \ --prompt "A clear short test sentence." \ --output /tmp/magpie.wav ``` @@ -91,7 +81,6 @@ python -m tensorrt_model_connect build nvidia/segformer-b0-finetuned-ade-512-512 --output /tmp/segformer.bundle trtmc segment /tmp/segformer.bundle \ - --runtime-root "$TRTMC_RUNTIME_ROOT" \ --image families/segformer/tests/data/test_img.jpeg ``` @@ -109,7 +98,6 @@ python -m tensorrt_model_connect build amazon/chronos-bolt-tiny \ --output /tmp/chronos.bundle trtmc forecast /tmp/chronos.bundle \ - --runtime-root "$TRTMC_RUNTIME_ROOT" \ --input /path/to/history.f32 ``` diff --git a/website/docs/getting-started/glossary.md b/website/docs/getting-started/glossary.md index e5471e7484..41483248cb 100644 --- a/website/docs/getting-started/glossary.md +++ b/website/docs/getting-started/glossary.md @@ -26,10 +26,10 @@ title: Glossary | Task | User-visible behavior. | Abstract interfaces such as text generation, transcription, segmentation, embedding, and forecast. | | Backend | Engine implementation. | `trt` or optional `trt_rtx`, selected by the bundle header. | | DSO | Linux shared library loaded at runtime. | Core, loader, one backend, and exactly one `libtrtmc_model_.so`. | -| Runtime root | Explicit DSO directory. | Required by every native execution command; no fallback search exists. | +| Runtime root | One selected DSO directory. | The CLI discovers or explicitly selects it; the Runtime Loader never searches or falls back. | | Precision | Numeric representation. | A build request such as FP32, FP16, or BF16 that the family validates. | | Quantization | Lower-precision graph/weights such as FP8. | Entirely family-owned and qualified per exact checkpoint/path. | -| ABI | Binary compatibility contract. | Runtime DSOs and TensorRT plans must match their software/hardware cohort. | +| ABI | Binary compatibility contract. | Runtime DSOs must share one exact product build and use a supported TensorRT environment. | ## Project building blocks diff --git a/website/docs/getting-started/quick-start.md b/website/docs/getting-started/quick-start.md index 26f29757ae..febafbab76 100644 --- a/website/docs/getting-started/quick-start.md +++ b/website/docs/getting-started/quick-start.md @@ -41,29 +41,40 @@ selecting another task supported by the same family. The build then imports only the selected `families.gpt2.model` and calls `build(request, writer)` once. A prepared local snapshot can be passed in place of the model ID. -For a wheel install, resolve its native runtime directory directly from the -installed package: +Run the bundle directly: ```bash -TRTMC_RUNTIME_ROOT="$(python -c 'import pathlib, tensorrt_model_connect as m; print(pathlib.Path(m.__file__).parent / "bin")')" trtmc run gpt2.bundle \ - --runtime-root "$TRTMC_RUNTIME_ROOT" \ --prompt "Hello" \ --max-new-tokens 32 ``` -For a native CMake install, point the loader at the directory containing the matching -`libtrtmc_core.so`, `libtrtmc_runtime.so`, `libtrtmc_backend_trt.so`, and -`libtrtmc_model_gpt2.so`. The loader reads the bundle header, loads exactly -those DSOs, and returns the abstract task interface declared by the bundle. +The CLI reads the bundle family and backend, then selects the first complete, +single-directory plugin root in this order: -```bash -trtmc run gpt2.bundle \ - --runtime-root /opt/trtmc/lib \ - --prompt "Hello" \ - --max-new-tokens 32 -``` +1. the directory containing the active `libtrtmc_runtime.so`; +2. colon-separated directories in `TRTMC_RUNTIME_PATH`. + +The wheel console command replaces itself with the native CLI stored beside +Core, Runtime, backend, and family DSOs. Native installs resolve their already +loaded Runtime directory in the same way, so discovery does not inspect Python +installation layouts or scan `PATH` directories. + +A complete GPT-2 TensorRT plugin root contains root-local +`libtrtmc_backend_trt.so` and `libtrtmc_model_gpt2.so` files. Candidates are +never combined across directories, and discovery never loads a candidate just +to inspect it. After selection, the Runtime Loader loads those exact paths and +requires their descriptors to match the active product build, plugin kinds, +and bundle IDs before it calls either factory. A mismatched selected root fails +immediately without falling back to another installation. + +Before bundle inspection crosses a C++ interface, the CLI verifies that its +own product-build identity matches the already loaded Runtime and Core. The +Runtime Loader repeats the Core check and validates selected plugins at load. -The shell variable above is only a convenient explicit argument. The CLI never -searches environment variables, the current directory, or an installed -fallback runtime. +The CLI prints the automatically selected directory. Use `--runtime-root DIR` +to override it with one exact root. An explicit root bypasses discovery but not +build and identity validation. The current directory is not searched +implicitly; use `TRTMC_RUNTIME_PATH=.` when that behavior is intended. +`LD_LIBRARY_PATH` remains a platform-loader setting evaluated before the CLI +starts. diff --git a/website/docs/getting-started/troubleshooting.md b/website/docs/getting-started/troubleshooting.md index a8bf1bc807..0ee4217c60 100644 --- a/website/docs/getting-started/troubleshooting.md +++ b/website/docs/getting-started/troubleshooting.md @@ -15,9 +15,9 @@ Identify the first boundary that fails in the [Quick Start](quick-start.md). | Build OOM or disk failure | Requested checkpoint, shape, precision, and cache capacity | Use the exact family manifest/profile or free capacity; retain the first error. | | No family or multiple families match | Root checkpoint identity metadata | Use a supported exact checkpoint; do not add prefix/fallback matching. | | Bundle inspection fails | Partial/corrupt bundle | Rebuild; failed builds must not publish a partial output. | -| Family/backend DSO missing | `--runtime-root` contents | Confirm `libtrtmc_core.so`, `libtrtmc_runtime.so`, selected backend, and exact family DSO are together. | +| Family/backend DSO missing | Selected runtime-root contents | Confirm `libtrtmc_core.so`, `libtrtmc_runtime.so`, selected backend, and exact family DSO are together. | | Task mismatch | Bundle `task` versus CLI command | Use the Task command named by the family manifest/header. | -| TensorRT/DSO ABI error | Mixed software or hardware cohort | Run with a compatible environment and rebuild the bundle when required. | +| TensorRT/DSO ABI error | Mixed native product builds or incompatible hardware/software | Use one product build in a compatible environment and rebuild the bundle when required. | | Output differs | Revision, input framing, precision, sampling, oracle | Reproduce the exact family testcase before changing code or thresholds. | Collect the source revision, model ID/revision, complete build/run commands, diff --git a/website/docs/reference/profiling.md b/website/docs/reference/profiling.md index b94e6fb472..3852311440 100644 --- a/website/docs/reference/profiling.md +++ b/website/docs/reference/profiling.md @@ -14,7 +14,6 @@ First prove that the same bundle and request complete normally: ```bash trtmc run model.bundle \ - --runtime-root /opt/trtmc/lib \ --prompt "The capital of France is" \ --max-new-tokens 20 \ --seed 1234 @@ -39,9 +38,9 @@ Hold these inputs constant across comparisons: - Use family-owned tests and reference comparisons to establish correctness before interpreting a speedup. -Profile loading separately from steady-state task execution. The public CLI -requires `--runtime-root`; include its exact directory in the evidence so the -loaded shared objects are reproducible. +Profile loading separately from steady-state task execution. Record the exact +runtime directory printed by automatic discovery in the evidence so the loaded +shared objects are reproducible. ## Interpreting results diff --git a/website/docs/tutorials/advanced/multi-device-inference.md b/website/docs/tutorials/advanced/multi-device-inference.md index 938f528357..927381cbf4 100644 --- a/website/docs/tutorials/advanced/multi-device-inference.md +++ b/website/docs/tutorials/advanced/multi-device-inference.md @@ -37,7 +37,6 @@ Use the MPI launcher and rank count required by the family-owned manifest: ```bash mpirun -n 4 trtmc run qwen-tp4.bundle \ - --runtime-root /opt/trtmc/lib \ --prompt "Explain tensor parallelism." \ --max-new-tokens 64 \ --seed 1234 diff --git a/website/docs/tutorials/advanced/quantization-and-runtime-knobs.md b/website/docs/tutorials/advanced/quantization-and-runtime-knobs.md index a3dcb89f26..790d81c46c 100644 --- a/website/docs/tutorials/advanced/quantization-and-runtime-knobs.md +++ b/website/docs/tutorials/advanced/quantization-and-runtime-knobs.md @@ -49,7 +49,6 @@ Every execution command requires the installed runtime directory: ```bash trtmc run qwen.bundle \ - --runtime-root /opt/trtmc/lib \ --runtime-cache /tmp/trtmc-cache \ --cuda-graphs \ --kv-cache-size 4096 \ diff --git a/website/docs/tutorials/beginner/inspect-bundles.md b/website/docs/tutorials/beginner/inspect-bundles.md index 5bf49ac16a..a51f35dce9 100644 --- a/website/docs/tutorials/beginner/inspect-bundles.md +++ b/website/docs/tutorials/beginner/inspect-bundles.md @@ -50,8 +50,8 @@ The relevant boundaries are: 1. If inspection cannot parse the header, investigate the build or artifact. 2. If required sections are absent, investigate the family builder. -3. If inspection succeeds but loading fails, verify `--runtime-root` contains - the installed core, backend, and family libraries from a compatible build. +3. If inspection succeeds but loading fails, verify the selected runtime root + contains the installed core, backend, and family libraries from one build. 4. If loading succeeds but a request fails, route the issue to the family task contract and model-owned tests. diff --git a/website/docs/tutorials/beginner/text-generation.md b/website/docs/tutorials/beginner/text-generation.md index 74f81038c5..de7b731f2c 100644 --- a/website/docs/tutorials/beginner/text-generation.md +++ b/website/docs/tutorials/beginner/text-generation.md @@ -13,7 +13,6 @@ reuses `./gpt2.bundle`. trtmc inspect ./gpt2.bundle trtmc run ./gpt2.bundle \ - --runtime-root /opt/trtmc/lib \ --prompt "Explain why KV caches help decoding." \ --max-new-tokens 80 \ --temperature 0.7 \ diff --git a/website/docs/tutorials/intermediate/canary-decoding.md b/website/docs/tutorials/intermediate/canary-decoding.md index 73fdc4d986..f7a2b6c26f 100644 --- a/website/docs/tutorials/intermediate/canary-decoding.md +++ b/website/docs/tutorials/intermediate/canary-decoding.md @@ -24,7 +24,6 @@ metadata, runtime orchestration, and validation. ```bash trtmc transcribe /tmp/canary-1b-v2.bundle \ - --runtime-root /opt/trtmc/lib \ --input /data/input.wav \ --max-output-tokens 80 \ --source-language en \ @@ -40,7 +39,6 @@ segmentation are request options: ```bash trtmc transcribe /tmp/canary-1b-v2.bundle \ - --runtime-root /opt/trtmc/lib \ --input /data/english.wav \ --source-language en \ --target-language fr \ diff --git a/website/docs/tutorials/intermediate/diffusion-and-time-series.md b/website/docs/tutorials/intermediate/diffusion-and-time-series.md index 0ff20ccdcb..a2739fa391 100644 --- a/website/docs/tutorials/intermediate/diffusion-and-time-series.md +++ b/website/docs/tutorials/intermediate/diffusion-and-time-series.md @@ -17,7 +17,6 @@ python -m tensorrt_model_connect build black-forest-labs/FLUX.1-schnell \ --image-width 1024 trtmc generate-image flux.bundle \ - --runtime-root /opt/trtmc/lib \ --prompt "A brass robot reading beside a window" \ --output robot.png \ --height 1024 \ @@ -41,7 +40,6 @@ python -m tensorrt_model_connect build Wan-AI/Wan2.1-T2V-1.3B \ --video-num-frames 81 trtmc generate-video wan.bundle \ - --runtime-root /opt/trtmc/lib \ --prompt "Ocean waves under moonlight" \ --output waves.mp4 \ --height 480 \ @@ -63,7 +61,6 @@ python -m tensorrt_model_connect build amazon/chronos-bolt-tiny \ --precision fp32 trtmc forecast chronos.bundle \ - --runtime-root /opt/trtmc/lib \ --input history.f32 \ --frequency H ``` diff --git a/website/docs/tutorials/intermediate/multimodal-and-speech.md b/website/docs/tutorials/intermediate/multimodal-and-speech.md index c04854e8ce..e772a1a881 100644 --- a/website/docs/tutorials/intermediate/multimodal-and-speech.md +++ b/website/docs/tutorials/intermediate/multimodal-and-speech.md @@ -19,7 +19,6 @@ python -m tensorrt_model_connect build Qwen/Qwen2.5-VL-3B-Instruct \ --max-sequence-length 384 trtmc run qwen-vl.bundle \ - --runtime-root /opt/trtmc/lib \ --image sample.png \ --prompt "Describe the image." \ --max-new-tokens 80 @@ -33,12 +32,10 @@ between vision and decoder engines are family-owned. The shared builder has no ```bash trtmc transcribe whisper.bundle \ - --runtime-root /opt/trtmc/lib \ --input recording.wav \ --max-output-tokens 128 trtmc transcribe-streaming nemotron-asr.bundle \ - --runtime-root /opt/trtmc/lib \ --input recording.wav \ --chunk-samples 16000 \ --max-new-tokens 128 @@ -52,13 +49,11 @@ family supports them. ```bash trtmc generate-audio audio.bundle \ - --runtime-root /opt/trtmc/lib \ --prompt "A calm spoken welcome" \ --output welcome.wav \ --seed 1234 trtmc speech-session voicechat.bundle \ - --runtime-root /opt/trtmc/lib \ --input question.wav \ --output answer.wav \ --system-prompt "Answer concisely." \ diff --git a/website/docs/user-guides/configure-runtime.md b/website/docs/user-guides/configure-runtime.md index 82f3f62dca..f3463ebcad 100644 --- a/website/docs/user-guides/configure-runtime.md +++ b/website/docs/user-guides/configure-runtime.md @@ -18,7 +18,6 @@ configs. Family-only state remains in family sections and implementation code. ```bash trtmc run model.bundle \ - --runtime-root /opt/trtmc/lib \ --kv-cache-size 4GiB \ --prompt "Hello" \ --temperature 0 diff --git a/website/docs/user-guides/image-video-generation.md b/website/docs/user-guides/image-video-generation.md index 0af4dc7ac8..4bb14461ab 100644 --- a/website/docs/user-guides/image-video-generation.md +++ b/website/docs/user-guides/image-video-generation.md @@ -16,7 +16,6 @@ python -m tensorrt_model_connect build MODEL_ID \ --output media.bundle trtmc generate-video media.bundle \ - --runtime-root /opt/trtmc/lib \ --prompt "A sunrise over a mountain lake" \ --output frames \ --num-steps 28 \ @@ -30,17 +29,15 @@ file plus comma-separated seeds for a family that declares the batch Task. ```bash trtmc classify classifier.bundle \ - --runtime-root /opt/trtmc/lib --image input.jpg + --image input.jpg trtmc segment segmenter.bundle \ - --runtime-root /opt/trtmc/lib --image input.jpg + --image input.jpg trtmc segment-prompted prompted.bundle \ - --runtime-root /opt/trtmc/lib \ --image input.jpg --point-x 0.5 --point-y 0.5 --foreground true trtmc geometry moge.bundle \ - --runtime-root /opt/trtmc/lib \ --image input.jpg --output geometry-output ``` diff --git a/website/docs/user-guides/multimodal-speech.md b/website/docs/user-guides/multimodal-speech.md index 95a7f89b71..faedef6e25 100644 --- a/website/docs/user-guides/multimodal-speech.md +++ b/website/docs/user-guides/multimodal-speech.md @@ -7,7 +7,6 @@ description: Vision-language, transcription, audio generation, and speech-sessio ```bash trtmc run vision-language.bundle \ - --runtime-root /opt/trtmc/lib \ --image input.jpg \ --prompt "Describe this image in one sentence." \ --max-new-tokens 48 @@ -20,7 +19,6 @@ request values belong to the selected family. ```bash trtmc transcribe speech-to-text.bundle \ - --runtime-root /opt/trtmc/lib \ --input input.wav \ --beam-size 1 \ --source-language en \ @@ -38,12 +36,10 @@ contracts. ```bash trtmc generate-audio text-to-audio.bundle \ - --runtime-root /opt/trtmc/lib \ --prompt "A clear short test sentence." \ --output output.wav trtmc speak speech-to-speech.bundle \ - --runtime-root /opt/trtmc/lib \ --input input.wav \ --output response.wav ``` @@ -54,7 +50,6 @@ application for a persistent local session: ```bash trtmc speech-session nemotron-voicechat.bundle \ - --runtime-root /opt/trtmc/lib \ --input input.wav \ --output response.wav \ --timeout-ms 30000 diff --git a/website/docs/user-guides/run-inference.md b/website/docs/user-guides/run-inference.md index 29b88c31da..b01794fe2f 100644 --- a/website/docs/user-guides/run-inference.md +++ b/website/docs/user-guides/run-inference.md @@ -4,7 +4,8 @@ description: Select the native command matching the bundle's declared Task. --- Inspect the bundle, then call the Task named by its family manifest/header. -Every execution command requires `--runtime-root DIR`. +Execution commands discover the active installation by default. Use +`--runtime-root DIR` only to select one exact plugin root. | Task | Command | Primary result | | --- | --- | --- | @@ -19,13 +20,13 @@ Every execution command requires `--runtime-root DIR`. ```bash trtmc run qwen3-0.6b.bundle \ - --runtime-root /opt/trtmc/lib \ --prompt "What is the capital of France? Answer in one word." \ --max-new-tokens 10 \ --temperature 0 \ --top-k 1 ``` -The CLI loads exactly one family DSO and backend DSO from the runtime root. A -wrong Task command fails instead of attempting another family or interface. -See the [CLI Reference](../api/cli-reference.md) for command-specific inputs. +The CLI selects one root, then the Runtime Loader validates and loads exactly +one family DSO and backend DSO from it. A wrong Task command or mismatched-build +plugin fails instead of attempting another root, family, or interface. See the +[CLI Reference](../api/cli-reference.md) for command-specific inputs. diff --git a/website/docs/user-guides/text-generation.md b/website/docs/user-guides/text-generation.md index 226576e7c1..18a185640d 100644 --- a/website/docs/user-guides/text-generation.md +++ b/website/docs/user-guides/text-generation.md @@ -7,7 +7,6 @@ Build and inspect an exact text checkpoint, then call the native Task: ```bash trtmc run model.bundle \ - --runtime-root /opt/trtmc/lib \ --prompt "Summarize TensorRT in one sentence." \ --max-new-tokens 48 \ --temperature 0 \ @@ -18,7 +17,6 @@ For reproducible stochastic sampling, fix every sampling input: ```bash trtmc run model.bundle \ - --runtime-root /opt/trtmc/lib \ --prompt "Write a two-line GPU poem." \ --max-new-tokens 64 \ --temperature 0.8 \ diff --git a/website/docs/user-guides/time-series.md b/website/docs/user-guides/time-series.md index d60b852535..384b8ddf66 100644 --- a/website/docs/user-guides/time-series.md +++ b/website/docs/user-guides/time-series.md @@ -9,7 +9,6 @@ input files. ```bash trtmc forecast forecast.bundle \ - --runtime-root /opt/trtmc/lib \ --input history.f32 ``` @@ -17,7 +16,6 @@ Other model contracts can use: ```bash trtmc solve operator.bundle \ - --runtime-root /opt/trtmc/lib \ --branch branch.f32 \ --trunk trunk.f32 ```