From 42cdea6c137839741443c9794610321fcfa59d8d Mon Sep 17 00:00:00 2001 From: Joshua Calafato Date: Thu, 10 Sep 2026 22:03:53 +0000 Subject: [PATCH 1/2] feat(edgellm): add newer Qwen API dispatch Provision pinned native Edge-LLM through optional CMake infrastructure. Let Qwen3.5 and Qwen3.8 own configuration/platform dispatch and thin builder/runtime API adapters, with warning and one native retry after builder failure. Add family-local contract and real inference validation. Keep legacy Qwen unchanged and document unqualified configurations and the architecture checks awaiting maintainer review. Signed-off-by: Joshua Calafato --- CMakeLists.txt | 8 + cmake/EdgeLLM.cmake | 94 ++++ cmake/edgellm/CheckNative.cmake | 34 ++ cmake/edgellm/EdgeLLMConfig.cmake.in | 40 ++ cmake/edgellm/Install.cmake.in | 22 + cmake/edgellm/Prepare.cmake.in | 36 ++ cmake/edgellm/README.md | 56 +++ cmake/edgellm/tests/package_contract.cmake | 76 +++ core/builder/tensorrt_model_connect/build.py | 38 ++ core/builder/tests/test_build.py | 56 +++ core/runtime/bundle/bundle_format.cpp | 27 ++ core/runtime/include/trtmc/bundle.h | 7 + core/runtime/tests/test_bundle_format_v1.cpp | 32 ++ families/qwen3_5/EDGE_LLM.md | 98 ++++ families/qwen3_5/dispatch.py | 109 +++++ families/qwen3_5/edge_llm.py | 123 +++++ families/qwen3_5/model.py | 9 +- families/qwen3_5/runtime/CMakeLists.txt | 87 +++- families/qwen3_5/runtime/edge_llm/adapter.cpp | 224 +++++++++ families/qwen3_5/runtime/edge_llm/adapter.h | 15 + families/qwen3_5/runtime/edge_llm/contract.h | 56 +++ .../qwen3_5/runtime/edge_llm/device_link.cu | 5 + families/qwen3_5/runtime/edge_llm/request.h | 29 ++ families/qwen3_5/runtime/plugin.cpp | 11 + .../tests/cpp/test_qwen3_5_edge_contract.cpp | 92 ++++ .../tests/cpp/test_qwen3_5_edge_factory.cpp | 95 ++++ .../tests/cpp/test_qwen3_5_edge_inference.cpp | 142 ++++++ .../tests/cpp/test_qwen3_5_edge_reference.cpp | 135 ++++++ .../tests/cpp/test_qwen3_5_edge_request.cpp | 41 ++ families/qwen3_5/tests/edge_validation.py | 371 ++++++++++++++ families/qwen3_5/tests/test_edge_llm.py | 452 ++++++++++++++++++ .../qwen3_5/tests/test_edge_validation.py | 158 ++++++ families/qwen3_8/EDGE_LLM.md | 55 +++ families/qwen3_8/dispatch.py | 109 +++++ families/qwen3_8/edge_llm.py | 123 +++++ families/qwen3_8/model.py | 9 +- families/qwen3_8/runtime/CMakeLists.txt | 77 ++- families/qwen3_8/runtime/edge_llm/adapter.cpp | 224 +++++++++ families/qwen3_8/runtime/edge_llm/adapter.h | 15 + families/qwen3_8/runtime/edge_llm/contract.h | 56 +++ .../qwen3_8/runtime/edge_llm/device_link.cu | 5 + families/qwen3_8/runtime/edge_llm/request.h | 29 ++ families/qwen3_8/runtime/plugin.cpp | 11 + .../tests/cpp/test_qwen3_8_edge_contract.cpp | 92 ++++ .../tests/cpp/test_qwen3_8_edge_factory.cpp | 95 ++++ .../tests/cpp/test_qwen3_8_edge_inference.cpp | 142 ++++++ .../tests/cpp/test_qwen3_8_edge_request.cpp | 41 ++ families/qwen3_8/tests/test_edge_llm.py | 452 ++++++++++++++++++ 48 files changed, 4309 insertions(+), 4 deletions(-) create mode 100644 cmake/EdgeLLM.cmake create mode 100644 cmake/edgellm/CheckNative.cmake create mode 100644 cmake/edgellm/EdgeLLMConfig.cmake.in create mode 100644 cmake/edgellm/Install.cmake.in create mode 100644 cmake/edgellm/Prepare.cmake.in create mode 100644 cmake/edgellm/README.md create mode 100644 cmake/edgellm/tests/package_contract.cmake create mode 100644 families/qwen3_5/EDGE_LLM.md create mode 100644 families/qwen3_5/dispatch.py create mode 100644 families/qwen3_5/edge_llm.py create mode 100644 families/qwen3_5/runtime/edge_llm/adapter.cpp create mode 100644 families/qwen3_5/runtime/edge_llm/adapter.h create mode 100644 families/qwen3_5/runtime/edge_llm/contract.h create mode 100644 families/qwen3_5/runtime/edge_llm/device_link.cu create mode 100644 families/qwen3_5/runtime/edge_llm/request.h create mode 100644 families/qwen3_5/tests/cpp/test_qwen3_5_edge_contract.cpp create mode 100644 families/qwen3_5/tests/cpp/test_qwen3_5_edge_factory.cpp create mode 100644 families/qwen3_5/tests/cpp/test_qwen3_5_edge_inference.cpp create mode 100644 families/qwen3_5/tests/cpp/test_qwen3_5_edge_reference.cpp create mode 100644 families/qwen3_5/tests/cpp/test_qwen3_5_edge_request.cpp create mode 100644 families/qwen3_5/tests/edge_validation.py create mode 100644 families/qwen3_5/tests/test_edge_llm.py create mode 100644 families/qwen3_5/tests/test_edge_validation.py create mode 100644 families/qwen3_8/EDGE_LLM.md create mode 100644 families/qwen3_8/dispatch.py create mode 100644 families/qwen3_8/edge_llm.py create mode 100644 families/qwen3_8/runtime/edge_llm/adapter.cpp create mode 100644 families/qwen3_8/runtime/edge_llm/adapter.h create mode 100644 families/qwen3_8/runtime/edge_llm/contract.h create mode 100644 families/qwen3_8/runtime/edge_llm/device_link.cu create mode 100644 families/qwen3_8/runtime/edge_llm/request.h create mode 100644 families/qwen3_8/tests/cpp/test_qwen3_8_edge_contract.cpp create mode 100644 families/qwen3_8/tests/cpp/test_qwen3_8_edge_factory.cpp create mode 100644 families/qwen3_8/tests/cpp/test_qwen3_8_edge_inference.cpp create mode 100644 families/qwen3_8/tests/cpp/test_qwen3_8_edge_request.cpp create mode 100644 families/qwen3_8/tests/test_edge_llm.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 2d653e3281..cee44bfe0b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,6 +3,9 @@ cmake_minimum_required(VERSION 3.20) project(tensorrt_model_connect VERSION 0.1.0 LANGUAGES CXX CUDA) +if(CMAKE_CROSSCOMPILING) + message(FATAL_ERROR "TensorRT Model Connect does not support cross compilation") +endif() set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) @@ -39,6 +42,8 @@ find_library(TRTMC_TRT_LIBRARY REQUIRED ) +include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/EdgeLLM.cmake") + option(TRTMC_ENABLE_BYOK "Enable the optional TVM-FFI BYOK bridge" ON) set(TRTMC_HAS_TVM_FFI OFF) if(TRTMC_ENABLE_BYOK) @@ -245,6 +250,9 @@ option(TRTMC_BUILD_TESTS "Build tests" ON) option(TRTMC_BUILD_EXAMPLES "Build examples" ON) if(TRTMC_BUILD_TESTS) enable_testing() + add_test(NAME test_edgellm_package_contract + COMMAND "${CMAKE_COMMAND}" "-DTEST_ROOT=${PROJECT_BINARY_DIR}/edgellm-package-contract" + -P "${PROJECT_SOURCE_DIR}/cmake/edgellm/tests/package_contract.cmake") endif() # A family owns its target, sources, dependencies, warnings, and installation. diff --git a/cmake/EdgeLLM.cmake b/cmake/EdgeLLM.cmake new file mode 100644 index 0000000000..26c6b38f5b --- /dev/null +++ b/cmake/EdgeLLM.cmake @@ -0,0 +1,94 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Optional native dependency provisioning. Model builds never acquire dependencies. +option(TRTMC_ENABLE_EDGELLM "Install the pinned native Edge-LLM SDK and builder" OFF) +if(NOT TRTMC_ENABLE_EDGELLM) + return() +endif() +if(CMAKE_CROSSCOMPILING) + message(FATAL_ERROR "Edge-LLM cross compilation is not supported") +endif() +include("${CMAKE_CURRENT_LIST_DIR}/edgellm/CheckNative.cmake") +set(_edge_version "0.10.1") +set(_edge_revision "e8b29522938901f6df19ebeedd4b69bc8edbcd97") +set(_edge_root "${CMAKE_BINARY_DIR}/_deps/edgellm") +set(_edge_prefix "${_edge_root}/install") +find_package(EdgeLLM ${_edge_version} EXACT CONFIG QUIET) +if(EdgeLLM_FOUND AND NOT EdgeLLM_PREFIX STREQUAL _edge_prefix) + if(NOT EdgeLLM_REVISION STREQUAL _edge_revision) + message(FATAL_ERROR "EdgeLLM package does not match the pinned GitHub revision") + endif() + install(FILES "$" DESTINATION "${CMAKE_INSTALL_LIBDIR}" COMPONENT EdgeLLM) + return() +endif() + +include(ExternalProject) +include(CMakePackageConfigHelpers) +find_package(Python3 3.10 REQUIRED COMPONENTS Interpreter) +find_package(Threads REQUIRED) +set(TRTMC_EDGELLM_TRT_ROOT "$ENV{TRT_ROOT}" CACHE PATH "Native TensorRT SDK, including its Python wheel") +set(TRTMC_EDGELLM_CUDA_ARCHITECTURE "${CMAKE_CUDA_ARCHITECTURES}" CACHE STRING "One local GPU architecture for Edge-LLM") +set(TRTMC_EDGELLM_JOBS 2 CACHE STRING "Parallel Edge-LLM native and AOT compilation jobs") +set(TRTMC_EDGELLM_WHEELHOUSE "" CACHE PATH "Optional complete offline Python wheelhouse") +set(TRTMC_EDGELLM_GIT_MIRROR "" CACHE PATH "Optional local mirror of the pinned upstream Git repository") +if(NOT TRTMC_EDGELLM_CUDA_ARCHITECTURE MATCHES "^[0-9]+$") + message(FATAL_ERROR "Set TRTMC_EDGELLM_CUDA_ARCHITECTURE to one local GPU architecture, e.g. 80") +endif() +if(NOT EXISTS "${TRTMC_EDGELLM_TRT_ROOT}/include/NvInfer.h") + message(FATAL_ERROR "TRTMC_EDGELLM_TRT_ROOT must contain the native TensorRT SDK") +endif() +_edgellm_check_gpu("${TRTMC_EDGELLM_CUDA_ARCHITECTURE}") +_edgellm_trt_version("${TRTMC_EDGELLM_TRT_ROOT}/include" _edge_trt_version) +set(_edge_source "${_edge_root}/source") +set(_edge_build "${_edge_root}/build") +set(_edge_python "${_edge_prefix}/libexec/trtmc-edge-llm/bin/python") +set(_edge_repository "https://github.com/NVIDIA/TensorRT-Edge-LLM.git") +if(TRTMC_EDGELLM_GIT_MIRROR) + set(_edge_repository "${TRTMC_EDGELLM_GIT_MIRROR}") +endif() +set(_edge_template_dir "${CMAKE_CURRENT_LIST_DIR}/edgellm") +file(MAKE_DIRECTORY "${_edge_prefix}/lib/cmake/EdgeLLM" "${_edge_prefix}/include/edgellm/cpp" + "${_edge_prefix}/include/edgellm/3rdParty/nlohmannJson/include" + "${_edge_prefix}/include/edgellm/3rdParty/stb" "${_edge_prefix}/include/edgellm/3rdParty/miniaudio") +configure_file("${_edge_template_dir}/CheckNative.cmake" "${_edge_prefix}/lib/cmake/EdgeLLM/CheckNative.cmake" COPYONLY) +foreach(_script IN ITEMS Prepare Install) + configure_file("${_edge_template_dir}/${_script}.cmake.in" "${_edge_root}/${_script}.cmake" @ONLY) +endforeach() +configure_file("${_edge_template_dir}/EdgeLLMConfig.cmake.in" + "${_edge_prefix}/lib/cmake/EdgeLLM/EdgeLLMConfig.cmake" @ONLY) +write_basic_package_version_file("${_edge_prefix}/lib/cmake/EdgeLLM/EdgeLLMConfigVersion.cmake" + VERSION "${_edge_version}" COMPATIBILITY ExactVersion) +ExternalProject_Add(trtmc_edgellm_dependency + PREFIX "${_edge_root}/ep" SOURCE_DIR "${_edge_source}" BINARY_DIR "${_edge_build}" + GIT_REPOSITORY "${_edge_repository}" GIT_TAG "${_edge_revision}" + GIT_SUBMODULES_RECURSE TRUE UPDATE_DISCONNECTED TRUE + LIST_SEPARATOR | + CMAKE_COMMAND "${_edge_prefix}/libexec/trtmc-edge-llm/bin/cmake" + PATCH_COMMAND "${CMAKE_COMMAND}" -P "${_edge_root}/Prepare.cmake" + CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release -DCMAKE_POSITION_INDEPENDENT_CODE=ON + "-DCMAKE_CUDA_COMPILER=${CMAKE_CUDA_COMPILER}" + "-DCMAKE_CUDA_ARCHITECTURES=${TRTMC_EDGELLM_CUDA_ARCHITECTURE}" + "-DCUDA_DIR=${CUDAToolkit_LIBRARY_ROOT}" "-DCUDAToolkit_ROOT=${CUDAToolkit_LIBRARY_ROOT}" + "-DCUDA_CTK_VERSION=${CUDAToolkit_VERSION_MAJOR}.${CUDAToolkit_VERSION_MINOR}" + "-DTRT_PACKAGE_DIR=${TRTMC_EDGELLM_TRT_ROOT}" "-DPython3_EXECUTABLE=${_edge_python}" + -DEDGELLM_WHEEL_PAYLOAD_DIR=unused -DENABLE_CUTE_DSL=fmha|gdn + "-DCUTE_DSL_ARTIFACT_TAG=sm_${TRTMC_EDGELLM_CUDA_ARCHITECTURE}" + BUILD_COMMAND "${CMAKE_COMMAND}" --build --target edgellmCore NvInfer_edgellm_plugin + --parallel "${TRTMC_EDGELLM_JOBS}" + INSTALL_COMMAND "${CMAKE_COMMAND}" -P "${_edge_root}/Install.cmake" + BUILD_BYPRODUCTS "${_edge_prefix}/lib/libedgellmCore.a" + "${_edge_prefix}/lib/libNvInfer_edgellm_plugin.so" + "${_edge_prefix}/lib/libcutedsl.a" + LOG_DOWNLOAD ON LOG_CONFIGURE ON LOG_BUILD ON LOG_INSTALL ON LOG_OUTPUT_ON_FAILURE ON) +ExternalProject_Add_StepDependencies(trtmc_edgellm_dependency patch "${_edge_root}/Prepare.cmake") +ExternalProject_Add_StepDependencies(trtmc_edgellm_dependency install "${_edge_root}/Install.cmake") +# Generated package targets refer to declared future byproducts; their build dependency +# prevents consumers from compiling or linking until installation completes. +find_package(EdgeLLM ${_edge_version} EXACT CONFIG REQUIRED + PATHS "${_edge_prefix}/lib/cmake/EdgeLLM" NO_DEFAULT_PATH) +add_dependencies(EdgeLLM::Core trtmc_edgellm_dependency) +add_dependencies(EdgeLLM::Plugin trtmc_edgellm_dependency) +install(DIRECTORY "${_edge_prefix}/" DESTINATION . USE_SOURCE_PERMISSIONS COMPONENT EdgeLLM) +# Family DSOs may use lib64; their dynamically loaded plugin must remain adjacent. +install(FILES "$" DESTINATION "${CMAKE_INSTALL_LIBDIR}" COMPONENT EdgeLLM) diff --git a/cmake/edgellm/CheckNative.cmake b/cmake/edgellm/CheckNative.cmake new file mode 100644 index 0000000000..a78a0b4901 --- /dev/null +++ b/cmake/edgellm/CheckNative.cmake @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Read the complete TensorRT SDK version using the compiler, including aliased macros. +# include_dir: native SDK include directory; output: caller variable receiving x.y.z.build. +function(_edgellm_trt_version include_dir output) + set(_version) + set(_probe "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/edgellm-version.cpp") + file(WRITE "${_probe}" "#include \n") + foreach(_part IN ITEMS MAJOR MINOR PATCH BUILD) + file(APPEND "${_probe}" "TRTMC_EDGE_${_part}=NV_TENSORRT_${_part}\n") + endforeach() + execute_process(COMMAND "${CMAKE_CXX_COMPILER}" -E -P -I "${include_dir}" "${_probe}" + OUTPUT_VARIABLE _expanded COMMAND_ERROR_IS_FATAL ANY) + foreach(_part IN ITEMS MAJOR MINOR PATCH BUILD) + if(NOT _expanded MATCHES "TRTMC_EDGE_${_part}=[ \t]*([0-9]+)") + message(FATAL_ERROR "Cannot determine TensorRT ${_part} from ${include_dir}") + endif() + list(APPEND _version "${CMAKE_MATCH_1}") + endforeach() + list(JOIN _version "." _version) + set(${output} "${_version}" PARENT_SCOPE) +endfunction() + +# Require the installed package GPU architecture to be present on this build host. +function(_edgellm_check_gpu architecture) + execute_process(COMMAND nvidia-smi --query-gpu=compute_cap --format=csv,noheader + OUTPUT_VARIABLE _sms RESULT_VARIABLE _result OUTPUT_STRIP_TRAILING_WHITESPACE) + string(REPLACE "." "" _sms "${_sms}") + string(REPLACE "\n" ";" _sms "${_sms}") + if(NOT _result EQUAL 0 OR NOT architecture IN_LIST _sms) + message(FATAL_ERROR "EdgeLLM requires a local GPU with architecture ${architecture}") + endif() +endfunction() diff --git a/cmake/edgellm/EdgeLLMConfig.cmake.in b/cmake/edgellm/EdgeLLMConfig.cmake.in new file mode 100644 index 0000000000..945b76a610 --- /dev/null +++ b/cmake/edgellm/EdgeLLMConfig.cmake.in @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +include(CMakeFindDependencyMacro) +find_dependency(CUDAToolkit) +find_dependency(Threads) +find_dependency(nlohmann_json 3.12.0 EXACT) +include("${CMAKE_CURRENT_LIST_DIR}/CheckNative.cmake") +get_filename_component(EdgeLLM_PREFIX "${CMAKE_CURRENT_LIST_DIR}/../../.." ABSOLUTE) +set(EdgeLLM_VERSION "@_edge_version@") +set(EdgeLLM_REVISION "@_edge_revision@") +set(EdgeLLM_CUDA_VERSION "@CUDAToolkit_VERSION@") +set(EdgeLLM_TENSORRT_VERSION "@_edge_trt_version@") +set(EdgeLLM_ARCH "@CMAKE_SYSTEM_PROCESSOR@") +set(EdgeLLM_CUDA_ARCHITECTURE "@TRTMC_EDGELLM_CUDA_ARCHITECTURE@") +if(CMAKE_CROSSCOMPILING OR NOT CMAKE_SYSTEM_PROCESSOR STREQUAL EdgeLLM_ARCH) + message(FATAL_ERROR "EdgeLLM is a native-only package for ${EdgeLLM_ARCH}") +endif() +if(NOT CUDAToolkit_VERSION_MAJOR EQUAL @CUDAToolkit_VERSION_MAJOR@ OR + NOT CUDAToolkit_VERSION_MINOR EQUAL @CUDAToolkit_VERSION_MINOR@) + message(FATAL_ERROR "EdgeLLM requires the CUDA SDK it was built with: ${EdgeLLM_CUDA_VERSION}") +endif() +set(EdgeLLM_PYTHON_EXECUTABLE "${EdgeLLM_PREFIX}/libexec/trtmc-edge-llm/bin/python") +set(EdgeLLM_BUILDER_LAUNCHER "${EdgeLLM_PREFIX}/bin/edgellm-builder") +find_path(EdgeLLM_TRT_INCLUDE_DIR NvInfer.h HINTS "$ENV{TRT_ROOT}" "@TRTMC_EDGELLM_TRT_ROOT@" PATH_SUFFIXES include REQUIRED) +find_library(EdgeLLM_TRT_LIBRARY nvinfer HINTS "$ENV{TRT_ROOT}" "@TRTMC_EDGELLM_TRT_ROOT@" PATH_SUFFIXES lib lib64 REQUIRED) +find_library(EdgeLLM_PARSER_LIBRARY nvonnxparser HINTS "$ENV{TRT_ROOT}" "@TRTMC_EDGELLM_TRT_ROOT@" PATH_SUFFIXES lib lib64 REQUIRED) +_edgellm_trt_version("${EdgeLLM_TRT_INCLUDE_DIR}" _edge_current_trt) +if(NOT _edge_current_trt STREQUAL EdgeLLM_TENSORRT_VERSION) + message(FATAL_ERROR "EdgeLLM requires TensorRT ${EdgeLLM_TENSORRT_VERSION}; found ${_edge_current_trt}") +endif() +_edgellm_check_gpu("${EdgeLLM_CUDA_ARCHITECTURE}") +if(NOT TARGET EdgeLLM::Core) + add_library(EdgeLLM::Core STATIC IMPORTED) + set_target_properties(EdgeLLM::Core PROPERTIES + IMPORTED_LOCATION "${EdgeLLM_PREFIX}/lib/libedgellmCore.a" + INTERFACE_INCLUDE_DIRECTORIES "${EdgeLLM_PREFIX}/include;${EdgeLLM_PREFIX}/include/edgellm/cpp;${EdgeLLM_PREFIX}/include/edgellm/3rdParty/nlohmannJson/include;${EdgeLLM_PREFIX}/include/edgellm/3rdParty/stb;${EdgeLLM_PREFIX}/include/edgellm/3rdParty/miniaudio;${EdgeLLM_TRT_INCLUDE_DIR}" + INTERFACE_LINK_LIBRARIES "${EdgeLLM_PREFIX}/lib/libcutedsl.a;${EdgeLLM_TRT_LIBRARY};${EdgeLLM_PARSER_LIBRARY};CUDA::cudart;CUDA::cuda_driver;Threads::Threads;${CMAKE_DL_LIBS}") + add_library(EdgeLLM::Plugin SHARED IMPORTED) + set_target_properties(EdgeLLM::Plugin PROPERTIES IMPORTED_LOCATION "${EdgeLLM_PREFIX}/lib/libNvInfer_edgellm_plugin.so") +endif() diff --git a/cmake/edgellm/Install.cmake.in b/cmake/edgellm/Install.cmake.in new file mode 100644 index 0000000000..72a6263a47 --- /dev/null +++ b/cmake/edgellm/Install.cmake.in @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +file(INSTALL "@_edge_build@/cpp/libedgellmCore.a" DESTINATION "@_edge_prefix@/lib") +file(INSTALL "@_edge_build@/libNvInfer_edgellm_plugin.so" DESTINATION "@_edge_prefix@/lib" FOLLOW_SYMLINK_CHAIN) +file(INSTALL "@_edge_source@/cpp/kernels/cuteDSLArtifact/@CMAKE_SYSTEM_PROCESSOR@/sm_@TRTMC_EDGELLM_CUDA_ARCHITECTURE@/libcutedsl_@CMAKE_SYSTEM_PROCESSOR@.a" + DESTINATION "@_edge_prefix@/lib" RENAME libcutedsl.a) +file(INSTALL "@_edge_source@/cpp" DESTINATION "@_edge_prefix@/include/edgellm" FILES_MATCHING PATTERN "*.h" PATTERN "*.cuh") +foreach(_third_party IN ITEMS nlohmannJson stb miniaudio) + file(INSTALL "@_edge_source@/3rdParty/${_third_party}" DESTINATION "@_edge_prefix@/include/edgellm/3rdParty" + FILES_MATCHING PATTERN "*.h" PATTERN "*.hpp") +endforeach() +file(MAKE_DIRECTORY "@_edge_prefix@/bin" "@_edge_prefix@/share/trtmc") +file(WRITE "@_edge_prefix@/bin/edgellm-builder" [=[#!/bin/sh +set -eu +prefix=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +exec "$prefix/libexec/trtmc-edge-llm/bin/python" -I -c 'from experimental.builder.cli import main; main()' "$@" +]=]) +file(CHMOD "@_edge_prefix@/bin/edgellm-builder" PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) +execute_process(COMMAND "@_edge_python@" -I -c "import tensorrt; print(tensorrt.__version__)" + OUTPUT_VARIABLE _trt_version OUTPUT_STRIP_TRAILING_WHITESPACE COMMAND_ERROR_IS_FATAL ANY) +# Paths are relative to the installation prefix, preserving relocatability. +file(WRITE "@_edge_prefix@/share/trtmc/edge-llm.json" "{\n \"schema_version\": 1,\n \"version\": \"@_edge_version@\",\n \"revision\": \"@_edge_revision@\",\n \"arch\": \"@CMAKE_SYSTEM_PROCESSOR@\",\n \"architectures\": [@TRTMC_EDGELLM_CUDA_ARCHITECTURE@],\n \"cuda_version\": \"@CUDAToolkit_VERSION_MAJOR@.@CUDAToolkit_VERSION_MINOR@\",\n \"tensorrt_version\": \"${_trt_version}\",\n \"python\": \"libexec/trtmc-edge-llm/bin/python\",\n \"builder\": \"bin/edgellm-builder\",\n \"plugin\": \"lib/libNvInfer_edgellm_plugin.so\"\n}\n") diff --git a/cmake/edgellm/Prepare.cmake.in b/cmake/edgellm/Prepare.cmake.in new file mode 100644 index 0000000000..c5d00e5495 --- /dev/null +++ b/cmake/edgellm/Prepare.cmake.in @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# Executed only by the explicit CMake dependency build, never by model dispatch. +function(run) + execute_process(COMMAND ${ARGV} COMMAND_ERROR_IS_FATAL ANY) +endfunction() +execute_process(COMMAND "@Python3_EXECUTABLE@" -I -c "import ensurepip" + RESULT_VARIABLE _has_ensurepip OUTPUT_QUIET ERROR_QUIET) +if(_has_ensurepip EQUAL 0) + run("@Python3_EXECUTABLE@" -I -m venv --copies "@_edge_prefix@/libexec/trtmc-edge-llm") +else() + # Debian minimal Python may omit ensurepip; use an already installed bootstrapper. + run("@Python3_EXECUTABLE@" -I -m virtualenv --copies --no-download --no-periodic-update + "@_edge_prefix@/libexec/trtmc-edge-llm") +endif() +set(_pip_options --isolated install --no-user) +if(NOT "@TRTMC_EDGELLM_WHEELHOUSE@" STREQUAL "") + list(APPEND _pip_options --no-index --find-links "@TRTMC_EDGELLM_WHEELHOUSE@") +endif() +file(GLOB _trt_wheels "@TRTMC_EDGELLM_TRT_ROOT@/python/tensorrt-*-cp@Python3_VERSION_MAJOR@@Python3_VERSION_MINOR@-none-linux_@CMAKE_SYSTEM_PROCESSOR@.whl") +list(LENGTH _trt_wheels _wheel_count) +if(NOT _wheel_count EQUAL 1) + message(FATAL_ERROR "Expected exactly one TensorRT SDK wheel matching the native Python ABI") +endif() +run("@_edge_python@" -I -m pip ${_pip_options} --report "@_edge_prefix@/pip-report.json" + ${_trt_wheels} numpy==2.2.6 transformers==5.14.1 jinja2==3.1.6 + scikit-build-core==0.11.6 wheel==0.45.1 cmake==3.31.10 ninja==1.13.0 + "cuda-python>=@CUDAToolkit_VERSION_MAJOR@.@CUDAToolkit_VERSION_MINOR@,<@CUDAToolkit_VERSION_MAJOR@.@CUDAToolkit_VERSION_MINOR@.999" + "nvidia-cutlass-dsl[cu@CUDAToolkit_VERSION_MAJOR@]==4.7.0" "cupy-cuda@CUDAToolkit_VERSION_MAJOR@x==13.6.0") +run("@_edge_python@" -I -m pip ${_pip_options} --no-deps --no-build-isolation "@_edge_source@") +run("@_edge_python@" -I -m pip --isolated check) +run("@_edge_python@" -I -c "print(__import__('tensorrt').__version__)") +set(ENV{PATH} "@CUDAToolkit_BIN_DIR@:$ENV{PATH}") +run("@_edge_python@" -I "@_edge_source@/kernelSrcs/build_cutedsl.py" + --gpu_arch "sm_@TRTMC_EDGELLM_CUDA_ARCHITECTURE@" --arch "@CMAKE_SYSTEM_PROCESSOR@" + --kernels fmha,gdn --cuda-version "@CUDAToolkit_VERSION_MAJOR@.@CUDAToolkit_VERSION_MINOR@" --jobs "@TRTMC_EDGELLM_JOBS@") diff --git a/cmake/edgellm/README.md b/cmake/edgellm/README.md new file mode 100644 index 0000000000..140f9a9b5f --- /dev/null +++ b/cmake/edgellm/README.md @@ -0,0 +1,56 @@ +# Pinned native Edge-LLM package + +Edge-LLM is optional. The default `TRTMC_ENABLE_EDGELLM=OFF` neither downloads +nor builds it. Enable it once while installing Model Connect; ordinary model +builds only use the installed package and never fetch or install dependencies. +Cross compilation is rejected. Configure and build on the inference GPU host. + +```bash +cmake -S . -B build \ + -DTRTMC_ENABLE_EDGELLM=ON \ + -DTRTMC_EDGELLM_CUDA_ARCHITECTURE=80 \ + -DTRTMC_EDGELLM_TRT_ROOT="$TRT_ROOT" \ + -DCUDAToolkit_ROOT="$CUDA_ROOT" \ + -DCMAKE_CUDA_COMPILER="$CUDA_ROOT/bin/nvcc" \ + -DCMAKE_INSTALL_PREFIX="$PWD/install" +cmake --build build --parallel 8 +cmake --install build +export CMAKE_PREFIX_PATH="$PWD/install${CMAKE_PREFIX_PATH:+:$CMAKE_PREFIX_PATH}" +``` + +The regular project dependencies remain required, including nlohmann_json +**3.12.0** when Edge is enabled (its C++ ABI must match upstream). The Python +interpreter needs `ensurepip` or an already installed `virtualenv` bootstrapper. +The native CUDA SDK must include NVCC, NVRTC, cuRAND headers and driver link +libraries; the TensorRT SDK must contain its matching CPython wheel. + +The provider first uses `find_package(EdgeLLM 0.10.1 EXACT CONFIG)`. If absent, +CMake `ExternalProject` clones the public NVIDIA TensorRT-Edge-LLM repository at +`e8b29522938901f6df19ebeedd4b69bc8edbcd97` (v0.10.1), initializes the pinned +submodules, builds the native core/plugin and FMHA/GDN CuTe archives, and installs +an isolated direct-builder Python environment. It does not install the exporter +or modify the caller Python environment. Downloads happen only during this +explicit dependency build. `TRTMC_EDGELLM_WHEELHOUSE` selects a complete offline +Python wheelhouse; `TRTMC_EDGELLM_GIT_MIRROR` optionally supplies a local Git +mirror, still checked out at the immutable upstream commit. + +Upstream 0.10.1 does not export a CMake SDK package, so these compact templates +supply that installation boundary. `EdgeLLM::Core` exposes the installed static +core, headers, CuTe archive and native dependencies. Consumers requiring CUDA +device linking enable separable compilation and device-symbol resolution. +`EdgeLLM::Plugin` identifies the plugin DSO; adapters load it, rather than linking +it twice. `EdgeLLM_PYTHON_EXECUTABLE` and `EdgeLLM_BUILDER_LAUNCHER` expose the +isolated upstream `experimental.builder.cli.main` API. + +`share/trtmc/edge-llm.json` records the pin, native architecture, CUDA/TensorRT +versions and prefix-relative Python/plugin paths. The manifest is written only +after successful installation. Build-tree package files live under +`build/_deps/edgellm/install`; `cmake --install` copies the package into the final +prefix. Package discovery rejects mismatched native CPU/GPU and SDK versions. +Model support and routing policies belong exclusively to the model families. + +GPU-independent package contract tests: + +```bash +cmake -P cmake/edgellm/tests/package_contract.cmake +``` diff --git a/cmake/edgellm/tests/package_contract.cmake b/cmake/edgellm/tests/package_contract.cmake new file mode 100644 index 0000000000..7b691e0e5a --- /dev/null +++ b/cmake/edgellm/tests/package_contract.cmake @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# Run without a GPU or CUDA SDK: cmake -P cmake/edgellm/tests/package_contract.cmake +cmake_minimum_required(VERSION 3.20) +get_filename_component(_templates "${CMAKE_CURRENT_LIST_DIR}/.." ABSOLUTE) +get_filename_component(_module "${_templates}/../EdgeLLM.cmake" ABSOLUTE) +if(NOT DEFINED TEST_ROOT) + set(TEST_ROOT "/tmp/trtmc-edgellm-cmake-contract") +endif() +file(REMOVE_RECURSE "${TEST_ROOT}") +file(MAKE_DIRECTORY "${TEST_ROOT}/package/lib/cmake/EdgeLLM" "${TEST_ROOT}/sdk/include" + "${TEST_ROOT}/sdk/lib" "${TEST_ROOT}/modules" "${TEST_ROOT}/bin" "${TEST_ROOT}/source" + "${TEST_ROOT}/package/include/edgellm/cpp" "${TEST_ROOT}/package/include/edgellm/3rdParty/nlohmannJson/include" + "${TEST_ROOT}/package/include/edgellm/3rdParty/stb" "${TEST_ROOT}/package/include/edgellm/3rdParty/miniaudio") +set(_edge_version "0.10.1") +set(_edge_revision "e8b29522938901f6df19ebeedd4b69bc8edbcd97") +set(_edge_trt_version "11.1.0.106") +set(CUDAToolkit_VERSION "13.3.73") +set(CUDAToolkit_VERSION_MAJOR 13) +set(CUDAToolkit_VERSION_MINOR 3) +cmake_host_system_information(RESULT CMAKE_SYSTEM_PROCESSOR QUERY OS_PLATFORM) +set(TRTMC_EDGELLM_CUDA_ARCHITECTURE 80) +set(TRTMC_EDGELLM_TRT_ROOT "${TEST_ROOT}/sdk") +configure_file("${_templates}/EdgeLLMConfig.cmake.in" "${TEST_ROOT}/package/lib/cmake/EdgeLLM/EdgeLLMConfig.cmake" @ONLY) +configure_file("${_templates}/CheckNative.cmake" "${TEST_ROOT}/package/lib/cmake/EdgeLLM/CheckNative.cmake" COPYONLY) +include(CMakePackageConfigHelpers) +write_basic_package_version_file("${TEST_ROOT}/package/lib/cmake/EdgeLLM/EdgeLLMConfigVersion.cmake" + VERSION "${_edge_version}" COMPATIBILITY ExactVersion ARCH_INDEPENDENT) +file(WRITE "${TEST_ROOT}/sdk/include/NvInfer.h" "// test fixture\n") +file(WRITE "${TEST_ROOT}/sdk/include/NvInferVersion.h" "#define ENTERPRISE_MAJOR 11\n#define NV_TENSORRT_MAJOR ENTERPRISE_MAJOR\n#define NV_TENSORRT_MINOR 1\n#define NV_TENSORRT_PATCH 0\n#define NV_TENSORRT_BUILD 106\n") +foreach(_library IN ITEMS nvinfer nvonnxparser) + file(WRITE "${TEST_ROOT}/sdk/lib/lib${_library}.so" "fixture") +endforeach() +file(WRITE "${TEST_ROOT}/modules/FindCUDAToolkit.cmake" [=[ +set(CUDAToolkit_FOUND TRUE) +set(CUDAToolkit_VERSION_MAJOR "${TEST_CUDA_MAJOR}") +set(CUDAToolkit_VERSION_MINOR 3) +add_library(CUDA::cudart INTERFACE IMPORTED) +add_library(CUDA::cuda_driver INTERFACE IMPORTED) +]=]) +file(WRITE "${TEST_ROOT}/modules/FindThreads.cmake" "set(Threads_FOUND TRUE)\nadd_library(Threads::Threads INTERFACE IMPORTED)\n") +file(WRITE "${TEST_ROOT}/modules/Findnlohmann_json.cmake" "set(nlohmann_json_FOUND TRUE)\n") +file(WRITE "${TEST_ROOT}/bin/nvidia-smi" [=[#!/bin/sh +echo "$TEST_GPU_CAP" +]=]) +file(CHMOD "${TEST_ROOT}/bin/nvidia-smi" PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE) +set(ENV{PATH} "${TEST_ROOT}/bin:$ENV{PATH}") +set(ENV{TRT_ROOT} "${TEST_ROOT}/sdk") +set(ENV{TEST_GPU_CAP} "8.0") +file(WRITE "${TEST_ROOT}/source/CMakeLists.txt" "cmake_minimum_required(VERSION 3.20)\nproject(package_contract LANGUAGES CXX)\nset(_version unrelated_caller_value)\ninclude(GNUInstallDirs)\nlist(PREPEND CMAKE_MODULE_PATH \"${TEST_ROOT}/modules\")\nset(EdgeLLM_DIR \"${TEST_ROOT}/package/lib/cmake/EdgeLLM\")\nif(TEST_OFF)\n set(TRTMC_ENABLE_EDGELLM OFF)\n include(\"${_module}\")\n if(TARGET EdgeLLM::Core OR TARGET trtmc_edgellm_dependency)\n message(FATAL_ERROR \"Optional-off configuration acquired Edge\")\n endif()\nelse()\n set(TRTMC_ENABLE_EDGELLM ON)\n include(\"${_module}\")\n if(NOT TARGET EdgeLLM::Core OR TARGET trtmc_edgellm_dependency)\n message(FATAL_ERROR \"Installed package was not reused\")\n endif()\nendif()\n") +# Each negative case must fail for its intended reason, not an incidental fixture error. +function(check name expected) + execute_process(COMMAND "${CMAKE_COMMAND}" -S "${TEST_ROOT}/source" -B "${TEST_ROOT}/${name}" + -DTEST_CUDA_MAJOR=13 ${ARGN} RESULT_VARIABLE _status OUTPUT_VARIABLE _stdout ERROR_VARIABLE _stderr) + file(WRITE "${TEST_ROOT}/${name}.log" "${_stdout}${_stderr}") + if(expected STREQUAL "PASS") + if(NOT _status EQUAL 0) + message(FATAL_ERROR "${name} failed: ${_stdout}${_stderr}") + endif() + elseif(_status EQUAL 0 OR NOT "${_stdout}${_stderr}" MATCHES "${expected}") + message(FATAL_ERROR "${name} did not reject with : ${_stdout}${_stderr}") + endif() + message(STATUS "PASS ${name}") +endfunction() +check(optional_off PASS -DTEST_OFF=ON) +check(installed_native PASS) +check(wrong_cuda "requires the CUDA SDK" -DTEST_CUDA_MAJOR=12) +check(cross_compile "cross compilation is not supported" -DCMAKE_SYSTEM_NAME=Linux) +set(ENV{TEST_GPU_CAP} "9.0") +check(wrong_gpu "requires a local GPU") +set(ENV{TEST_GPU_CAP} "8.0") +file(APPEND "${TEST_ROOT}/sdk/include/NvInferVersion.h" "#undef NV_TENSORRT_BUILD\n#define NV_TENSORRT_BUILD 107\n") +check(wrong_trt_build "requires TensorRT 11.1.0.106") +file(APPEND "${TEST_ROOT}/sdk/include/NvInferVersion.h" "#undef NV_TENSORRT_BUILD\n#define NV_TENSORRT_BUILD 106\n") +file(APPEND "${TEST_ROOT}/package/lib/cmake/EdgeLLM/EdgeLLMConfig.cmake" "set(EdgeLLM_REVISION wrong)\n") +check(wrong_revision "does not match the pinned GitHub revision") diff --git a/core/builder/tensorrt_model_connect/build.py b/core/builder/tensorrt_model_connect/build.py index d0d5d68638..eb265a59ce 100644 --- a/core/builder/tensorrt_model_connect/build.py +++ b/core/builder/tensorrt_model_connect/build.py @@ -7,6 +7,8 @@ import hashlib import importlib +import os +import platform import re import sys from dataclasses import dataclass @@ -72,6 +74,42 @@ def __post_init__(self) -> None: raise ValueError("graph_transform must be callable when provided") +def cmake_prefixes() -> list[Path]: + """Return explicit standard CMake prefixes followed by the Python prefix.""" + prefixes = [Path(value) for value in os.environ.get("CMAKE_PREFIX_PATH", "").split(os.pathsep) if value] + return [*prefixes, Path(sys.prefix)] + + +def detect_local_platform() -> dict: + """Return executing GPU and native SDK identity without selecting a model. + + Returns: + OS/release, CPU architecture, GPU SM, CUDA and TensorRT versions. + + Raises: + ImportError: Native SDK Python bindings are unavailable. + RuntimeError: CUDA cannot identify the executing device. + """ + import tensorrt as trt + from cuda.bindings import runtime + + def checked(result): + if int(result[0]) != 0: + raise RuntimeError(f"CUDA device discovery failed: {result[0]}") + return result[1] + + device = checked(runtime.cudaGetDevice()) + gpu = checked(runtime.cudaGetDeviceProperties(device)) + cuda = checked(runtime.cudaRuntimeGetVersion()) + release = platform.freedesktop_os_release() if sys.platform == "linux" else {} + return { + "os": sys.platform, "os_version": release.get("VERSION_ID", platform.release()), + "arch": platform.machine(), "sm": gpu.major * 10 + gpu.minor, + "cuda_version": f"{cuda // 1000}.{cuda % 1000 // 10}", + "tensorrt_version": trt.__version__, + } + + def _validate_id(field: str, value: object) -> str: if not isinstance(value, str) or _ID.fullmatch(value) is None: raise ValueError( diff --git a/core/builder/tests/test_build.py b/core/builder/tests/test_build.py index 5eff69de1b..cbbb7ae26e 100644 --- a/core/builder/tests/test_build.py +++ b/core/builder/tests/test_build.py @@ -287,3 +287,59 @@ def abort(self) -> None: with pytest.raises(OSError, match="publish failed"): build_core.build(_request(tmp_path)) assert events == ["finish", "abort"] + + +@pytest.mark.parametrize("value", ["", "/one", "/one:/two", ":/one::/two:"]) +def test_cmake_prefixes_preserve_standard_search_order(monkeypatch, value): + monkeypatch.setenv("CMAKE_PREFIX_PATH", value) + monkeypatch.setattr(build_core.sys, "prefix", "/python") + expected = [Path(item) for item in value.split(build_core.os.pathsep) if item] + assert build_core.cmake_prefixes() == [*expected, Path("/python")] + + +def test_cmake_prefixes_without_environment_use_python_prefix(monkeypatch): + monkeypatch.delenv("CMAKE_PREFIX_PATH", raising=False) + monkeypatch.setattr(build_core.sys, "prefix", "/python") + assert build_core.cmake_prefixes() == [Path("/python")] + + +@pytest.fixture +def native_platform_bindings(monkeypatch): + from unittest.mock import Mock + + runtime = SimpleNamespace( + cudaGetDevice=Mock(return_value=(0, 3)), + cudaGetDeviceProperties=Mock(return_value=(0, SimpleNamespace(major=8, minor=6))), + cudaRuntimeGetVersion=Mock(return_value=(0, 13030)), + ) + monkeypatch.setitem(sys.modules, "tensorrt", SimpleNamespace(__version__="11.1.0.106")) + monkeypatch.setitem(sys.modules, "cuda.bindings", SimpleNamespace(runtime=runtime)) + monkeypatch.setattr(build_core.sys, "platform", "linux") + monkeypatch.setattr(build_core.platform, "machine", lambda: "x86_64") + monkeypatch.setattr(build_core.platform, "freedesktop_os_release", lambda: {"VERSION_ID": "24.04"}) + monkeypatch.setattr(build_core.platform, "release", lambda: "fallback-release") + return runtime + + +def test_native_platform_uses_executing_cuda_device_and_full_sdk(native_platform_bindings): + assert build_core.detect_local_platform() == { + "os": "linux", "os_version": "24.04", "arch": "x86_64", "sm": 86, + "cuda_version": "13.3", "tensorrt_version": "11.1.0.106", + } + native_platform_bindings.cudaGetDevice.assert_called_once_with() + native_platform_bindings.cudaGetDeviceProperties.assert_called_once_with(3) + native_platform_bindings.cudaRuntimeGetVersion.assert_called_once_with() + + +@pytest.mark.parametrize("failing", ["cudaGetDevice", "cudaGetDeviceProperties", "cudaRuntimeGetVersion"]) +def test_native_platform_propagates_cuda_discovery_failure(native_platform_bindings, failing): + getattr(native_platform_bindings, failing).return_value = (35,) + with pytest.raises(RuntimeError, match="CUDA device discovery failed: 35"): + build_core.detect_local_platform() + + +def test_native_platform_retains_nonlinux_identity(native_platform_bindings, monkeypatch): + monkeypatch.setattr(build_core.sys, "platform", "win32") + result = build_core.detect_local_platform() + assert result["os"] == "win32" + assert result["os_version"] == "fallback-release" diff --git a/core/runtime/bundle/bundle_format.cpp b/core/runtime/bundle/bundle_format.cpp index eb11f20674..2c1661f088 100644 --- a/core/runtime/bundle/bundle_format.cpp +++ b/core/runtime/bundle/bundle_format.cpp @@ -6,6 +6,7 @@ #include "runtime/bundle/bundle_format.h" #include +#include #include #include #include @@ -215,6 +216,32 @@ std::vector BundleReader::read_section(std::string_view name) const { return data; } +void BundleReader::copy_section(std::string_view name, std::ostream& output) const { + const auto* section = find_section(name); + if (section == nullptr) + throw std::runtime_error("Bundle section not found: " + std::string(name)); + const auto offset = checked_section_file_offset(*section, data_offset_, file_size_, path_); + if (offset > static_cast(std::numeric_limits::max())) + throw std::runtime_error("Bundle section has an unsupported file offset: " + path_); + std::ifstream input(path_, std::ios::binary); + input.seekg(static_cast(offset)); + if (!input || !output) + throw std::runtime_error("Cannot copy bundle section: " + std::string(name)); + std::array buffer; + auto remaining = section->length; + while (remaining != 0) { + const auto count = + static_cast(std::min(remaining, buffer.size())); + input.read(buffer.data(), count); + if (!input) + throw std::runtime_error("Failed reading bundle section: " + std::string(name)); + output.write(buffer.data(), count); + if (!output) + throw std::runtime_error("Failed writing bundle section: " + std::string(name)); + remaining -= static_cast(count); + } +} + BundleInfo InspectBundle(const std::string& bundle_path) { return BundleReader(bundle_path).info(); } diff --git a/core/runtime/include/trtmc/bundle.h b/core/runtime/include/trtmc/bundle.h index 154d697ef1..2b32776d2e 100644 --- a/core/runtime/include/trtmc/bundle.h +++ b/core/runtime/include/trtmc/bundle.h @@ -6,6 +6,7 @@ #pragma once #include +#include #include #include #include @@ -44,6 +45,12 @@ class BundleReader { const BundleSectionInfo* find_section(std::string_view name) const noexcept; std::vector read_section(std::string_view name) const; + /// Copy a named section to an output stream using bounded working memory. + /// @param name Validated bundle section name. + /// @param output Caller-owned stream; may contain partial data on failure. + /// @throws std::runtime_error If the section is absent or input/output fails. + void copy_section(std::string_view name, std::ostream& output) const; + private: std::string path_; BundleInfo info_; diff --git a/core/runtime/tests/test_bundle_format_v1.cpp b/core/runtime/tests/test_bundle_format_v1.cpp index 0da636b49b..6c5faa420f 100644 --- a/core/runtime/tests/test_bundle_format_v1.cpp +++ b/core/runtime/tests/test_bundle_format_v1.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -52,6 +53,36 @@ bool read_throws(const std::filesystem::path& path) { } } +/// Verify bounded-chunk section boundaries, empty sections and late I/O failures. +void test_copy_section(const std::filesystem::path& directory) { + const auto path = directory / "stream.bundle"; + const std::string payload(2 * 64 * 1024 + 17, 'x'); + const std::string header = + R"({"format":1,"family":"fake","task":"text","backend":"fake","sections":{"data":{"offset":3,"length":)" + + std::to_string(payload.size()) + R"(},"empty":{"offset":0,"length":0}}})"; + write_bundle(path, header, "PRE" + payload + "POST"); + const trtmc::BundleReader reader(path.string()); + std::ostringstream output; + reader.copy_section("data", output); + check(output.str() == payload, "stream copy preserves boundaries across chunks"); + reader.copy_section("empty", output); + check(output.str() == payload, "empty section appends nothing"); + auto fails = [&](const char* section, std::ostream& destination) { + try { + reader.copy_section(section, destination); + return false; + } catch (const std::runtime_error&) { + return true; + } + }; + check(fails("missing", output), "stream copy rejects missing sections"); + std::ostringstream broken; + broken.setstate(std::ios::badbit); + check(fails("data", broken), "stream copy reports output failure"); + std::filesystem::resize_file(path, 16 + header.size() + 3 + payload.size() - 1); + check(fails("data", output), "stream copy detects truncation after validation"); +} + } // namespace int main() { @@ -103,6 +134,7 @@ int main() { "PLAN"); check(read_throws(out_of_bounds), "out of bounds section rejected"); + test_copy_section(directory); std::filesystem::remove_all(directory); std::cerr << (failures == 0 ? "ALL PASSED\n" : "SOME FAILED\n"); return failures; diff --git a/families/qwen3_5/EDGE_LLM.md b/families/qwen3_5/EDGE_LLM.md new file mode 100644 index 0000000000..f9734d54de --- /dev/null +++ b/families/qwen3_5/EDGE_LLM.md @@ -0,0 +1,98 @@ +# Qwen3.5 Edge-LLM adapter + +This family owns its dispatch map, builder argument mapping, runtime adapter, +and tests. Qwen3 and older retain their original native implementation. + +## Install once, dispatch automatically + +Provision the optional, pinned native SDK using the +[CMake dependency instructions](../../cmake/edgellm/README.md). Cross compilation +is unsupported. Normal model build and inference never clone or install Edge. + +Set `CMAKE_PREFIX_PATH` to that installation, then use the ordinary commands: + +```bash +export CMAKE_PREFIX_PATH=/path/to/edge-install +python -m tensorrt_model_connect build /path/to/Qwen3.5-0.8B \ + --precision fp16 --max-sequence-length 1024 -o qwen35.bundle +/path/to/model-connect-build/trtmc run qwen35.bundle \ + --runtime-root /path/to/model-connect-build \ + --prompt "What is the capital of France? Answer in one word." \ + --max-new-tokens 10 --temperature 0 --top-k 1 \ + --use-chat-template true --enable-thinking false +``` + +The runtime build must also have `TRTMC_ENABLE_EDGELLM=ON`. The package pin is +Edge-LLM v0.10.1, commit `e8b29522938901f6df19ebeedd4b69bc8edbcd97`. +The bundle contains `edge_llm.json` and complete upstream engine/checkpoint +assets, rather than a native-family engine. An Edge-marked bundle requires an +Edge-enabled runtime on its recorded native platform and SDK. + +## Selection and API boundaries + +```mermaid +flowchart TD + A[Resolve qwen3_5 family] --> B{Supported model and request profile?} + B -->|No| N[Original native builder] + B -->|Yes| C[Detect executing GPU and SDK] + C --> D{Family platform map matches?} + D -->|No| N + D -->|Yes| E[Resolve installed pinned package] + E --> F[Invoke Edge builder main in isolated staging] + F --> G[Validate complete upstream artifacts] + G --> P[Publish Edge bundle] + C -->|Discovery error| W[Warn and retain diagnostics] + E -->|Dependency error| W + F -->|Builder error| W + G -->|Incomplete artifacts| W + W --> N + P --> R[Family runtime calls persistent Edge inference APIs] +``` + +- `dispatch.py` owns the exact platform map and model/request admission checks. + The initial profile is dense, unquantized FP16 text generation, batch/TP/CP 1, + with the pinned builder's 128-dimensional GDN state contract. Unsupported + quantization, graph transforms, explicit FP32 layers, dynamic KV or multimodal + requests remain native. Actual configuration, not a marketing-name comparison, + distinguishes this family from Qwen3.8. +- `edge_llm.py` calls `experimental.builder.cli.main` in the CMake-installed + isolated Python environment. It maps model/output directories, sequence + capacity, batch size and FP16 precision. Edge owns conversion, topology, + TensorRT engine building, tokenizer/template processing and external weights. +- Ordinary preparation failures warn before one native retry with the **same + request**. Diagnostics stay beside the requested output as `.NAME.edge-*.log`. + If native also fails, its error retains the Edge cause. Cancellation and bundle + publication failures propagate without retry. +- `runtime/edge_llm` directly owns one `LLMInferenceRuntime`, maps requests to + `countPromptTokens` and `handleRequest`, and returns Edge text/token IDs. + It supports token limits, temperature, top-k/top-p and chat/thinking controls; + unsupported non-default controls are rejected, not silently ignored. + **Inference errors never switch to native.** + +## Validation scope + +The real Qwen/Qwen3.5-0.8B checkpoint has built and generated `Paris` through +ordinary Model Connect dispatch on A30 SM80, Ubuntu 24.04 x86_64, CUDA 13.3 and +TensorRT 11.1.0.106. This is not qualification evidence for every platform in +the candidate map. Persistent Model Connect requests match direct Edge API text +and greedy token IDs for raw prompts, chat and EOS termination, including repeat +requests after unsupported controls and capacity overruns are rejected. To +repeat this check with the compiled family validation drivers: + +```bash +python -m families.qwen3_5.tests.edge_validation \ + --bundle qwen35.bundle --runtime-root /path/to/model-connect-build \ + --output /tmp/qwen35-edge-validation-new +``` + +The output directory must not already exist. The harness rejects native bundles +and retains commands, logs and JSON evidence. Family CPU tests cover admission, +API argument mapping, failure isolation, bundle publication and runtime contracts. + +Independent CPU FP32 Hugging Face output also passes this family's existing +normalized-text quality criterion (maximum edit distance 0.15), without changing +its assertions. Add `--hf-checkpoint /path/to/Qwen3.5-0.8B` to run that reference; +this optional validation requires PyTorch and the model's Transformers support +in the validation interpreter, not in the production Edge builder environment. +Exact token-ID parity is required against direct Edge, not across different HF +and TensorRT numerical implementations. diff --git a/families/qwen3_5/dispatch.py b/families/qwen3_5/dispatch.py new file mode 100644 index 0000000000..7fe9ca55ac --- /dev/null +++ b/families/qwen3_5/dispatch.py @@ -0,0 +1,109 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Family-owned complete-network route map; native is the default.""" + +from __future__ import annotations + +import json +import logging +import os +from pathlib import Path +import tempfile +import traceback + +from . import edge_llm + +_LOG = logging.getLogger(__name__) + +# Explicit native platform routes supported by pinned Edge. No version-name +# comparison, nearest-GPU matching, shared registry, or legacy-Qwen entries. +EDGE_DISPATCH = { + ("linux", architecture, sm, "fp16"): edge_llm.prepare + for architecture, sms in ( + ("x86_64", (80, 86, 100, 120)), + ("aarch64", (87, 110, 121)), + ) + for sm in sms +} + + +def candidate(request, raw: dict) -> bool: + """Return whether this family's model/request contract can delegate to Edge.""" + config = raw.get("text_config", raw) + return ( + isinstance(config, dict) + and raw.get("model_type") == "qwen3_5" + and ("output_gate_type" not in config or "mlp_only_layers" in config) + and config.get("linear_key_head_dim") == config.get("linear_value_head_dim") == 128 + and not config.get("num_experts") + and not raw.get("quantization_config") and not config.get("quantization_config") + and not any( + (Path(request.model_dir) / name).exists() + for name in ("hf_quant_config.json", "quantize_config.json", "quant_config.json") + ) + and request.backend == "trt" and request.task == "text_generation" + and request.precision.lower() == "fp16" + and request.quantization in {None, "none"} + and request.max_batch_size == request.tensor_parallel_size == request.context_parallel_size == 1 + and not request.dynamic_kv_cache and not request.fp32_layers and request.graph_transform is None + and all(value is None for value in (request.image_height, request.image_width, request.video_num_frames)) + ) + + +def build(request, writer, native) -> None: + """Dispatch locally or warn and retry native once with the original request. + + Args: + request: Unmodified Model Connect build request. + writer: Unpublished bundle writer. + native: This family's original native builder callback. + + Raises: + Exception: Common input/publication error, or native build error with + Edge cause after a failed preparation. Cancellation never retries. + """ + raw = json.loads((Path(request.model_dir) / "config.json").read_text(encoding="utf-8")) + if not isinstance(raw, dict): + raise ValueError("checkpoint config.json must contain an object") + config = raw.get("text_config", raw) + if not isinstance(config, dict): + raise ValueError("checkpoint text_config must contain an object") + if not candidate(request, raw): + native(request, writer) + return + capacity = config.get("max_position_embeddings") + if type(capacity) is not int or capacity <= 0: + raise ValueError("checkpoint max_position_embeddings must be a positive integer") + if request.max_sequence_length and request.max_sequence_length > capacity: + raise ValueError("max_sequence_length exceeds checkpoint context capacity") + failure = None + descriptor, name = tempfile.mkstemp(prefix=f".{request.output_path.name}.edge-", suffix=".log", + dir=request.output_path.parent) + os.close(descriptor) + log_path = Path(name) + with tempfile.TemporaryDirectory(prefix="trtmc-qwen3_5-edge-") as directory: + try: + target = edge_llm.local_target() + key = (target["os"], target["arch"], target["sm"], request.precision.lower()) + adapter = EDGE_DISPATCH.get(key) + if adapter is not None: + files, marker = adapter(request, raw, target, Path(directory), log_path) + except Exception as error: + failure = error + with log_path.open("a", encoding="utf-8") as log: + traceback.print_exception(error, file=log) + _LOG.warning("qwen3_5 Edge build failed: %s. Diagnostics: %s. " + "Retrying native once with the unchanged request.", error, log_path, exc_info=True) + else: + # Edge preparation did not touch writer; publication cannot fallback. + if adapter is not None: + edge_llm.publish(request, writer, files, marker) + return + log_path.unlink() # A platform non-match is not an Edge failure. + try: + native(request, writer) + except Exception as error: + if failure is not None: + raise error from failure + raise diff --git a/families/qwen3_5/edge_llm.py b/families/qwen3_5/edge_llm.py new file mode 100644 index 0000000000..6490c4725e --- /dev/null +++ b/families/qwen3_5/edge_llm.py @@ -0,0 +1,123 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Thin family-owned adapter to the pinned Edge direct-builder API.""" + +from __future__ import annotations + +import json +from pathlib import Path +import shutil +import subprocess + +from tensorrt_model_connect.build import cmake_prefixes, detect_local_platform + +EDGE_REVISION = "e8b29522938901f6df19ebeedd4b69bc8edbcd97" + + +def local_target() -> dict: + """Return the executing worker identity supplied by generic build mechanics.""" + return detect_local_platform() + + +def installed_package(target: dict) -> dict: + """Resolve CMake installation via standard prefixes; never install anything. + + Args: + target: Executing device and SDK identity. + + Returns: + Validated package metadata with absolute Python and plugin paths. + + Raises: + FileNotFoundError: No CMake installation or required artifact exists. + ValueError: Pin, architecture, SDK or contained-path contract differs. + """ + for prefix in cmake_prefixes(): + manifest = prefix / "share/trtmc/edge-llm.json" + if not manifest.is_file(): + continue + package = json.loads(manifest.read_text(encoding="utf-8")) + if package.get("schema_version") != 1 or package.get("revision") != EDGE_REVISION: + raise ValueError(f"Edge package has an unsupported revision/schema: {manifest}") + if package.get("version") != "0.10.1" or package.get("arch") != target["arch"]: + raise ValueError("Edge package version/architecture differs from executing worker") + if target["sm"] not in package.get("architectures", []): + raise ValueError("Edge package was not built for this local GPU") + cuda_version = ".".join(str(package.get("cuda_version", "")).split(".")[:2]) + if cuda_version != target["cuda_version"] or package.get("tensorrt_version") != target["tensorrt_version"]: + raise ValueError("Edge package CUDA/TensorRT differs from executing worker") + for name in ("python", "plugin"): + relative = Path(package[name]) + path = (prefix / relative).resolve() + if relative.is_absolute() or not path.is_relative_to(prefix.resolve()): + raise ValueError(f"Edge package {name} must be contained in its installation") + if not path.is_file(): + raise FileNotFoundError(f"Edge package {name} is missing: {path}") + package[name] = str(path) + return package + raise FileNotFoundError("Edge-LLM is not installed; enable the optional Edge-LLM CMake dependency " + "and set CMAKE_PREFIX_PATH to its install prefix") + + +def prepare(request, raw: dict, target: dict, staging: Path, log_path: Path) -> tuple[dict, dict]: + """Map the request to Edge main(argv), returning complete unpublished assets. + + Edge owns model selection, configuration, conversion, graphs and engine + composition. The family adapter only maps text-generation arguments and + preserves the checkpoint needed by Edge external-weight APIs. + + Returns: + (section-name to file mapping, runtime marker). + + Raises: + Exception: Dependency, upstream build or artifact validation failed. + """ + package = installed_package(target) + checkpoint = staging / "edge_llm/checkpoint" + checkpoint.mkdir(parents=True) + for source in Path(request.model_dir).iterdir(): + if source.is_file() and (source.suffix in {".json", ".safetensors", ".model", ".jinja"} + or source.name in {"merges.txt", "vocab.txt"}): + shutil.copy2(source, checkpoint / source.name) + if not list(checkpoint.glob("*.safetensors")): + raise ValueError("Edge direct builder requires a safetensors checkpoint") + config = raw.get("text_config", raw) + limit = request.max_sequence_length or min(int(config["max_position_embeddings"]), 256) + engine = staging / "edge_llm/engine" + # Calling upstream main preserves its complete build/artifact orchestration. + command = [package["python"], "-I", "-c", + "from experimental.builder.cli import main; main()", + "--model-dir", str(checkpoint), "--engine-dir", str(engine), + "--components", "llm", "--plugin-path", package["plugin"], + "--dense", "fp16", "--max-input-len", str(limit), + "--max-kv-cache-capacity", str(limit), "--max-batch-size", "1", + "--externalize-weights", "all"] + if request.verbose: + command.append("--verbose") + with log_path.open("a", encoding="utf-8") as log: + subprocess.run(command, check=True, stdout=log, stderr=subprocess.STDOUT, cwd=staging) + for name in ("llm.engine", "config.json", "tokenizer.json", "tokenizer_config.json", "processed_chat_template.json"): + if not (engine / name).is_file() or (engine / name).stat().st_size == 0: + raise ValueError(f"Edge builder did not produce required artifact: {name}") + files = {} + for directory in (engine, checkpoint): + for path in sorted(directory.rglob("*")): + if path.is_symlink(): + raise ValueError(f"Edge output must not contain symlinks: {path}") + if path.is_file(): + files[path.relative_to(staging).as_posix()] = path + return files, { + "version": 1, "edge_revision": EDGE_REVISION, "target": target, "precision": "fp16", + "max_sequence_length": limit, "max_input_length": limit, "max_batch_size": 1, + "artifacts": list(files), + } + + +def publish(request, writer, files: dict, marker: dict) -> None: + """Stream complete Edge sections; publication errors must not retry native.""" + writer.set_header(family=request.family, task=request.task, backend=request.backend) + for name, path in files.items(): + with path.open("rb") as source, writer.open_section(name) as destination: + shutil.copyfileobj(source, destination, length=1024 * 1024) + writer.add_json("edge_llm.json", marker) diff --git a/families/qwen3_5/model.py b/families/qwen3_5/model.py index 3af3d7c9f4..8421e3f489 100644 --- a/families/qwen3_5/model.py +++ b/families/qwen3_5/model.py @@ -1360,7 +1360,7 @@ def _runtime_config(model_dir: Path, config: ModelConfig, model: _Qwen35Model, * return runtime -def build(request: "BuildRequest", writer: "BundleWriter") -> None: +def _build_native(request: "BuildRequest", writer: "BundleWriter") -> None: """Build one Qwen3.5 hybrid bundle through family-owned code only.""" if request.dynamic_kv_cache: raise NotImplementedError("qwen3_5 does not support dynamic_kv_cache") @@ -1432,3 +1432,10 @@ def build(request: "BuildRequest", writer: "BundleWriter") -> None: path = model_dir / filename if path.is_file(): writer.add_bytes(filename, path.read_bytes()) + + +def build(request, writer) -> None: + """Select a family-owned complete-network adapter or the native builder.""" + from .dispatch import build as dispatch_build + + dispatch_build(request, writer, _build_native) diff --git a/families/qwen3_5/runtime/CMakeLists.txt b/families/qwen3_5/runtime/CMakeLists.txt index 769359c270..223625bf86 100644 --- a/families/qwen3_5/runtime/CMakeLists.txt +++ b/families/qwen3_5/runtime/CMakeLists.txt @@ -28,7 +28,7 @@ target_link_libraries(trtmc_model_qwen3_5 PRIVATE ) target_compile_options(trtmc_model_qwen3_5 PRIVATE "$<$:-Wall;-Wextra;-Wpedantic>" - "$<$:-Wall;-Wextra>" + "$<$:-Xcompiler=-Wall,-Wextra>" ) set_target_properties(trtmc_model_qwen3_5 PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" @@ -55,3 +55,88 @@ if(TRTMC_BUILD_TESTS) ) add_test(NAME ${test_name} COMMAND ${test_name}) endif() + +# Complete-network offload is family-owned and absent from native-only builds. +if(TARGET EdgeLLM::Core) + target_sources(trtmc_model_qwen3_5 PRIVATE + edge_llm/adapter.cpp + edge_llm/device_link.cu + ) + target_compile_definitions(trtmc_model_qwen3_5 PRIVATE TRTMC_HAS_EDGE_LLM=1) + target_link_libraries(trtmc_model_qwen3_5 PRIVATE EdgeLLM::Core) + set_target_properties(trtmc_model_qwen3_5 PROPERTIES + CUDA_ARCHITECTURES "${EdgeLLM_CUDA_ARCHITECTURE}" + CUDA_SEPARABLE_COMPILATION ON + CUDA_RESOLVE_DEVICE_SYMBOLS ON + ) +endif() + +if(TRTMC_BUILD_TESTS) + add_executable(test_qwen3_5_edge_contract + ${PROJECT_SOURCE_DIR}/families/qwen3_5/tests/cpp/test_qwen3_5_edge_contract.cpp + ) + target_include_directories(test_qwen3_5_edge_contract PRIVATE + ${PROJECT_SOURCE_DIR} + ${PROJECT_SOURCE_DIR}/core/runtime/include + ) + add_test(NAME test_qwen3_5_edge_contract COMMAND test_qwen3_5_edge_contract) +endif() + +if(TARGET EdgeLLM::Core) + add_custom_command(TARGET trtmc_model_qwen3_5 POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + $ $ + VERBATIM + ) + if(TRTMC_BUILD_TESTS) + add_executable(test_qwen3_5_edge_request + ${PROJECT_SOURCE_DIR}/families/qwen3_5/tests/cpp/test_qwen3_5_edge_request.cpp + ) + target_include_directories(test_qwen3_5_edge_request PRIVATE + ${PROJECT_SOURCE_DIR} + ${PROJECT_SOURCE_DIR}/core/runtime/include + ) + target_link_libraries(test_qwen3_5_edge_request PRIVATE EdgeLLM::Core) + add_test(NAME test_qwen3_5_edge_request COMMAND test_qwen3_5_edge_request) + endif() +endif() + +if(TRTMC_BUILD_TESTS) + add_executable(test_qwen3_5_edge_factory + ${PROJECT_SOURCE_DIR}/families/qwen3_5/tests/cpp/test_qwen3_5_edge_factory.cpp + ) + target_include_directories(test_qwen3_5_edge_factory PRIVATE + ${PROJECT_SOURCE_DIR}/core/runtime/include + ${TRTMC_CUDA_INCLUDE_DIR} + ) + target_link_libraries(test_qwen3_5_edge_factory PRIVATE + trtmc_model_qwen3_5 trtmc_core nlohmann_json::nlohmann_json + ) + if(TARGET EdgeLLM::Core) + target_compile_definitions(test_qwen3_5_edge_factory PRIVATE TRTMC_HAS_EDGE_LLM=1) + endif() + add_test(NAME test_qwen3_5_edge_factory COMMAND test_qwen3_5_edge_factory) +endif() + +# Explicit GPU integration driver, not an unprovisioned default CTest. +if(TRTMC_BUILD_TESTS) + add_executable(qwen3_5_edge_inference + ${PROJECT_SOURCE_DIR}/families/qwen3_5/tests/cpp/test_qwen3_5_edge_inference.cpp + ) + target_link_libraries(qwen3_5_edge_inference PRIVATE + trtmc_runtime trtmc_core nlohmann_json::nlohmann_json + ) +endif() + +if(TRTMC_BUILD_TESTS AND TARGET EdgeLLM::Core) + add_executable(qwen3_5_edge_reference + ${PROJECT_SOURCE_DIR}/families/qwen3_5/tests/cpp/test_qwen3_5_edge_reference.cpp + edge_llm/device_link.cu + ) + target_link_libraries(qwen3_5_edge_reference PRIVATE EdgeLLM::Core) + set_target_properties(qwen3_5_edge_reference PROPERTIES + CUDA_ARCHITECTURES "${EdgeLLM_CUDA_ARCHITECTURE}" + CUDA_SEPARABLE_COMPILATION ON + CUDA_RESOLVE_DEVICE_SYMBOLS ON + ) +endif() diff --git a/families/qwen3_5/runtime/edge_llm/adapter.cpp b/families/qwen3_5/runtime/edge_llm/adapter.cpp new file mode 100644 index 0000000000..59f57cebf2 --- /dev/null +++ b/families/qwen3_5/runtime/edge_llm/adapter.cpp @@ -0,0 +1,224 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +#include "families/qwen3_5/runtime/edge_llm/adapter.h" + +#include "families/qwen3_5/runtime/edge_llm/request.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace trtmc::qwen3_5::edge_llm { +namespace { +namespace fs = std::filesystem; + +/// Turn CUDA failures into caller-visible load or inference errors. +void check_cuda(cudaError_t result) { + if (result != cudaSuccess) + throw std::runtime_error(std::string("Qwen3.5 Edge CUDA error: ") + + cudaGetErrorString(result)); +} + +/// Reject an engine built for a different local GPU or CUDA/TensorRT runtime. +void validate_target(const nlohmann::json& target) { + utsname host{}; + if (uname(&host) != 0) + throw std::runtime_error("Cannot identify Qwen3.5 Edge runtime host"); + std::ifstream release("/etc/os-release"); + std::string line, os_version; + while (std::getline(release, line)) { + if (line.rfind("VERSION_ID=", 0) == 0) { + os_version = line.substr(11); + if (os_version.size() >= 2 && os_version.front() == char(34) && + os_version.back() == char(34)) + os_version = os_version.substr(1, os_version.size() - 2); + } + } + int device = 0, cuda_version = 0; + check_cuda(cudaGetDevice(&device)); + check_cuda(cudaRuntimeGetVersion(&cuda_version)); + cudaDeviceProp gpu{}; + check_cuda(cudaGetDeviceProperties(&gpu, device)); + const int trt_version = getInferLibVersion(); + const std::string trt = + std::to_string(trt_version / 10000) + "." + std::to_string((trt_version % 10000) / 100) + + "." + std::to_string(trt_version % 100) + "." + std::to_string(getInferLibBuildVersion()); + const std::string cuda = + std::to_string(cuda_version / 1000) + "." + std::to_string((cuda_version % 1000) / 10); + if (target.at("os") != "linux" || target.at("os_version") != os_version || + target.at("arch") != host.machine || target.at("sm") != gpu.major * 10 + gpu.minor || + target.at("cuda_version") != cuda || target.at("tensorrt_version") != trt) + throw std::runtime_error( + "Qwen3.5 Edge bundle requires its build GPU and CUDA/TensorRT stack"); +} + +/// Own extracted engine/checkpoint files until after the Edge runtime is destroyed. +class Artifacts { + public: + explicit Artifacts(const BundleReader& bundle, const nlohmann::json& marker) { + std::set names; + for (const auto& entry : marker.at("artifacts")) { + const auto name = entry.get(); + if (!safe_artifact_path(name) || !names.insert(name).second || + !bundle.find_section(name)) + throw std::runtime_error("Invalid Qwen3.5 Edge artifact: " + name); + } + for (const auto* required : + {"edge_llm/engine/llm.engine", "edge_llm/engine/config.json", + "edge_llm/engine/tokenizer.json", "edge_llm/engine/tokenizer_config.json", + "edge_llm/engine/processed_chat_template.json", "edge_llm/checkpoint/config.json"}) + if (!names.count(required) || bundle.find_section(required)->length == 0) + throw std::runtime_error(std::string("Required Qwen3.5 Edge artifact missing: ") + + required); + std::string pattern = (fs::temp_directory_path() / "trtmc-qwen3_5-edge-XXXXXX").string(); + if (!mkdtemp(pattern.data())) + throw std::runtime_error("Cannot create Qwen3.5 Edge artifact directory"); + root_ = pattern; + try { + for (const auto& name : names) { + const auto destination = root_ / name; + fs::create_directories(destination.parent_path()); + std::ofstream output(destination, std::ios::binary); + bundle.copy_section(name, output); + output.close(); + if (!output) + throw std::runtime_error("Cannot extract Qwen3.5 Edge artifact: " + name); + } + } catch (...) { + cleanup(); + throw; + } + } + ~Artifacts() { cleanup(); } + Artifacts(const Artifacts&) = delete; + Artifacts& operator=(const Artifacts&) = delete; + std::string engine() const { return (root_ / "edge_llm/engine").string(); } + std::string checkpoint() const { return (root_ / "edge_llm/checkpoint").string(); } + + private: + void cleanup() noexcept { + std::error_code ignored; + fs::remove_all(root_, ignored); + } + fs::path root_; +}; + +/// Close the plugin handle after runtime destruction; registrations remain mapped. +struct CloseLibrary { + void operator()(void* handle) const noexcept { + if (handle) + dlclose(handle); + } +}; + +/// Initialize the CMake-installed adjacent plugin without process-global environment mutation. +std::unique_ptr load_plugin() { + Dl_info location{}; + if (!dladdr(reinterpret_cast(&create), &location) || !location.dli_fname) + throw std::runtime_error("Cannot locate Qwen3.5 family library"); + const auto path = + fs::absolute(location.dli_fname).parent_path() / "libNvInfer_edgellm_plugin.so"; + std::unique_ptr plugin( + dlopen(path.c_str(), RTLD_NOW | RTLD_GLOBAL | RTLD_NODELETE)); + if (!plugin) + throw std::runtime_error("Cannot load CMake-installed Edge plugin: " + + std::string(dlerror())); + using Initialize = bool (*)(void*, const char*); + auto initialize = reinterpret_cast(dlsym(plugin.get(), "initEdgellmPlugins")); + if (!initialize || !initialize(static_cast(&trt_edgellm::gLogger), "")) + throw std::runtime_error("Cannot initialize Qwen3.5 Edge plugin"); + return plugin; +} + +/// Stream ownership is independent of construction success and outlives the Edge instance. +class Stream { + public: + Stream() { check_cuda(cudaStreamCreateWithFlags(&value_, cudaStreamNonBlocking)); } + ~Stream() { cudaStreamDestroy(value_); } + Stream(const Stream&) = delete; + Stream& operator=(const Stream&) = delete; + cudaStream_t get() const { return value_; } + + private: + cudaStream_t value_{nullptr}; +}; + +/// Thin persistent Edge API adapter; serialization prevents concurrent use of Edge request state. +class EdgeTask final : public ITextGeneration { + public: + EdgeTask(const BundleReader& bundle, const nlohmann::json& marker) + : artifacts_(bundle, marker), plugin_(load_plugin()), + runtime_(artifacts_.engine(), "", std::unordered_map{}, + stream_.get(), trt_edgellm::rt::ContextCacheConfig{}, artifacts_.checkpoint()), + capacity_(marker.at("max_sequence_length").get()), + input_limit_(marker.at("max_input_length").get()) {} + + std::int32_t default_max_new_tokens() const override { return std::min(128, capacity_ - 1); } + + /// Drain work from failed requests before destroying the runtime and its weight buffers. + ~EdgeTask() override { cudaStreamSynchronize(stream_.get()); } + + /// Invoke Edge once; failures propagate without attempting native inference. + TextResult generate(const std::string& prompt, const TextGenerationConfig& config) override { + auto request = make_request(prompt, config, default_max_new_tokens()); + std::lock_guard lock(mutex_); + const auto counts = runtime_.countPromptTokens(request); + if (counts.size() != 1) + throw std::runtime_error("Qwen3.5 Edge returned invalid prompt counts"); + validate_capacity(counts.front(), input_limit_, capacity_, request.maxGenerateLength); + trt_edgellm::rt::LLMGenerationResponse response{}; + if (!runtime_.handleRequest(request, response, stream_.get()) || + response.outputIds.size() != 1 || response.outputTexts.size() != 1 || + response.outputIds.front().empty() || + response.outputIds.front().size() > static_cast(request.maxGenerateLength)) + throw std::runtime_error("Qwen3.5 Edge generation failed"); + if (response.finishReasons.size() != 1 || + (response.finishReasons.front() != trt_edgellm::rt::FinishReason::kEndId && + response.finishReasons.front() != trt_edgellm::rt::FinishReason::kLength)) + throw std::runtime_error("Qwen3.5 Edge generation did not complete successfully"); + // This API does not expose per-request stage times; zero means unavailable. + return {std::move(response.outputTexts.front()), std::move(response.outputIds.front())}; + } + + private: + // Reverse destruction order keeps weights, plugin and stream alive throughout Edge teardown. + Artifacts artifacts_; + std::unique_ptr plugin_; + Stream stream_; + trt_edgellm::rt::LLMInferenceRuntime runtime_; + int capacity_; + int input_limit_; + std::mutex mutex_; +}; +} // namespace + +ITask* create(const BundleReader& bundle) { + const auto bytes = bundle.read_section("edge_llm.json"); + const auto marker = nlohmann::json::parse(bytes.begin(), bytes.end()); + if (marker.at("version") != 1 || marker.at("edge_revision") != kRevision || + marker.at("max_sequence_length").get() <= 1 || + marker.at("max_input_length").get() <= 0 || + marker.at("max_input_length").get() > marker.at("max_sequence_length").get() || + marker.at("max_batch_size") != 1 || marker.at("precision") != "fp16" || + !marker.at("artifacts").is_array()) + throw std::runtime_error("Invalid Qwen3.5 Edge bundle contract"); + validate_target(marker.at("target")); + return new EdgeTask(bundle, marker); +} + +} // namespace trtmc::qwen3_5::edge_llm diff --git a/families/qwen3_5/runtime/edge_llm/adapter.h b/families/qwen3_5/runtime/edge_llm/adapter.h new file mode 100644 index 0000000000..ac3277daae --- /dev/null +++ b/families/qwen3_5/runtime/edge_llm/adapter.h @@ -0,0 +1,15 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include "trtmc/bundle.h" +#include "trtmc/task.h" + +namespace trtmc::qwen3_5::edge_llm { + +/// Create a persistent Edge task from a self-contained bundle; throws on load failure. +ITask* create(const BundleReader& bundle); + +} // namespace trtmc::qwen3_5::edge_llm diff --git a/families/qwen3_5/runtime/edge_llm/contract.h b/families/qwen3_5/runtime/edge_llm/contract.h new file mode 100644 index 0000000000..fe9d58be94 --- /dev/null +++ b/families/qwen3_5/runtime/edge_llm/contract.h @@ -0,0 +1,56 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include "trtmc/task.h" + +#include +#include +#include +#include + +namespace trtmc::qwen3_5::edge_llm { + +inline constexpr const char* kRevision = "e8b29522938901f6df19ebeedd4b69bc8edbcd97"; + +/// Return whether an artifact is a normalized file below one of the two Edge roots. +inline bool safe_artifact_path(const std::string& name) { + if (name.find('\\') != std::string::npos || name.find('\0') != std::string::npos) + return false; + const std::filesystem::path path(name); + if (path.is_absolute() || path.filename().empty()) + return false; + for (const auto& part : path) + if (part == "." || part == "..") + return false; + return path.generic_string() == name && + (name.rfind("edge_llm/engine/", 0) == 0 || name.rfind("edge_llm/checkpoint/", 0) == 0); +} + +/// Reject invalid sampling settings and controls with no equivalent Edge request API. +inline void validate_generation(const TextGenerationConfig& c) { + if (!std::isfinite(c.temperature) || c.temperature < 0 || !std::isfinite(c.top_p) || + c.top_p <= 0 || c.top_p > 1 || c.top_k < 0) + throw std::invalid_argument("Invalid Qwen3.5 Edge sampling parameters"); + if (c.min_p != 0 || c.seed != -1 || c.eos_token_id != -1 || c.repetition_penalty != 1 || + !c.lora_adapter_id.empty() || c.stop_on_boxed_answer || + (c.text_generation_mode != "auto" && c.text_generation_mode != "autoregressive") || + c.source_language_token_id != -1 || c.forced_bos_token_id != -1 || c.guidance_scale != -1 || + c.cfg_scale != -1 || c.num_steps != -1 || c.sde_gamma != -1 || !c.initial_latents.empty() || + !c.condition_latents.empty() || !c.condition_mask.empty() || !c.sampling_steps.empty() || + !c.sde_noises.empty() || c.block_length != 0 || c.confidence_threshold != -1) + throw std::invalid_argument( + "Requested generation controls are unsupported by Qwen3.5 Edge"); +} + +/// Enforce prompt and total capacity without allowing Edge to silently clip generation. +inline void validate_capacity(int prompt_tokens, int input_limit, int capacity, + std::int64_t generated_tokens) { + if (prompt_tokens <= 0 || prompt_tokens > input_limit || generated_tokens <= 0 || + generated_tokens > static_cast(capacity) - prompt_tokens) + throw std::invalid_argument("Qwen3.5 Edge prompt and generation exceed bundle capacity"); +} + +} // namespace trtmc::qwen3_5::edge_llm diff --git a/families/qwen3_5/runtime/edge_llm/device_link.cu b/families/qwen3_5/runtime/edge_llm/device_link.cu new file mode 100644 index 0000000000..1ea4768d60 --- /dev/null +++ b/families/qwen3_5/runtime/edge_llm/device_link.cu @@ -0,0 +1,5 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +// Enable the final CUDA device-link step for Edge's static runtime dependencies. diff --git a/families/qwen3_5/runtime/edge_llm/request.h b/families/qwen3_5/runtime/edge_llm/request.h new file mode 100644 index 0000000000..ba15809542 --- /dev/null +++ b/families/qwen3_5/runtime/edge_llm/request.h @@ -0,0 +1,29 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include "families/qwen3_5/runtime/edge_llm/contract.h" + +#include + +namespace trtmc::qwen3_5::edge_llm { + +/// Map Model Connect text arguments to the pinned Edge API; rejects unmapped controls. +inline trt_edgellm::rt::LLMGenerationRequest +make_request(const std::string& prompt, const TextGenerationConfig& config, int default_length) { + validate_generation(config); + trt_edgellm::rt::LLMGenerationRequest request{}; + request.requests.resize(1); + request.requests.front().messages.push_back({"user", {{"text", prompt}}}); + request.applyChatTemplate = config.use_chat_template; + request.enableThinking = config.enable_thinking; + request.temperature = config.temperature; + request.topK = config.top_k; + request.topP = config.top_p; + request.maxGenerateLength = config.max_new_tokens > 0 ? config.max_new_tokens : default_length; + return request; +} + +} // namespace trtmc::qwen3_5::edge_llm diff --git a/families/qwen3_5/runtime/plugin.cpp b/families/qwen3_5/runtime/plugin.cpp index 7786e34b43..d855bc7a2e 100644 --- a/families/qwen3_5/runtime/plugin.cpp +++ b/families/qwen3_5/runtime/plugin.cpp @@ -9,6 +9,9 @@ #include "families/qwen3_5/runtime/plugin_helpers.h" #include "families/qwen3_5/runtime/recurrent_state.h" #include "trtmc/runtime/family_factory.h" +#ifdef TRTMC_HAS_EDGE_LLM +#include "families/qwen3_5/runtime/edge_llm/adapter.h" +#endif #include #include @@ -119,6 +122,14 @@ std::string chat_template(const BundleReader& bundle) { } // namespace ITask* create(const FamilyContext& context) { + if (context.reader.find_section("edge_llm.json")) { +#ifdef TRTMC_HAS_EDGE_LLM + return edge_llm::create(context.reader); +#else + throw std::runtime_error("Qwen3.5 Edge bundle requires a runtime configured with " + "-DTRTMC_ENABLE_EDGELLM=ON; rebuild and install Model Connect"); +#endif + } const RuntimeConfig config = parse_runtime_config(context.reader); auto decoder = load_engine(context.backend, require_section(context.reader, "engine.plan"), "qwen3_5 decoder"); diff --git a/families/qwen3_5/tests/cpp/test_qwen3_5_edge_contract.cpp b/families/qwen3_5/tests/cpp/test_qwen3_5_edge_contract.cpp new file mode 100644 index 0000000000..8675a7e3b4 --- /dev/null +++ b/families/qwen3_5/tests/cpp/test_qwen3_5_edge_contract.cpp @@ -0,0 +1,92 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +#include "families/qwen3_5/runtime/edge_llm/contract.h" + +#include +#include +#include +#include + +namespace edge = trtmc::qwen3_5::edge_llm; +namespace { +int failures = 0; + +/// Record a failed behavioral assertion without depending on NDEBUG. +void check(bool condition, const char* message) { + if (!condition) { + std::cerr << message << std::endl; + ++failures; + } +} + +/// Verify unsupported input is rejected by the public argument validation boundary. +void rejects(const std::function& operation) { + try { + operation(); + check(false, "Expected invalid_argument"); + } catch (const std::invalid_argument&) { + } +} +} // namespace + +int main() { + trtmc::TextGenerationConfig config; + edge::validate_generation(config); + config.temperature = 0; + config.use_chat_template = true; + config.enable_thinking = false; + config.text_generation_mode = "autoregressive"; + edge::validate_generation(config); + using Change = std::function; + const std::vector unsupported{ + [](auto& c) { c.temperature = -1; }, + [](auto& c) { c.temperature = std::numeric_limits::quiet_NaN(); }, + [](auto& c) { c.top_p = 0; }, + [](auto& c) { c.top_p = 1.01F; }, + [](auto& c) { c.top_p = std::numeric_limits::infinity(); }, + [](auto& c) { c.top_k = -1; }, + [](auto& c) { c.min_p = 0.1F; }, + [](auto& c) { c.seed = 42; }, + [](auto& c) { c.eos_token_id = 9; }, + [](auto& c) { c.repetition_penalty = 1.1F; }, + [](auto& c) { c.lora_adapter_id = "adapter"; }, + [](auto& c) { c.stop_on_boxed_answer = true; }, + [](auto& c) { c.text_generation_mode = "diffusion"; }, + [](auto& c) { c.source_language_token_id = 0; }, + [](auto& c) { c.forced_bos_token_id = 0; }, + [](auto& c) { c.guidance_scale = 0; }, + [](auto& c) { c.cfg_scale = 0; }, + [](auto& c) { c.num_steps = 0; }, + [](auto& c) { c.sde_gamma = 0; }, + [](auto& c) { c.initial_latents = {1}; }, + [](auto& c) { c.condition_latents = {1}; }, + [](auto& c) { c.condition_mask = {1}; }, + [](auto& c) { c.sampling_steps = {1}; }, + [](auto& c) { c.sde_noises = {1}; }, + [](auto& c) { c.block_length = 1; }, + [](auto& c) { c.confidence_threshold = 0.5F; }, + }; + for (const auto& change : unsupported) { + auto invalid = config; + change(invalid); + rejects([&] { edge::validate_generation(invalid); }); + } + edge::validate_capacity(5, 1024, 1024, 1019); + rejects([] { edge::validate_capacity(5, 1024, 1024, 1020); }); + rejects([] { edge::validate_capacity(1024, 512, 2048, 1); }); + rejects([] { edge::validate_capacity(0, 1024, 1024, 1); }); + rejects([] { edge::validate_capacity(5, 1024, 1024, 0); }); + rejects([] { edge::validate_capacity(5, 1024, 1024, INT64_MAX); }); + for (const auto* name : {"edge_llm/engine/llm.engine", "edge_llm/checkpoint/model.safetensors", + "edge_llm/engine/rank0/config.json"}) + check(edge::safe_artifact_path(name), "Valid artifact path rejected"); + for (const auto* name : + {"/tmp/engine", "edge_llm/engine/../../outside", "edge_llm/engine/./file", + "edge_llm/engine//file", "edge_llm/engine/", "other/file", "edge_llm/engine/a\\b"}) + check(!edge::safe_artifact_path(name), "Unsafe artifact path accepted"); + check(!edge::safe_artifact_path(std::string("edge_llm/engine/a\0b", 19)), + "Embedded null artifact accepted"); + return failures == 0 ? 0 : 1; +} diff --git a/families/qwen3_5/tests/cpp/test_qwen3_5_edge_factory.cpp b/families/qwen3_5/tests/cpp/test_qwen3_5_edge_factory.cpp new file mode 100644 index 0000000000..4cafbc3156 --- /dev/null +++ b/families/qwen3_5/tests/cpp/test_qwen3_5_edge_factory.cpp @@ -0,0 +1,95 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +#include "trtmc/runtime/family_factory.h" +#include "trtmc/runtime/trt_backend.h" + +#include +#include +#include +#include +#include + +namespace { +/// A routing test must fail before attempting any native engine load. +class RejectBackend final : public trtmc::IBackend { + public: + std::unique_ptr create_module(const void*, size_t, + const trtmc::ModuleCreateOptions&) override { + throw std::logic_error("unexpected engine load"); + } + std::unique_ptr + create_module_prebound(const void*, size_t, const trtmc::ModuleCreateOptions&, + const std::vector&) override { + throw std::logic_error("unexpected prebound engine load"); + } + trtmc::BackendDualProfileModules + create_dual_profile_modules(const void*, size_t, const trtmc::ModuleCreateOptions&) override { + throw std::logic_error("unexpected dual engine load"); + } + const char* name() const override { return "trt"; } +}; + +/// Write a deliberately incomplete bundle to observe which factory owns its error. +void write_bundle(const std::filesystem::path& path, bool edge) { + nlohmann::json sections = nlohmann::json::object(); + if (edge) + sections["edge_llm.json"] = {{"offset", 0}, {"length", 2}}; + const auto header = nlohmann::json{ + {"format", 1}, + {"family", "qwen3_5"}, + {"task", "text_generation"}, + {"backend", "trt"}, + {"sections", + sections}}.dump(); + std::ofstream out(path, std::ios::binary); + out.write("BUNDLE\x01\x00", 8); + const std::uint64_t size = header.size(); + for (int shift = 0; shift < 64; shift += 8) + out.put(static_cast((size >> shift) & 0xff)); + out << header; + if (edge) + out << "{}"; + if (!out) + throw std::runtime_error("Cannot write factory test bundle"); +} +} // namespace + +int main() { + char pattern[] = "/tmp/trtmc_qwen3_5_factory_XXXXXX"; + const auto* directory = mkdtemp(pattern); + if (!directory) + return 1; + const auto path = std::filesystem::path(directory) / "test.bundle"; + RejectBackend backend; + int failures = 0; + for (const bool edge : {false, true}) { + write_bundle(path, edge); + trtmc::BundleReader bundle(path.string()); + try { + std::unique_ptr task(trtmc_create_family({bundle, backend, 0})); + ++failures; + } catch (const std::exception& error) { + const std::string message = error.what(); + if (!edge) { + if (message.find("runtime.json") == std::string::npos) + ++failures; + } else { +#ifdef TRTMC_HAS_EDGE_LLM + // The malformed Edge marker must not reach native metadata parsing. + if (message.find("runtime.json") != std::string::npos || + message.find("unexpected") != std::string::npos) + ++failures; +#else + if (message.find("TRTMC_ENABLE_EDGELLM=ON") == std::string::npos) + ++failures; +#endif + } + } + } + std::filesystem::remove_all(directory); + if (failures) + std::cerr << "Family Edge/native factory routing failed" << std::endl; + return failures == 0 ? 0 : 1; +} diff --git a/families/qwen3_5/tests/cpp/test_qwen3_5_edge_inference.cpp b/families/qwen3_5/tests/cpp/test_qwen3_5_edge_inference.cpp new file mode 100644 index 0000000000..a06d3bcd5b --- /dev/null +++ b/families/qwen3_5/tests/cpp/test_qwen3_5_edge_inference.cpp @@ -0,0 +1,142 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +#include "trtmc/bundle.h" +#include "trtmc/runtime/family_loader.h" +#include "trtmc/task.h" + +#include +#include +#include +#include +#include +#include + +namespace { +/// Apply only explicitly named public configuration fields; unknown test inputs are errors. +trtmc::TextGenerationConfig generation_config(const nlohmann::json& values) { + trtmc::TextGenerationConfig result; + if (!values.is_object()) + throw std::invalid_argument("config must be an object"); + for (const auto& [key, value] : values.items()) { + if (key == "max_new_tokens") + result.max_new_tokens = value.get(); + else if (key == "temperature") + result.temperature = value.get(); + else if (key == "top_k") + result.top_k = value.get(); + else if (key == "top_p") + result.top_p = value.get(); + else if (key == "min_p") + result.min_p = value.get(); + else if (key == "seed") + result.seed = value.get(); + else if (key == "eos_token_id") + result.eos_token_id = value.get(); + else if (key == "use_chat_template") + result.use_chat_template = value.get(); + else if (key == "enable_thinking") + result.enable_thinking = value.get(); + else if (key == "text_generation_mode") + result.text_generation_mode = value.get(); + else if (key == "stop_on_boxed_answer") + result.stop_on_boxed_answer = value.get(); + else if (key == "lora_adapter_id") + result.lora_adapter_id = value.get(); + else if (key == "repetition_penalty") + result.repetition_penalty = value.get(); + else if (key == "block_length") + result.block_length = value.get(); + else if (key == "confidence_threshold") + result.confidence_threshold = value.get(); + else if (key == "forced_bos_token_id") + result.forced_bos_token_id = value.get(); + else if (key == "source_language_token_id") + result.source_language_token_id = value.get(); + else + throw std::invalid_argument("Unknown public generation field: " + key); + } + return result; +} + +/// Parse required qualification inputs without an environment or runtime-root fallback. +std::map arguments(int argc, char** argv) { + std::map result; + for (int i = 1; i < argc; i += 2) { + if (i + 1 >= argc || !result.emplace(argv[i], argv[i + 1]).second) + throw std::invalid_argument("Expected unique --option VALUE pairs"); + } + for (const auto* name : {"--bundle", "--runtime-root", "--requests", "--output"}) + if (!result.count(name)) + throw std::invalid_argument(std::string("Missing ") + name); + if (result.size() != 4) + throw std::invalid_argument("Unknown qualification option"); + return result; +} +} // namespace + +/// Exercise the public load/generate path once, retaining one task across the complete case list. +int main(int argc, char** argv) { + try { + const auto args = arguments(argc, argv); + const trtmc::BundleReader bundle(args.at("--bundle")); + if (bundle.info().family != "qwen3_5" || !bundle.find_section("edge_llm.json")) + throw std::runtime_error("Expected a qwen3_5 Edge bundle, not native fallback"); + nlohmann::json report{{"backend", "native"}, {"results", nlohmann::json::array()}}; + if (bundle.find_section("edge_llm.json")) { + const auto bytes = bundle.read_section("edge_llm.json"); + const auto marker = nlohmann::json::parse(bytes.begin(), bytes.end()); + report["backend"] = "edge_llm"; + report["edge_revision"] = marker.at("edge_revision"); + } + std::ifstream request_file(args.at("--requests")); + const auto requests = nlohmann::json::parse(request_file); + if (!requests.is_array() || requests.empty()) + throw std::invalid_argument("Requests must be a nonempty JSON array"); + auto task = trtmc::load_task(args.at("--bundle"), args.at("--runtime-root")); + auto* generator = dynamic_cast(task.get()); + if (!generator) + throw std::runtime_error("Bundle does not expose ITextGeneration"); + bool passed = true; + std::map previous; + for (const auto& request : requests) { + nlohmann::json output{{"id", request.at("id")}}; + try { + const auto config = + generation_config(request.value("config", nlohmann::json::object())); + const auto result = + generator->generate(request.at("prompt").get(), config); + output["text"] = result.text; + output["token_ids"] = result.token_ids; + } catch (const std::exception& error) { + output["error"] = error.what(); + } + if (output.contains("error") != request.value("expect_error", false)) + passed = false; + if (request.contains("equal_to")) { + const auto earlier = previous.find(request.at("equal_to").get()); + if (earlier == previous.end() || output.contains("error") || + earlier->second.value("text", "") != output.value("text", "") || + earlier->second.value("token_ids", nlohmann::json::array()) != + output.value("token_ids", nlohmann::json::array())) + passed = false; + } + if (!output.contains("error") && output.at("token_ids").empty()) + passed = false; + previous[request.at("id").get()] = output; + report["results"].push_back(std::move(output)); + } + report["passed"] = passed; + task.reset(); + std::ofstream output(args.at("--output")); + output << report.dump(2) << '\n'; + output.close(); + if (!output) + throw std::runtime_error("Cannot write qualification output"); + return passed ? 0 : 1; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/families/qwen3_5/tests/cpp/test_qwen3_5_edge_reference.cpp b/families/qwen3_5/tests/cpp/test_qwen3_5_edge_reference.cpp new file mode 100644 index 0000000000..e1fc25fdfc --- /dev/null +++ b/families/qwen3_5/tests/cpp/test_qwen3_5_edge_reference.cpp @@ -0,0 +1,135 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +/// Own the explicit native stream until after the Edge runtime is destroyed. +struct Stream { + cudaStream_t value{nullptr}; + Stream() { + const auto status = cudaStreamCreateWithFlags(&value, cudaStreamNonBlocking); + if (status != cudaSuccess) + throw std::runtime_error(cudaGetErrorString(status)); + } + ~Stream() { + if (value) + cudaStreamDestroy(value); + } +}; +struct CloseLibrary { + void operator()(void* handle) const noexcept { + if (handle) + dlclose(handle); + } +}; + +/// Map the independently specified native request schema, not Model Connect configuration. +trt_edgellm::rt::LLMGenerationRequest native_request(const nlohmann::json& value) { + trt_edgellm::rt::LLMGenerationRequest request{}; + request.requests.resize(1); + for (const auto& message : value.at("messages")) { + trt_edgellm::rt::Message native_message; + native_message.role = message.at("role").get(); + for (const auto& content : message.at("content")) { + if (content.at("type") != "text") + throw std::invalid_argument("Reference runner accepts text content only"); + native_message.contents.push_back({"text", content.at("text").get()}); + } + request.requests.front().messages.push_back(std::move(native_message)); + } + request.applyChatTemplate = value.at("apply_chat_template").get(); + request.enableThinking = value.at("enable_thinking").get(); + request.temperature = value.at("temperature").get(); + request.topK = value.at("top_k").get(); + request.topP = value.at("top_p").get(); + request.maxGenerateLength = value.at("max_generate_length").get(); + return request; +} + +/// Parse native reference inputs with an explicit plugin path and no environment fallback. +std::map arguments(int argc, char** argv) { + std::map result; + for (int i = 1; i < argc; i += 2) + if (i + 1 >= argc || !result.emplace(argv[i], argv[i + 1]).second) + throw std::invalid_argument("Expected unique --option VALUE pairs"); + for (const auto* name : + {"--engine-dir", "--checkpoint-dir", "--requests", "--output", "--plugin-path"}) + if (!result.count(name)) + throw std::invalid_argument(std::string("Missing ") + name); + if (result.size() != 5) + throw std::invalid_argument("Unknown native reference option"); + return result; +} +} // namespace + +/// Run a persistent direct Edge baseline without loading Model Connect or its argument mapper. +int main(int argc, char** argv) { + try { + const auto args = arguments(argc, argv); + std::unique_ptr plugin( + dlopen(args.at("--plugin-path").c_str(), RTLD_NOW | RTLD_GLOBAL | RTLD_NODELETE)); + if (!plugin) + throw std::runtime_error(std::string("Cannot load Edge plugin: ") + dlerror()); + using InitializeFn = bool (*)(void*, const char*); + auto initialize = reinterpret_cast(dlsym(plugin.get(), "initEdgellmPlugins")); + if (!initialize || !initialize(static_cast(&trt_edgellm::gLogger), "")) + throw std::runtime_error("Cannot initialize Edge plugin"); + Stream stream; + trt_edgellm::rt::LLMInferenceRuntime runtime( + args.at("--engine-dir"), "", std::unordered_map{}, + stream.value, trt_edgellm::rt::ContextCacheConfig{}, args.at("--checkpoint-dir")); + std::ifstream request_file(args.at("--requests")); + const auto requests = nlohmann::json::parse(request_file); + if (!requests.is_array() || requests.empty()) + throw std::invalid_argument("Requests must be a nonempty JSON array"); + nlohmann::json report{{"backend", "direct_edge"}, {"results", nlohmann::json::array()}}; + bool passed = true; + for (const auto& input : requests) { + nlohmann::json output{{"id", input.at("id")}}; + try { + const auto request = native_request(input); + trt_edgellm::rt::LLMGenerationResponse response{}; + if (!runtime.handleRequest(request, response, stream.value) || + response.outputIds.size() != 1 || response.outputTexts.size() != 1 || + response.outputIds.front().empty() || + response.outputIds.front().size() > + static_cast(request.maxGenerateLength) || + response.finishReasons.size() != 1 || + (response.finishReasons.front() != trt_edgellm::rt::FinishReason::kEndId && + response.finishReasons.front() != trt_edgellm::rt::FinishReason::kLength)) + throw std::runtime_error("Direct Edge generation failed"); + output["text"] = response.outputTexts.front(); + output["token_ids"] = response.outputIds.front(); + output["finish_reason"] = static_cast(response.finishReasons.front()); + } catch (const std::exception& error) { + output["error"] = error.what(); + passed = false; + } + report["results"].push_back(std::move(output)); + } + report["passed"] = passed; + std::ofstream output(args.at("--output")); + output << report.dump(2) << '\n'; + output.close(); + if (!output) + throw std::runtime_error("Cannot write native reference output"); + return passed ? 0 : 1; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/families/qwen3_5/tests/cpp/test_qwen3_5_edge_request.cpp b/families/qwen3_5/tests/cpp/test_qwen3_5_edge_request.cpp new file mode 100644 index 0000000000..fb0ff027cb --- /dev/null +++ b/families/qwen3_5/tests/cpp/test_qwen3_5_edge_request.cpp @@ -0,0 +1,41 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +#include "families/qwen3_5/runtime/edge_llm/request.h" + +#include + +int main() { + trtmc::TextGenerationConfig config; + config.max_new_tokens = 27; + config.temperature = 0.7F; + config.top_k = 12; + config.top_p = 0.8F; + config.use_chat_template = true; + config.enable_thinking = false; + const auto request = trtmc::qwen3_5::edge_llm::make_request("Hello", config, 128); + if (request.requests.size() != 1 || request.requests[0].messages.size() != 1 || + request.requests[0].messages[0].role != "user" || + request.requests[0].messages[0].contents[0].content != "Hello" || + request.temperature != config.temperature || request.topK != config.top_k || + request.topP != config.top_p || request.maxGenerateLength != config.max_new_tokens || + !request.applyChatTemplate || request.enableThinking || !request.addGenerationPrompt || + request.saveSystemPromptKVCache || !request.loraWeightsName.empty()) { + std::cerr << "Model Connect arguments did not map to the pinned Edge request" << std::endl; + return 1; + } + config.max_new_tokens = 0; + config.temperature = 0; + config.use_chat_template = false; + const auto raw = trtmc::qwen3_5::edge_llm::make_request("raw", config, 32); + if (raw.maxGenerateLength != 32 || raw.temperature != 0 || raw.applyChatTemplate) + return 1; + config.seed = 42; + try { + trtmc::qwen3_5::edge_llm::make_request("unsupported", config, 32); + return 1; + } catch (const std::invalid_argument&) { + } + return 0; +} diff --git a/families/qwen3_5/tests/edge_validation.py b/families/qwen3_5/tests/edge_validation.py new file mode 100644 index 0000000000..3f5a2940ec --- /dev/null +++ b/families/qwen3_5/tests/edge_validation.py @@ -0,0 +1,371 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Validate a real Qwen3.5 Edge bundle through persistent public and direct APIs. + +Run as ``python -m families.qwen3_5.tests.edge_validation --help``. No model +builds, downloads, admission overrides, or native fallback occur in this harness. +""" + +from __future__ import annotations + +import argparse +import json +import struct +import subprocess +from pathlib import Path, PurePosixPath + +from tensorrt_model_connect.bundle_writer import BUNDLE_MAGIC +from families.qwen3_5.edge_llm import EDGE_REVISION + + +def write_json(path: Path, value: object) -> None: + """Write one human-readable validation artifact.""" + path.write_text(json.dumps(value, indent=2) + "\n", encoding="utf-8") + + +def extract_bundle(bundle: Path, destination: Path) -> dict: + """Extract only declared Edge assets using bounded reads; reject unsafe bundles.""" + destination.mkdir(parents=True, exist_ok=False) + size = bundle.stat().st_size + with bundle.open("rb") as source: + if source.read(8) != BUNDLE_MAGIC: + raise ValueError("Invalid bundle magic") + header_size_bytes = source.read(8) + if len(header_size_bytes) != 8: + raise ValueError("Truncated bundle header") + header_size = struct.unpack(" int: + section = sections[name] + offset, length = section["offset"], section["length"] + if ( + type(offset) is not int + or type(length) is not int + or offset < 0 + or length < 0 + or offset + length > size - start + ): + raise ValueError(f"Invalid bundle section bounds: {name}") + source.seek(start + offset) + return length + + marker_size = seek_section("edge_llm.json") + if marker_size > 100 * 1024 * 1024: + raise ValueError("Invalid Edge marker size") + marker = json.loads(source.read(marker_size)) + if marker["version"] != 1 or marker["edge_revision"] != EDGE_REVISION: + raise ValueError("Wrong Edge API revision") + artifacts = marker["artifacts"] + if ( + not isinstance(artifacts, list) + or not artifacts + or len(set(artifacts)) != len(artifacts) + ): + raise ValueError("Invalid Edge artifact list") + for name in artifacts: + relative = PurePosixPath(name) + if ( + not isinstance(name, str) + or relative.is_absolute() + or ".." in relative.parts + or str(relative) != name + or "\\" in name + or "\0" in name + or not name.startswith(("edge_llm/engine/", "edge_llm/checkpoint/")) + ): + raise ValueError(f"Unsafe Edge artifact: {name}") + remaining = seek_section(name) + target = destination / name + target.parent.mkdir(parents=True, exist_ok=True) + with target.open("wb") as output: + while remaining: + chunk = source.read(min(64 * 1024, remaining)) + if not chunk: + raise ValueError(f"Truncated artifact: {name}") + output.write(chunk) + remaining -= len(chunk) + return marker + + +def requests(capacity: int) -> tuple[list[dict], list[dict]]: + """Return matched greedy cases and public-only rejection/recovery checks.""" + common = { + "temperature": 0, + "top_k": 1, + "top_p": 1, + "enable_thinking": False, + "repetition_penalty": 1.0, + } + cases = [ + { + "id": "raw", + "prompt": "The capital of France is", + "config": {**common, "max_new_tokens": 8, "use_chat_template": False}, + }, + { + "id": "chat", + "prompt": "What is the capital of France? Answer in one word.", + "config": {**common, "max_new_tokens": 10, "use_chat_template": True}, + }, + { + "id": "eos", + "prompt": "Reply with exactly one word: Yes.", + "config": {**common, "max_new_tokens": 32, "use_chat_template": True}, + }, + ] + public = list(cases) + for control, value in ( + ("seed", 42), + ("min_p", 0.1), + ("eos_token_id", 0), + ("repetition_penalty", 1.1), + ("max_new_tokens", capacity), + ): + public.append( + { + "id": f"reject_{control}", + "prompt": cases[0]["prompt"], + "config": {**cases[0]["config"], control: value}, + "expect_error": True, + } + ) + repeated = {**cases[0], "id": "raw_after_rejections", "equal_to": "raw"} + public.append(repeated) + direct = [] + for case in [*cases, repeated]: + config = case["config"] + direct.append( + { + "id": case["id"], + "messages": [ + {"role": "user", "content": [{"type": "text", "text": case["prompt"]}]} + ], + "apply_chat_template": config["use_chat_template"], + "enable_thinking": config["enable_thinking"], + "temperature": config["temperature"], + "top_k": config["top_k"], + "top_p": config["top_p"], + "max_generate_length": config["max_new_tokens"], + } + ) + return public, direct + + +def run_driver(command: list[str], log: Path) -> None: + """Run one finite driver, retaining the command and combined diagnostics on failure.""" + write_json(log.with_suffix(".command.json"), command) + with log.open("w", encoding="utf-8") as output: + subprocess.run(command, check=True, stdout=output, stderr=subprocess.STDOUT, timeout=1800) + + +def compare(public: dict, direct: dict, cases: list[dict]) -> None: + """Require real Edge selection, exact direct-API parity, EOS, rejection, and reset behavior.""" + assert public["backend"] == "edge_llm" and public["edge_revision"] == EDGE_REVISION + assert direct["backend"] == "direct_edge" and public["passed"] and direct["passed"] + assert [item["id"] for item in public["results"]] == [item["id"] for item in cases] + actual = {item["id"]: item for item in public["results"]} + expected_ids = [case["id"] for case in cases if not case.get("expect_error")] + assert [item["id"] for item in direct["results"]] == expected_ids + for case in cases: + result = actual[case["id"]] + if case.get("expect_error"): + assert result.get("error"), result + else: + assert "error" not in result and result["token_ids"], result + assert len(result["token_ids"]) <= case["config"]["max_new_tokens"] + assert all(type(token) is int for token in result["token_ids"]) + for reference in direct["results"]: + result = actual[reference["id"]] + assert "error" not in reference, reference + assert result["token_ids"] == reference["token_ids"], (result, reference) + assert result["text"] == reference["text"], (result, reference) + if reference["id"] == "eos": + assert reference["finish_reason"] == 1, ( + "EOS case exhausted its budget instead of stopping" + ) + assert actual["raw"]["token_ids"] == actual["raw_after_rejections"]["token_ids"] + assert actual["raw"]["text"] == actual["raw_after_rejections"]["text"] + + +def hf_reference(checkpoint: Path, cases: list[dict], output: Path) -> dict: + """Generate a CPU FP32 reference with official HF APIs and family prompt rendering.""" + import torch + import transformers + from transformers import AutoModelForImageTextToText, AutoTokenizer + + from families.qwen3_5.tests.test_e2e import _render_prompt + + tokenizer = AutoTokenizer.from_pretrained(checkpoint, local_files_only=True) + model = ( + AutoModelForImageTextToText.from_pretrained( + checkpoint, local_files_only=True, dtype=torch.float32 + ) + .eval() + .to("cpu") + ) + report = { + "device": "cpu", + "precision": "fp32", + "checkpoint": str(checkpoint), + "transformers_version": transformers.__version__, + "results": [], + } + with torch.inference_mode(): + for case in cases: + if case.get("expect_error") or case.get("equal_to"): + continue + config = case["config"] + inputs = _render_prompt(tokenizer, case["prompt"], config).to("cpu") + generated = model.generate( + **inputs, + max_new_tokens=config["max_new_tokens"], + do_sample=False, + repetition_penalty=config["repetition_penalty"], + ) + tokens = generated[0, inputs["input_ids"].shape[1] :].tolist() + report["results"].append( + { + "id": case["id"], + "token_ids": tokens, + "text": tokenizer.decode(tokens, skip_special_tokens=True).strip(), + } + ) + write_json(output, report) + return report + + +def compare_hf(checkpoint: Path, public: dict, reference: dict, cases: list[dict]) -> None: + """Apply the existing family correctness assertions and default NED threshold unchanged.""" + from transformers import AutoTokenizer + + from families.qwen3_5.tests.test_e2e import _assert_correctness + + tokenizer = AutoTokenizer.from_pretrained(checkpoint, local_files_only=True) + actual = {item["id"]: item for item in public["results"]} + expected = {item["id"]: item for item in reference["results"]} + for case in cases: + if case.get("expect_error") or case.get("equal_to"): + continue + result, oracle = actual[case["id"]], expected[case["id"]] + decoded = tokenizer.decode(result["token_ids"], skip_special_tokens=True).strip() + _assert_correctness( + result, case["config"], {}, oracle["token_ids"], oracle["text"], None, decoded + ) + + +def validate(args: argparse.Namespace) -> None: + """Generate retained proof artifacts, raising rather than weakening any failed check.""" + args.output.mkdir(parents=True, exist_ok=False) + assets = args.output / "assets" + marker = extract_bundle(args.bundle, assets) + public_cases, direct_cases = requests(marker["max_sequence_length"]) + write_json(args.output / "public-requests.json", public_cases) + write_json(args.output / "direct-requests.json", direct_cases) + run_driver( + [ + str(args.runtime_root / "families/qwen3_5/qwen3_5_edge_inference"), + "--bundle", + str(args.bundle), + "--runtime-root", + str(args.runtime_root), + "--requests", + str(args.output / "public-requests.json"), + "--output", + str(args.output / "public.json"), + ], + args.output / "public.log", + ) + run_driver( + [ + str(args.runtime_root / "families/qwen3_5/qwen3_5_edge_reference"), + "--engine-dir", + str(assets / "edge_llm/engine"), + "--checkpoint-dir", + str(assets / "edge_llm/checkpoint"), + "--plugin-path", + str(args.runtime_root / "libNvInfer_edgellm_plugin.so"), + "--requests", + str(args.output / "direct-requests.json"), + "--output", + str(args.output / "direct.json"), + ], + args.output / "direct.log", + ) + public = json.loads((args.output / "public.json").read_text()) + direct = json.loads((args.output / "direct.json").read_text()) + compare(public, direct, public_cases) + hf_status = "not_run" + if args.hf_checkpoint: + if args.hf_reference: + reference = json.loads(args.hf_reference.read_text()) + write_json(args.output / "hf-reference.json", reference) + else: + reference = hf_reference( + args.hf_checkpoint, public_cases, args.output / "hf-reference.json" + ) + compare_hf(args.hf_checkpoint, public, reference, public_cases) + hf_status = "passed_existing_family_criteria" + write_json( + args.output / "evidence.json", + { + "status": "passed", + "edge_revision": EDGE_REVISION, + "target": marker["target"], + "checks": [ + "direct_api_greedy_token_and_text_parity", + "raw_and_chat", + "eos", + "unsupported_controls", + "capacity_rejection", + "persistent_repeat_after_rejections", + "self_contained_bundle", + ], + "hf_reference": hf_status, + }, + ) + + +def main() -> None: + """Parse explicit local assets and execute validation; never fetch dependencies.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--bundle", type=Path, required=True) + parser.add_argument("--runtime-root", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument( + "--hf-checkpoint", + type=Path, + help="Optionally compare CPU FP32 HF output using existing family criteria", + ) + parser.add_argument( + "--hf-reference", + type=Path, + help="Reuse a precomputed hf_reference() report with --hf-checkpoint", + ) + args = parser.parse_args() + args.bundle = args.bundle.resolve() + args.runtime_root = args.runtime_root.resolve() + args.output = args.output.resolve() + if args.hf_reference and not args.hf_checkpoint: + parser.error("--hf-reference requires --hf-checkpoint") + if args.hf_checkpoint: + args.hf_checkpoint = args.hf_checkpoint.resolve() + validate(args) + + +if __name__ == "__main__": + main() diff --git a/families/qwen3_5/tests/test_edge_llm.py b/families/qwen3_5/tests/test_edge_llm.py new file mode 100644 index 0000000000..3402837f25 --- /dev/null +++ b/families/qwen3_5/tests/test_edge_llm.py @@ -0,0 +1,452 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Family-owned Edge selection, API mapping and fault-isolation contracts.""" + +from __future__ import annotations + +from dataclasses import replace +import importlib +import json +import logging +from pathlib import Path +import shutil +import struct +import subprocess +from unittest.mock import Mock + +import pytest + +from tensorrt_model_connect.build import BuildRequest +from tensorrt_model_connect.bundle_writer import BundleWriter +from families.qwen3_5 import dispatch, edge_llm + +FAMILY = "qwen3_5" +NEWER_VARIANT = False +TARGET = { + "os": "linux", "os_version": "24.04", "arch": "x86_64", "sm": 80, + "cuda_version": "13.3", "tensorrt_version": "11.1.0.106", +} +ENGINE_FILES = ("llm.engine", "config.json", "tokenizer.json", "tokenizer_config.json", + "processed_chat_template.json") + + +@pytest.fixture +def inputs(tmp_path): + checkpoint = tmp_path / "checkpoint" + checkpoint.mkdir() + text = {"linear_key_head_dim": 128, "linear_value_head_dim": 128, + "max_position_embeddings": 32768} + if NEWER_VARIANT: + text["output_gate_type"] = "sigmoid" + raw = {"model_type": "qwen3_5", "text_config": text} + (checkpoint / "config.json").write_text(json.dumps(raw)) + (checkpoint / "model.safetensors").write_bytes(b"original checkpoint contents") + (checkpoint / "tokenizer.json").write_text("{}") + (checkpoint / "tokenizer_config.json").write_text("{}") + (checkpoint / "unrelated.log").write_text("not part of deployment") + request = BuildRequest(checkpoint, tmp_path / "model.bundle", FAMILY, + "text_generation", "fp16", max_sequence_length=1024) + return request, raw + + +def route(monkeypatch, prepare): + monkeypatch.setattr(edge_llm, "local_target", lambda: dict(TARGET)) + monkeypatch.setattr(dispatch, "EDGE_DISPATCH", {("linux", "x86_64", 80, "fp16"): prepare}) + + +def test_supported_family_config_matches(inputs): + request, raw = inputs + assert dispatch.candidate(request, raw) + assert dispatch.EDGE_DISPATCH[("linux", "x86_64", 80, "fp16")] is edge_llm.prepare + + +@pytest.mark.parametrize("updates", [ + {"precision": "fp32"}, {"backend": "trt_rtx"}, {"task": "image_generation"}, + {"quantization": "fp8"}, {"max_batch_size": 2}, {"tensor_parallel_size": 2}, + {"context_parallel_size": 2}, {"dynamic_kv_cache": True}, {"fp32_layers": (0,)}, + {"graph_transform": lambda *args: None}, {"image_height": 32}, + {"image_width": 32}, {"video_num_frames": 1}, +]) +def test_unmapped_build_options_stay_native(inputs, updates): + request, raw = inputs + assert not dispatch.candidate(replace(request, **updates), raw) + + +@pytest.mark.parametrize("model_type", ["qwen2", "qwen3", "qwen3_5_moe", "llama"]) +def test_old_or_other_model_types_are_not_edge_candidates(inputs, model_type): + request, raw = inputs + raw["model_type"] = model_type + assert not dispatch.candidate(request, raw) + + +@pytest.mark.parametrize("updates", [ + {"linear_key_head_dim": 64}, {"linear_value_head_dim": 64}, {"num_experts": 8}, + {"quantization_config": {"quant_method": "awq"}}, +]) +def test_unsupported_checkpoint_profiles_do_not_dispatch(inputs, updates): + request, raw = inputs + raw["text_config"].update(updates) + assert not dispatch.candidate(request, raw) + + +def test_family_variant_marker_is_not_cross_owned(inputs): + request, raw = inputs + if NEWER_VARIANT: + raw["text_config"].pop("output_gate_type") + else: + raw["text_config"]["output_gate_type"] = "sigmoid" + assert not dispatch.candidate(request, raw) + + +def test_nonmatch_calls_native_once_even_when_native_fails(inputs, monkeypatch): + request, _ = inputs + request = replace(request, precision="fp32") + failure = RuntimeError("native failed independently") + native = Mock(side_effect=failure) + discover = Mock(side_effect=AssertionError("must not discover Edge")) + monkeypatch.setattr(edge_llm, "local_target", discover) + writer = object() + with pytest.raises(RuntimeError) as caught: + dispatch.build(request, writer, native) + assert caught.value is failure + assert caught.value.__cause__ is None + native.assert_called_once_with(request, writer) + discover.assert_not_called() + + +def test_gpu_nonmatch_is_silent_native_once(inputs, monkeypatch, caplog): + request, _ = inputs + monkeypatch.setattr(edge_llm, "local_target", lambda: {**TARGET, "sm": 75}) + native, writer = Mock(), object() + dispatch.build(request, writer, native) + native.assert_called_once_with(request, writer) + assert not caplog.records + assert not list(request.output_path.parent.glob(".*.edge-*.log")) + + +@pytest.mark.parametrize("invalid", [[], {"text_config": []}]) +def test_invalid_common_config_does_not_attempt_either_builder(inputs, invalid, monkeypatch): + request, _ = inputs + (request.model_dir / "config.json").write_text(json.dumps(invalid)) + native, discovery = Mock(), Mock() + monkeypatch.setattr(edge_llm, "local_target", discovery) + with pytest.raises(ValueError): + dispatch.build(request, object(), native) + native.assert_not_called() + discovery.assert_not_called() + + +@pytest.mark.parametrize("failure", [FileNotFoundError("Edge missing"), RuntimeError("Edge failed")]) +def test_edge_failure_warns_before_one_unchanged_native_retry(inputs, monkeypatch, caplog, failure): + request, _ = inputs + prepare = Mock(side_effect=failure) + route(monkeypatch, prepare) + writer = object() + calls = [] + + def native(actual_request, actual_writer): + assert actual_request is request and actual_writer is writer + assert any(record.levelno == logging.WARNING and "Retrying native once" in record.message + for record in caplog.records) + calls.append(actual_request) + + dispatch.build(request, writer, native) + assert calls == [request] + prepare.assert_called_once() + logs = list(request.output_path.parent.glob(".*.edge-*.log")) + assert len(logs) == 1 and str(failure) in logs[0].read_text() + + +def test_native_failure_preserves_original_edge_cause(inputs, monkeypatch): + request, _ = inputs + edge_failure = RuntimeError("upstream builder") + native_failure = ValueError("native builder") + route(monkeypatch, Mock(side_effect=edge_failure)) + native = Mock(side_effect=native_failure) + with pytest.raises(ValueError) as caught: + dispatch.build(request, object(), native) + assert caught.value is native_failure + assert caught.value.__cause__ is edge_failure + native.assert_called_once() + + +@pytest.mark.parametrize("cancel", [KeyboardInterrupt(), SystemExit(130)]) +def test_cancellation_never_falls_back(inputs, monkeypatch, cancel): + request, _ = inputs + route(monkeypatch, Mock(side_effect=cancel)) + native = Mock() + with pytest.raises(type(cancel)): + dispatch.build(request, object(), native) + native.assert_not_called() + + +def test_success_publishes_once_without_native(inputs, monkeypatch): + request, _ = inputs + files, marker = {}, {"version": 1} + route(monkeypatch, Mock(return_value=(files, marker))) + publish, native, writer = Mock(), Mock(), object() + monkeypatch.setattr(edge_llm, "publish", publish) + dispatch.build(request, writer, native) + publish.assert_called_once_with(request, writer, files, marker) + native.assert_not_called() + + +def test_publication_failure_does_not_retry_native(inputs, monkeypatch): + request, _ = inputs + route(monkeypatch, Mock(return_value=({}, {}))) + failure = OSError("disk full during publication") + monkeypatch.setattr(edge_llm, "publish", Mock(side_effect=failure)) + native = Mock() + with pytest.raises(OSError) as caught: + dispatch.build(request, object(), native) + assert caught.value is failure + native.assert_not_called() + + +@pytest.fixture +def package(tmp_path, monkeypatch): + prefix = tmp_path / "install" + (prefix / "share/trtmc").mkdir(parents=True) + (prefix / "bin").mkdir() + (prefix / "lib").mkdir() + (prefix / "bin/python").write_bytes(b"python") + (prefix / "lib/plugin.so").write_bytes(b"plugin") + metadata = {"schema_version": 1, "version": "0.10.1", "revision": edge_llm.EDGE_REVISION, + "arch": "x86_64", "architectures": [80], "cuda_version": "13.3", + "tensorrt_version": TARGET["tensorrt_version"], + "python": "bin/python", "plugin": "lib/plugin.so"} + manifest = prefix / "share/trtmc/edge-llm.json" + manifest.write_text(json.dumps(metadata)) + monkeypatch.setattr(edge_llm, "cmake_prefixes", lambda: [prefix]) + return prefix, manifest, metadata + + +def test_package_resolution_uses_pinned_contained_artifacts(package): + prefix, _, _ = package + found = edge_llm.installed_package(TARGET) + assert found["python"] == str(prefix / "bin/python") + assert found["plugin"] == str(prefix / "lib/plugin.so") + + +@pytest.mark.parametrize("updates", [ + {"schema_version": 2}, {"version": "0.10.0"}, {"revision": "main"}, + {"arch": "aarch64"}, {"architectures": [86]}, {"cuda_version": "13.2"}, + {"tensorrt_version": "11.1.0.105"}, {"python": "/usr/bin/python3"}, + {"plugin": "../outside.so"}, +]) +def test_package_pin_platform_and_path_mismatches_are_rejected(package, updates): + _, manifest, metadata = package + manifest.write_text(json.dumps({**metadata, **updates})) + with pytest.raises(ValueError): + edge_llm.installed_package(TARGET) + + +def test_package_symlink_escape_is_rejected(package, tmp_path): + prefix, _, _ = package + target = tmp_path / "outside" + target.write_bytes(b"outside") + plugin = prefix / "lib/plugin.so" + plugin.unlink() + plugin.symlink_to(target) + with pytest.raises(ValueError, match="contained"): + edge_llm.installed_package(TARGET) + + +def test_missing_package_and_artifact_fail_without_installing(package): + prefix, manifest, _ = package + (prefix / "lib/plugin.so").unlink() + with pytest.raises(FileNotFoundError, match="missing"): + edge_llm.installed_package(TARGET) + manifest.unlink() + with pytest.raises(FileNotFoundError, match="not installed"): + edge_llm.installed_package(TARGET) + + +def fake_upstream(command, *, check, stdout, stderr, cwd): + assert check is True and stderr is subprocess.STDOUT + assert command[1:4] == ["-I", "-c", "from experimental.builder.cli import main; main()"] + options = dict(zip(command[4::2], command[5::2])) + assert options["--components"] == "llm" + assert options["--max-input-len"] == options["--max-kv-cache-capacity"] == "1024" + assert options["--max-batch-size"] == "1" + assert options["--dense"] == "fp16" + assert options["--externalize-weights"] == "all" + checkpoint = Path(options["--model-dir"]) + assert checkpoint.is_relative_to(cwd) + assert (checkpoint / "model.safetensors").read_bytes() == b"original checkpoint contents" + engine = Path(options["--engine-dir"]) + engine.mkdir(parents=True) + for name in ENGINE_FILES: + (engine / name).write_bytes(b"upstream artifact") + stdout.write("upstream API called\n") + return subprocess.CompletedProcess(command, 0) + + +def test_prepare_maps_api_and_preserves_private_checkpoint(inputs, package, tmp_path, monkeypatch): + request, raw = inputs + upstream = Mock(side_effect=fake_upstream) + monkeypatch.setattr(edge_llm.subprocess, "run", upstream) + staging = tmp_path / "stage" + files, marker = edge_llm.prepare(request, raw, TARGET, staging, tmp_path / "edge.log") + upstream.assert_called_once() + assert upstream.call_args.args[0][0] == str(package[0] / "bin/python") + assert marker["target"] == TARGET and marker["edge_revision"] == edge_llm.EDGE_REVISION + assert marker["artifacts"] == list(files) + assert marker["precision"] == "fp16" and marker["max_sequence_length"] == 1024 + assert all(name.startswith("edge_llm/") for name in files) + assert "edge_llm/checkpoint/unrelated.log" not in files + shutil.rmtree(request.model_dir) + assert files["edge_llm/checkpoint/model.safetensors"].read_bytes() == b"original checkpoint contents" + + +@pytest.mark.parametrize("missing", ENGINE_FILES) +def test_incomplete_upstream_output_is_rejected(inputs, package, tmp_path, monkeypatch, missing): + request, raw = inputs + + def incomplete(command, **kwargs): + result = fake_upstream(command, **kwargs) + Path(command[command.index("--engine-dir") + 1], missing).unlink() + return result + + monkeypatch.setattr(edge_llm.subprocess, "run", incomplete) + with pytest.raises(ValueError, match="required artifact"): + edge_llm.prepare(request, raw, TARGET, tmp_path / "stage", tmp_path / "edge.log") + + +def test_upstream_process_failure_propagates(inputs, package, tmp_path, monkeypatch): + request, raw = inputs + failure = subprocess.CalledProcessError(1, ["edge"], output="compile failed") + monkeypatch.setattr(edge_llm.subprocess, "run", Mock(side_effect=failure)) + with pytest.raises(subprocess.CalledProcessError) as caught: + edge_llm.prepare(request, raw, TARGET, tmp_path / "stage", tmp_path / "edge.log") + assert caught.value is failure + + +def test_publish_streams_self_contained_bundle(inputs, package, tmp_path, monkeypatch): + request, raw = inputs + monkeypatch.setattr(edge_llm.subprocess, "run", fake_upstream) + files, marker = edge_llm.prepare(request, raw, TARGET, tmp_path / "stage", tmp_path / "edge.log") + writer = BundleWriter(request.output_path) + real_copy = shutil.copyfileobj + copies = [] + + def stream(source, destination, length): + assert length == 1024 * 1024 + copies.append(source.name) + real_copy(source, destination, length) + + monkeypatch.setattr(edge_llm.shutil, "copyfileobj", stream) + edge_llm.publish(request, writer, files, marker) + assert len(copies) == len(files) + monkeypatch.setattr(edge_llm.shutil, "copyfileobj", real_copy) + writer.finish() + shutil.rmtree(request.model_dir) + shutil.rmtree(tmp_path / "stage") + with request.output_path.open("rb") as bundle: + assert bundle.read(8) == b"BUNDLE\x01\x00" + header = json.loads(bundle.read(struct.unpack(" B{Family configuration and request match?} + B -->|No| N[Original native builder] + B -->|Yes| C[Detect executing GPU and SDK] + C --> D{Family platform map matches?} + D -->|No| N + D -->|Yes| E[Resolve installed pinned Edge package] + E --> F[Invoke upstream builder main and validate assets] + F --> P[Publish Edge bundle] + C -->|Discovery error| W[Warn and retain diagnostics] + E -->|Dependency error| W + F -->|Builder or artifact error| W + W --> N + P --> R[Persistent Edge inference APIs] +``` + +The initial profile is dense unquantized FP16 text generation with batch/TP/CP 1 +and 128-dimensional GDN state. The owning family recognizes Qwen3.8 via its +configuration markers (`output_gate_type` present, `mlp_only_layers` absent), +not a numerical comparison of model names. Quantized, transformed, multimodal +and other unmapped configurations retain native behavior. + +After setting `CMAKE_PREFIX_PATH` to the installed package, ordinary +`python -m tensorrt_model_connect build MODEL --precision fp16 -o MODEL.bundle` +selects the adapter automatically for a matching native platform. Build the +runtime with `TRTMC_ENABLE_EDGELLM=ON` and use ordinary `trtmc run` commands. + +The adapter maps arguments to `experimental.builder.cli.main`; Edge owns all +model conversion and engine/artifact construction. The runtime directly calls +`LLMInferenceRuntime`, `countPromptTokens` and `handleRequest`, retaining one +instance across requests. Token limits, sampling and chat/thinking controls map +to Edge arguments. Unsupported non-default controls fail explicitly. + +Ordinary Edge preparation failures emit a warning and retain a `.NAME.edge-*.log` +beside the output before exactly one native retry with the unchanged request. +Native failure chains the Edge cause. Cancellation, publication errors and +runtime/inference failures do not trigger fallback. + +Both native-only and Edge-enabled runtime compilation and CPU contract tests +have passed. **Real Qwen3.8 GPU inference is not yet validated**: the available +A30 cannot hold the selected 27B checkpoint in the initial FP16 profile. The +Qwen3.5-0.8B GPU result must not be presented as Qwen3.8 quality evidence. diff --git a/families/qwen3_8/dispatch.py b/families/qwen3_8/dispatch.py new file mode 100644 index 0000000000..382f9db807 --- /dev/null +++ b/families/qwen3_8/dispatch.py @@ -0,0 +1,109 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Family-owned complete-network route map; native is the default.""" + +from __future__ import annotations + +import json +import logging +import os +from pathlib import Path +import tempfile +import traceback + +from . import edge_llm + +_LOG = logging.getLogger(__name__) + +# Explicit native platform routes supported by pinned Edge. No version-name +# comparison, nearest-GPU matching, shared registry, or legacy-Qwen entries. +EDGE_DISPATCH = { + ("linux", architecture, sm, "fp16"): edge_llm.prepare + for architecture, sms in ( + ("x86_64", (80, 86, 100, 120)), + ("aarch64", (87, 110, 121)), + ) + for sm in sms +} + + +def candidate(request, raw: dict) -> bool: + """Return whether this family's model/request contract can delegate to Edge.""" + config = raw.get("text_config", raw) + return ( + isinstance(config, dict) + and raw.get("model_type") == "qwen3_5" + and ("output_gate_type" in config and "mlp_only_layers" not in config) + and config.get("linear_key_head_dim") == config.get("linear_value_head_dim") == 128 + and not config.get("num_experts") + and not raw.get("quantization_config") and not config.get("quantization_config") + and not any( + (Path(request.model_dir) / name).exists() + for name in ("hf_quant_config.json", "quantize_config.json", "quant_config.json") + ) + and request.backend == "trt" and request.task == "text_generation" + and request.precision.lower() == "fp16" + and request.quantization in {None, "none"} + and request.max_batch_size == request.tensor_parallel_size == request.context_parallel_size == 1 + and not request.dynamic_kv_cache and not request.fp32_layers and request.graph_transform is None + and all(value is None for value in (request.image_height, request.image_width, request.video_num_frames)) + ) + + +def build(request, writer, native) -> None: + """Dispatch locally or warn and retry native once with the original request. + + Args: + request: Unmodified Model Connect build request. + writer: Unpublished bundle writer. + native: This family's original native builder callback. + + Raises: + Exception: Common input/publication error, or native build error with + Edge cause after a failed preparation. Cancellation never retries. + """ + raw = json.loads((Path(request.model_dir) / "config.json").read_text(encoding="utf-8")) + if not isinstance(raw, dict): + raise ValueError("checkpoint config.json must contain an object") + config = raw.get("text_config", raw) + if not isinstance(config, dict): + raise ValueError("checkpoint text_config must contain an object") + if not candidate(request, raw): + native(request, writer) + return + capacity = config.get("max_position_embeddings") + if type(capacity) is not int or capacity <= 0: + raise ValueError("checkpoint max_position_embeddings must be a positive integer") + if request.max_sequence_length and request.max_sequence_length > capacity: + raise ValueError("max_sequence_length exceeds checkpoint context capacity") + failure = None + descriptor, name = tempfile.mkstemp(prefix=f".{request.output_path.name}.edge-", suffix=".log", + dir=request.output_path.parent) + os.close(descriptor) + log_path = Path(name) + with tempfile.TemporaryDirectory(prefix="trtmc-qwen3_8-edge-") as directory: + try: + target = edge_llm.local_target() + key = (target["os"], target["arch"], target["sm"], request.precision.lower()) + adapter = EDGE_DISPATCH.get(key) + if adapter is not None: + files, marker = adapter(request, raw, target, Path(directory), log_path) + except Exception as error: + failure = error + with log_path.open("a", encoding="utf-8") as log: + traceback.print_exception(error, file=log) + _LOG.warning("qwen3_8 Edge build failed: %s. Diagnostics: %s. " + "Retrying native once with the unchanged request.", error, log_path, exc_info=True) + else: + # Edge preparation did not touch writer; publication cannot fallback. + if adapter is not None: + edge_llm.publish(request, writer, files, marker) + return + log_path.unlink() # A platform non-match is not an Edge failure. + try: + native(request, writer) + except Exception as error: + if failure is not None: + raise error from failure + raise diff --git a/families/qwen3_8/edge_llm.py b/families/qwen3_8/edge_llm.py new file mode 100644 index 0000000000..6490c4725e --- /dev/null +++ b/families/qwen3_8/edge_llm.py @@ -0,0 +1,123 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Thin family-owned adapter to the pinned Edge direct-builder API.""" + +from __future__ import annotations + +import json +from pathlib import Path +import shutil +import subprocess + +from tensorrt_model_connect.build import cmake_prefixes, detect_local_platform + +EDGE_REVISION = "e8b29522938901f6df19ebeedd4b69bc8edbcd97" + + +def local_target() -> dict: + """Return the executing worker identity supplied by generic build mechanics.""" + return detect_local_platform() + + +def installed_package(target: dict) -> dict: + """Resolve CMake installation via standard prefixes; never install anything. + + Args: + target: Executing device and SDK identity. + + Returns: + Validated package metadata with absolute Python and plugin paths. + + Raises: + FileNotFoundError: No CMake installation or required artifact exists. + ValueError: Pin, architecture, SDK or contained-path contract differs. + """ + for prefix in cmake_prefixes(): + manifest = prefix / "share/trtmc/edge-llm.json" + if not manifest.is_file(): + continue + package = json.loads(manifest.read_text(encoding="utf-8")) + if package.get("schema_version") != 1 or package.get("revision") != EDGE_REVISION: + raise ValueError(f"Edge package has an unsupported revision/schema: {manifest}") + if package.get("version") != "0.10.1" or package.get("arch") != target["arch"]: + raise ValueError("Edge package version/architecture differs from executing worker") + if target["sm"] not in package.get("architectures", []): + raise ValueError("Edge package was not built for this local GPU") + cuda_version = ".".join(str(package.get("cuda_version", "")).split(".")[:2]) + if cuda_version != target["cuda_version"] or package.get("tensorrt_version") != target["tensorrt_version"]: + raise ValueError("Edge package CUDA/TensorRT differs from executing worker") + for name in ("python", "plugin"): + relative = Path(package[name]) + path = (prefix / relative).resolve() + if relative.is_absolute() or not path.is_relative_to(prefix.resolve()): + raise ValueError(f"Edge package {name} must be contained in its installation") + if not path.is_file(): + raise FileNotFoundError(f"Edge package {name} is missing: {path}") + package[name] = str(path) + return package + raise FileNotFoundError("Edge-LLM is not installed; enable the optional Edge-LLM CMake dependency " + "and set CMAKE_PREFIX_PATH to its install prefix") + + +def prepare(request, raw: dict, target: dict, staging: Path, log_path: Path) -> tuple[dict, dict]: + """Map the request to Edge main(argv), returning complete unpublished assets. + + Edge owns model selection, configuration, conversion, graphs and engine + composition. The family adapter only maps text-generation arguments and + preserves the checkpoint needed by Edge external-weight APIs. + + Returns: + (section-name to file mapping, runtime marker). + + Raises: + Exception: Dependency, upstream build or artifact validation failed. + """ + package = installed_package(target) + checkpoint = staging / "edge_llm/checkpoint" + checkpoint.mkdir(parents=True) + for source in Path(request.model_dir).iterdir(): + if source.is_file() and (source.suffix in {".json", ".safetensors", ".model", ".jinja"} + or source.name in {"merges.txt", "vocab.txt"}): + shutil.copy2(source, checkpoint / source.name) + if not list(checkpoint.glob("*.safetensors")): + raise ValueError("Edge direct builder requires a safetensors checkpoint") + config = raw.get("text_config", raw) + limit = request.max_sequence_length or min(int(config["max_position_embeddings"]), 256) + engine = staging / "edge_llm/engine" + # Calling upstream main preserves its complete build/artifact orchestration. + command = [package["python"], "-I", "-c", + "from experimental.builder.cli import main; main()", + "--model-dir", str(checkpoint), "--engine-dir", str(engine), + "--components", "llm", "--plugin-path", package["plugin"], + "--dense", "fp16", "--max-input-len", str(limit), + "--max-kv-cache-capacity", str(limit), "--max-batch-size", "1", + "--externalize-weights", "all"] + if request.verbose: + command.append("--verbose") + with log_path.open("a", encoding="utf-8") as log: + subprocess.run(command, check=True, stdout=log, stderr=subprocess.STDOUT, cwd=staging) + for name in ("llm.engine", "config.json", "tokenizer.json", "tokenizer_config.json", "processed_chat_template.json"): + if not (engine / name).is_file() or (engine / name).stat().st_size == 0: + raise ValueError(f"Edge builder did not produce required artifact: {name}") + files = {} + for directory in (engine, checkpoint): + for path in sorted(directory.rglob("*")): + if path.is_symlink(): + raise ValueError(f"Edge output must not contain symlinks: {path}") + if path.is_file(): + files[path.relative_to(staging).as_posix()] = path + return files, { + "version": 1, "edge_revision": EDGE_REVISION, "target": target, "precision": "fp16", + "max_sequence_length": limit, "max_input_length": limit, "max_batch_size": 1, + "artifacts": list(files), + } + + +def publish(request, writer, files: dict, marker: dict) -> None: + """Stream complete Edge sections; publication errors must not retry native.""" + writer.set_header(family=request.family, task=request.task, backend=request.backend) + for name, path in files.items(): + with path.open("rb") as source, writer.open_section(name) as destination: + shutil.copyfileobj(source, destination, length=1024 * 1024) + writer.add_json("edge_llm.json", marker) diff --git a/families/qwen3_8/model.py b/families/qwen3_8/model.py index bfe571aa04..17060d0b24 100644 --- a/families/qwen3_8/model.py +++ b/families/qwen3_8/model.py @@ -80,7 +80,7 @@ def _runtime_config(model_dir: Path, config: ModelConfig, model: Qwen38Model, ** return {name: runtime[name] for name in fields} -def build(request, writer) -> None: +def _build_native(request, writer) -> None: """Build one Qwen3.8 hybrid text-generation bundle.""" if request.dynamic_kv_cache: raise NotImplementedError("qwen3_8 does not support dynamic_kv_cache") @@ -148,3 +148,10 @@ def build(request, writer) -> None: path = model_dir / filename if path.is_file(): writer.add_bytes(filename, path.read_bytes()) + + +def build(request, writer) -> None: + """Select a family-owned complete-network adapter or the native builder.""" + from .dispatch import build as dispatch_build + + dispatch_build(request, writer, _build_native) diff --git a/families/qwen3_8/runtime/CMakeLists.txt b/families/qwen3_8/runtime/CMakeLists.txt index af423a3801..abd643455e 100644 --- a/families/qwen3_8/runtime/CMakeLists.txt +++ b/families/qwen3_8/runtime/CMakeLists.txt @@ -21,7 +21,10 @@ target_link_libraries(trtmc_model_qwen3_8 PRIVATE nlohmann_json::nlohmann_json ${TRTMC_CUDART_LIBRARY} ) -target_compile_options(trtmc_model_qwen3_8 PRIVATE -Wall -Wextra -Wpedantic) +target_compile_options(trtmc_model_qwen3_8 PRIVATE + "$<$:-Wall;-Wextra;-Wpedantic>" + "$<$:-Xcompiler=-Wall,-Wextra>" +) set_target_properties(trtmc_model_qwen3_8 PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" BUILD_RPATH "\$ORIGIN" @@ -71,3 +74,75 @@ if(TRTMC_BUILD_TESTS) SKIP_RETURN_CODE 77 ) endif() + +# Complete-network offload is family-owned and absent from native-only builds. +if(TARGET EdgeLLM::Core) + target_sources(trtmc_model_qwen3_8 PRIVATE + edge_llm/adapter.cpp + edge_llm/device_link.cu + ) + target_compile_definitions(trtmc_model_qwen3_8 PRIVATE TRTMC_HAS_EDGE_LLM=1) + target_link_libraries(trtmc_model_qwen3_8 PRIVATE EdgeLLM::Core) + set_target_properties(trtmc_model_qwen3_8 PROPERTIES + CUDA_ARCHITECTURES "${EdgeLLM_CUDA_ARCHITECTURE}" + CUDA_SEPARABLE_COMPILATION ON + CUDA_RESOLVE_DEVICE_SYMBOLS ON + ) +endif() + +if(TRTMC_BUILD_TESTS) + add_executable(test_qwen3_8_edge_contract + ${PROJECT_SOURCE_DIR}/families/qwen3_8/tests/cpp/test_qwen3_8_edge_contract.cpp + ) + target_include_directories(test_qwen3_8_edge_contract PRIVATE + ${PROJECT_SOURCE_DIR} + ${PROJECT_SOURCE_DIR}/core/runtime/include + ) + add_test(NAME test_qwen3_8_edge_contract COMMAND test_qwen3_8_edge_contract) +endif() + +if(TARGET EdgeLLM::Core) + add_custom_command(TARGET trtmc_model_qwen3_8 POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + $ $ + VERBATIM + ) + if(TRTMC_BUILD_TESTS) + add_executable(test_qwen3_8_edge_request + ${PROJECT_SOURCE_DIR}/families/qwen3_8/tests/cpp/test_qwen3_8_edge_request.cpp + ) + target_include_directories(test_qwen3_8_edge_request PRIVATE + ${PROJECT_SOURCE_DIR} + ${PROJECT_SOURCE_DIR}/core/runtime/include + ) + target_link_libraries(test_qwen3_8_edge_request PRIVATE EdgeLLM::Core) + add_test(NAME test_qwen3_8_edge_request COMMAND test_qwen3_8_edge_request) + endif() +endif() + +if(TRTMC_BUILD_TESTS) + add_executable(test_qwen3_8_edge_factory + ${PROJECT_SOURCE_DIR}/families/qwen3_8/tests/cpp/test_qwen3_8_edge_factory.cpp + ) + target_include_directories(test_qwen3_8_edge_factory PRIVATE + ${PROJECT_SOURCE_DIR}/core/runtime/include + ${TRTMC_CUDA_INCLUDE_DIR} + ) + target_link_libraries(test_qwen3_8_edge_factory PRIVATE + trtmc_model_qwen3_8 trtmc_core nlohmann_json::nlohmann_json + ) + if(TARGET EdgeLLM::Core) + target_compile_definitions(test_qwen3_8_edge_factory PRIVATE TRTMC_HAS_EDGE_LLM=1) + endif() + add_test(NAME test_qwen3_8_edge_factory COMMAND test_qwen3_8_edge_factory) +endif() + +# Explicit GPU integration driver, not an unprovisioned default CTest. +if(TRTMC_BUILD_TESTS) + add_executable(qwen3_8_edge_inference + ${PROJECT_SOURCE_DIR}/families/qwen3_8/tests/cpp/test_qwen3_8_edge_inference.cpp + ) + target_link_libraries(qwen3_8_edge_inference PRIVATE + trtmc_runtime trtmc_core nlohmann_json::nlohmann_json + ) +endif() diff --git a/families/qwen3_8/runtime/edge_llm/adapter.cpp b/families/qwen3_8/runtime/edge_llm/adapter.cpp new file mode 100644 index 0000000000..719900fb72 --- /dev/null +++ b/families/qwen3_8/runtime/edge_llm/adapter.cpp @@ -0,0 +1,224 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +#include "families/qwen3_8/runtime/edge_llm/adapter.h" + +#include "families/qwen3_8/runtime/edge_llm/request.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace trtmc::qwen3_8::edge_llm { +namespace { +namespace fs = std::filesystem; + +/// Turn CUDA failures into caller-visible load or inference errors. +void check_cuda(cudaError_t result) { + if (result != cudaSuccess) + throw std::runtime_error(std::string("Qwen3.8 Edge CUDA error: ") + + cudaGetErrorString(result)); +} + +/// Reject an engine built for a different local GPU or CUDA/TensorRT runtime. +void validate_target(const nlohmann::json& target) { + utsname host{}; + if (uname(&host) != 0) + throw std::runtime_error("Cannot identify Qwen3.8 Edge runtime host"); + std::ifstream release("/etc/os-release"); + std::string line, os_version; + while (std::getline(release, line)) { + if (line.rfind("VERSION_ID=", 0) == 0) { + os_version = line.substr(11); + if (os_version.size() >= 2 && os_version.front() == char(34) && + os_version.back() == char(34)) + os_version = os_version.substr(1, os_version.size() - 2); + } + } + int device = 0, cuda_version = 0; + check_cuda(cudaGetDevice(&device)); + check_cuda(cudaRuntimeGetVersion(&cuda_version)); + cudaDeviceProp gpu{}; + check_cuda(cudaGetDeviceProperties(&gpu, device)); + const int trt_version = getInferLibVersion(); + const std::string trt = + std::to_string(trt_version / 10000) + "." + std::to_string((trt_version % 10000) / 100) + + "." + std::to_string(trt_version % 100) + "." + std::to_string(getInferLibBuildVersion()); + const std::string cuda = + std::to_string(cuda_version / 1000) + "." + std::to_string((cuda_version % 1000) / 10); + if (target.at("os") != "linux" || target.at("os_version") != os_version || + target.at("arch") != host.machine || target.at("sm") != gpu.major * 10 + gpu.minor || + target.at("cuda_version") != cuda || target.at("tensorrt_version") != trt) + throw std::runtime_error( + "Qwen3.8 Edge bundle requires its build GPU and CUDA/TensorRT stack"); +} + +/// Own extracted engine/checkpoint files until after the Edge runtime is destroyed. +class Artifacts { + public: + explicit Artifacts(const BundleReader& bundle, const nlohmann::json& marker) { + std::set names; + for (const auto& entry : marker.at("artifacts")) { + const auto name = entry.get(); + if (!safe_artifact_path(name) || !names.insert(name).second || + !bundle.find_section(name)) + throw std::runtime_error("Invalid Qwen3.8 Edge artifact: " + name); + } + for (const auto* required : + {"edge_llm/engine/llm.engine", "edge_llm/engine/config.json", + "edge_llm/engine/tokenizer.json", "edge_llm/engine/tokenizer_config.json", + "edge_llm/engine/processed_chat_template.json", "edge_llm/checkpoint/config.json"}) + if (!names.count(required) || bundle.find_section(required)->length == 0) + throw std::runtime_error(std::string("Required Qwen3.8 Edge artifact missing: ") + + required); + std::string pattern = (fs::temp_directory_path() / "trtmc-qwen3_8-edge-XXXXXX").string(); + if (!mkdtemp(pattern.data())) + throw std::runtime_error("Cannot create Qwen3.8 Edge artifact directory"); + root_ = pattern; + try { + for (const auto& name : names) { + const auto destination = root_ / name; + fs::create_directories(destination.parent_path()); + std::ofstream output(destination, std::ios::binary); + bundle.copy_section(name, output); + output.close(); + if (!output) + throw std::runtime_error("Cannot extract Qwen3.8 Edge artifact: " + name); + } + } catch (...) { + cleanup(); + throw; + } + } + ~Artifacts() { cleanup(); } + Artifacts(const Artifacts&) = delete; + Artifacts& operator=(const Artifacts&) = delete; + std::string engine() const { return (root_ / "edge_llm/engine").string(); } + std::string checkpoint() const { return (root_ / "edge_llm/checkpoint").string(); } + + private: + void cleanup() noexcept { + std::error_code ignored; + fs::remove_all(root_, ignored); + } + fs::path root_; +}; + +/// Close the plugin handle after runtime destruction; registrations remain mapped. +struct CloseLibrary { + void operator()(void* handle) const noexcept { + if (handle) + dlclose(handle); + } +}; + +/// Initialize the CMake-installed adjacent plugin without process-global environment mutation. +std::unique_ptr load_plugin() { + Dl_info location{}; + if (!dladdr(reinterpret_cast(&create), &location) || !location.dli_fname) + throw std::runtime_error("Cannot locate Qwen3.8 family library"); + const auto path = + fs::absolute(location.dli_fname).parent_path() / "libNvInfer_edgellm_plugin.so"; + std::unique_ptr plugin( + dlopen(path.c_str(), RTLD_NOW | RTLD_GLOBAL | RTLD_NODELETE)); + if (!plugin) + throw std::runtime_error("Cannot load CMake-installed Edge plugin: " + + std::string(dlerror())); + using Initialize = bool (*)(void*, const char*); + auto initialize = reinterpret_cast(dlsym(plugin.get(), "initEdgellmPlugins")); + if (!initialize || !initialize(static_cast(&trt_edgellm::gLogger), "")) + throw std::runtime_error("Cannot initialize Qwen3.8 Edge plugin"); + return plugin; +} + +/// Stream ownership is independent of construction success and outlives the Edge instance. +class Stream { + public: + Stream() { check_cuda(cudaStreamCreateWithFlags(&value_, cudaStreamNonBlocking)); } + ~Stream() { cudaStreamDestroy(value_); } + Stream(const Stream&) = delete; + Stream& operator=(const Stream&) = delete; + cudaStream_t get() const { return value_; } + + private: + cudaStream_t value_{nullptr}; +}; + +/// Thin persistent Edge API adapter; serialization prevents concurrent use of Edge request state. +class EdgeTask final : public ITextGeneration { + public: + EdgeTask(const BundleReader& bundle, const nlohmann::json& marker) + : artifacts_(bundle, marker), plugin_(load_plugin()), + runtime_(artifacts_.engine(), "", std::unordered_map{}, + stream_.get(), trt_edgellm::rt::ContextCacheConfig{}, artifacts_.checkpoint()), + capacity_(marker.at("max_sequence_length").get()), + input_limit_(marker.at("max_input_length").get()) {} + + std::int32_t default_max_new_tokens() const override { return std::min(128, capacity_ - 1); } + + /// Drain work from failed requests before destroying the runtime and its weight buffers. + ~EdgeTask() override { cudaStreamSynchronize(stream_.get()); } + + /// Invoke Edge once; failures propagate without attempting native inference. + TextResult generate(const std::string& prompt, const TextGenerationConfig& config) override { + auto request = make_request(prompt, config, default_max_new_tokens()); + std::lock_guard lock(mutex_); + const auto counts = runtime_.countPromptTokens(request); + if (counts.size() != 1) + throw std::runtime_error("Qwen3.8 Edge returned invalid prompt counts"); + validate_capacity(counts.front(), input_limit_, capacity_, request.maxGenerateLength); + trt_edgellm::rt::LLMGenerationResponse response{}; + if (!runtime_.handleRequest(request, response, stream_.get()) || + response.outputIds.size() != 1 || response.outputTexts.size() != 1 || + response.outputIds.front().empty() || + response.outputIds.front().size() > static_cast(request.maxGenerateLength)) + throw std::runtime_error("Qwen3.8 Edge generation failed"); + if (response.finishReasons.size() != 1 || + (response.finishReasons.front() != trt_edgellm::rt::FinishReason::kEndId && + response.finishReasons.front() != trt_edgellm::rt::FinishReason::kLength)) + throw std::runtime_error("Qwen3.8 Edge generation did not complete successfully"); + // This API does not expose per-request stage times; zero means unavailable. + return {std::move(response.outputTexts.front()), std::move(response.outputIds.front())}; + } + + private: + // Reverse destruction order keeps weights, plugin and stream alive throughout Edge teardown. + Artifacts artifacts_; + std::unique_ptr plugin_; + Stream stream_; + trt_edgellm::rt::LLMInferenceRuntime runtime_; + int capacity_; + int input_limit_; + std::mutex mutex_; +}; +} // namespace + +ITask* create(const BundleReader& bundle) { + const auto bytes = bundle.read_section("edge_llm.json"); + const auto marker = nlohmann::json::parse(bytes.begin(), bytes.end()); + if (marker.at("version") != 1 || marker.at("edge_revision") != kRevision || + marker.at("max_sequence_length").get() <= 1 || + marker.at("max_input_length").get() <= 0 || + marker.at("max_input_length").get() > marker.at("max_sequence_length").get() || + marker.at("max_batch_size") != 1 || marker.at("precision") != "fp16" || + !marker.at("artifacts").is_array()) + throw std::runtime_error("Invalid Qwen3.8 Edge bundle contract"); + validate_target(marker.at("target")); + return new EdgeTask(bundle, marker); +} + +} // namespace trtmc::qwen3_8::edge_llm diff --git a/families/qwen3_8/runtime/edge_llm/adapter.h b/families/qwen3_8/runtime/edge_llm/adapter.h new file mode 100644 index 0000000000..86cf5a89ce --- /dev/null +++ b/families/qwen3_8/runtime/edge_llm/adapter.h @@ -0,0 +1,15 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include "trtmc/bundle.h" +#include "trtmc/task.h" + +namespace trtmc::qwen3_8::edge_llm { + +/// Create a persistent Edge task from a self-contained bundle; throws on load failure. +ITask* create(const BundleReader& bundle); + +} // namespace trtmc::qwen3_8::edge_llm diff --git a/families/qwen3_8/runtime/edge_llm/contract.h b/families/qwen3_8/runtime/edge_llm/contract.h new file mode 100644 index 0000000000..21c0a80761 --- /dev/null +++ b/families/qwen3_8/runtime/edge_llm/contract.h @@ -0,0 +1,56 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include "trtmc/task.h" + +#include +#include +#include +#include + +namespace trtmc::qwen3_8::edge_llm { + +inline constexpr const char* kRevision = "e8b29522938901f6df19ebeedd4b69bc8edbcd97"; + +/// Return whether an artifact is a normalized file below one of the two Edge roots. +inline bool safe_artifact_path(const std::string& name) { + if (name.find('\\') != std::string::npos || name.find('\0') != std::string::npos) + return false; + const std::filesystem::path path(name); + if (path.is_absolute() || path.filename().empty()) + return false; + for (const auto& part : path) + if (part == "." || part == "..") + return false; + return path.generic_string() == name && + (name.rfind("edge_llm/engine/", 0) == 0 || name.rfind("edge_llm/checkpoint/", 0) == 0); +} + +/// Reject invalid sampling settings and controls with no equivalent Edge request API. +inline void validate_generation(const TextGenerationConfig& c) { + if (!std::isfinite(c.temperature) || c.temperature < 0 || !std::isfinite(c.top_p) || + c.top_p <= 0 || c.top_p > 1 || c.top_k < 0) + throw std::invalid_argument("Invalid Qwen3.8 Edge sampling parameters"); + if (c.min_p != 0 || c.seed != -1 || c.eos_token_id != -1 || c.repetition_penalty != 1 || + !c.lora_adapter_id.empty() || c.stop_on_boxed_answer || + (c.text_generation_mode != "auto" && c.text_generation_mode != "autoregressive") || + c.source_language_token_id != -1 || c.forced_bos_token_id != -1 || c.guidance_scale != -1 || + c.cfg_scale != -1 || c.num_steps != -1 || c.sde_gamma != -1 || !c.initial_latents.empty() || + !c.condition_latents.empty() || !c.condition_mask.empty() || !c.sampling_steps.empty() || + !c.sde_noises.empty() || c.block_length != 0 || c.confidence_threshold != -1) + throw std::invalid_argument( + "Requested generation controls are unsupported by Qwen3.8 Edge"); +} + +/// Enforce prompt and total capacity without allowing Edge to silently clip generation. +inline void validate_capacity(int prompt_tokens, int input_limit, int capacity, + std::int64_t generated_tokens) { + if (prompt_tokens <= 0 || prompt_tokens > input_limit || generated_tokens <= 0 || + generated_tokens > static_cast(capacity) - prompt_tokens) + throw std::invalid_argument("Qwen3.8 Edge prompt and generation exceed bundle capacity"); +} + +} // namespace trtmc::qwen3_8::edge_llm diff --git a/families/qwen3_8/runtime/edge_llm/device_link.cu b/families/qwen3_8/runtime/edge_llm/device_link.cu new file mode 100644 index 0000000000..1ea4768d60 --- /dev/null +++ b/families/qwen3_8/runtime/edge_llm/device_link.cu @@ -0,0 +1,5 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +// Enable the final CUDA device-link step for Edge's static runtime dependencies. diff --git a/families/qwen3_8/runtime/edge_llm/request.h b/families/qwen3_8/runtime/edge_llm/request.h new file mode 100644 index 0000000000..ec471afa30 --- /dev/null +++ b/families/qwen3_8/runtime/edge_llm/request.h @@ -0,0 +1,29 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include "families/qwen3_8/runtime/edge_llm/contract.h" + +#include + +namespace trtmc::qwen3_8::edge_llm { + +/// Map Model Connect text arguments to the pinned Edge API; rejects unmapped controls. +inline trt_edgellm::rt::LLMGenerationRequest +make_request(const std::string& prompt, const TextGenerationConfig& config, int default_length) { + validate_generation(config); + trt_edgellm::rt::LLMGenerationRequest request{}; + request.requests.resize(1); + request.requests.front().messages.push_back({"user", {{"text", prompt}}}); + request.applyChatTemplate = config.use_chat_template; + request.enableThinking = config.enable_thinking; + request.temperature = config.temperature; + request.topK = config.top_k; + request.topP = config.top_p; + request.maxGenerateLength = config.max_new_tokens > 0 ? config.max_new_tokens : default_length; + return request; +} + +} // namespace trtmc::qwen3_8::edge_llm diff --git a/families/qwen3_8/runtime/plugin.cpp b/families/qwen3_8/runtime/plugin.cpp index 49a8b4c57f..114a0f4533 100644 --- a/families/qwen3_8/runtime/plugin.cpp +++ b/families/qwen3_8/runtime/plugin.cpp @@ -9,6 +9,9 @@ #include "families/qwen3_8/runtime/plugin_helpers.h" #include "families/qwen3_8/runtime/recurrent_state.h" #include "trtmc/runtime/family_factory.h" +#ifdef TRTMC_HAS_EDGE_LLM +#include "families/qwen3_8/runtime/edge_llm/adapter.h" +#endif #include #include @@ -119,6 +122,14 @@ std::string chat_template(const BundleReader& bundle) { } // namespace ITask* create(const FamilyContext& context) { + if (context.reader.find_section("edge_llm.json")) { +#ifdef TRTMC_HAS_EDGE_LLM + return edge_llm::create(context.reader); +#else + throw std::runtime_error("Qwen3.8 Edge bundle requires a runtime configured with " + "-DTRTMC_ENABLE_EDGELLM=ON; rebuild and install Model Connect"); +#endif + } const RuntimeConfig config = parse_runtime_config(context.reader); auto decoder = load_engine(context.backend, require_section(context.reader, "engine.plan"), "qwen3_8 decoder"); diff --git a/families/qwen3_8/tests/cpp/test_qwen3_8_edge_contract.cpp b/families/qwen3_8/tests/cpp/test_qwen3_8_edge_contract.cpp new file mode 100644 index 0000000000..2d091aa443 --- /dev/null +++ b/families/qwen3_8/tests/cpp/test_qwen3_8_edge_contract.cpp @@ -0,0 +1,92 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +#include "families/qwen3_8/runtime/edge_llm/contract.h" + +#include +#include +#include +#include + +namespace edge = trtmc::qwen3_8::edge_llm; +namespace { +int failures = 0; + +/// Record a failed behavioral assertion without depending on NDEBUG. +void check(bool condition, const char* message) { + if (!condition) { + std::cerr << message << std::endl; + ++failures; + } +} + +/// Verify unsupported input is rejected by the public argument validation boundary. +void rejects(const std::function& operation) { + try { + operation(); + check(false, "Expected invalid_argument"); + } catch (const std::invalid_argument&) { + } +} +} // namespace + +int main() { + trtmc::TextGenerationConfig config; + edge::validate_generation(config); + config.temperature = 0; + config.use_chat_template = true; + config.enable_thinking = false; + config.text_generation_mode = "autoregressive"; + edge::validate_generation(config); + using Change = std::function; + const std::vector unsupported{ + [](auto& c) { c.temperature = -1; }, + [](auto& c) { c.temperature = std::numeric_limits::quiet_NaN(); }, + [](auto& c) { c.top_p = 0; }, + [](auto& c) { c.top_p = 1.01F; }, + [](auto& c) { c.top_p = std::numeric_limits::infinity(); }, + [](auto& c) { c.top_k = -1; }, + [](auto& c) { c.min_p = 0.1F; }, + [](auto& c) { c.seed = 42; }, + [](auto& c) { c.eos_token_id = 9; }, + [](auto& c) { c.repetition_penalty = 1.1F; }, + [](auto& c) { c.lora_adapter_id = "adapter"; }, + [](auto& c) { c.stop_on_boxed_answer = true; }, + [](auto& c) { c.text_generation_mode = "diffusion"; }, + [](auto& c) { c.source_language_token_id = 0; }, + [](auto& c) { c.forced_bos_token_id = 0; }, + [](auto& c) { c.guidance_scale = 0; }, + [](auto& c) { c.cfg_scale = 0; }, + [](auto& c) { c.num_steps = 0; }, + [](auto& c) { c.sde_gamma = 0; }, + [](auto& c) { c.initial_latents = {1}; }, + [](auto& c) { c.condition_latents = {1}; }, + [](auto& c) { c.condition_mask = {1}; }, + [](auto& c) { c.sampling_steps = {1}; }, + [](auto& c) { c.sde_noises = {1}; }, + [](auto& c) { c.block_length = 1; }, + [](auto& c) { c.confidence_threshold = 0.5F; }, + }; + for (const auto& change : unsupported) { + auto invalid = config; + change(invalid); + rejects([&] { edge::validate_generation(invalid); }); + } + edge::validate_capacity(5, 1024, 1024, 1019); + rejects([] { edge::validate_capacity(5, 1024, 1024, 1020); }); + rejects([] { edge::validate_capacity(1024, 512, 2048, 1); }); + rejects([] { edge::validate_capacity(0, 1024, 1024, 1); }); + rejects([] { edge::validate_capacity(5, 1024, 1024, 0); }); + rejects([] { edge::validate_capacity(5, 1024, 1024, INT64_MAX); }); + for (const auto* name : {"edge_llm/engine/llm.engine", "edge_llm/checkpoint/model.safetensors", + "edge_llm/engine/rank0/config.json"}) + check(edge::safe_artifact_path(name), "Valid artifact path rejected"); + for (const auto* name : + {"/tmp/engine", "edge_llm/engine/../../outside", "edge_llm/engine/./file", + "edge_llm/engine//file", "edge_llm/engine/", "other/file", "edge_llm/engine/a\\b"}) + check(!edge::safe_artifact_path(name), "Unsafe artifact path accepted"); + check(!edge::safe_artifact_path(std::string("edge_llm/engine/a\0b", 19)), + "Embedded null artifact accepted"); + return failures == 0 ? 0 : 1; +} diff --git a/families/qwen3_8/tests/cpp/test_qwen3_8_edge_factory.cpp b/families/qwen3_8/tests/cpp/test_qwen3_8_edge_factory.cpp new file mode 100644 index 0000000000..04976fd2f1 --- /dev/null +++ b/families/qwen3_8/tests/cpp/test_qwen3_8_edge_factory.cpp @@ -0,0 +1,95 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +#include "trtmc/runtime/family_factory.h" +#include "trtmc/runtime/trt_backend.h" + +#include +#include +#include +#include +#include + +namespace { +/// A routing test must fail before attempting any native engine load. +class RejectBackend final : public trtmc::IBackend { + public: + std::unique_ptr create_module(const void*, size_t, + const trtmc::ModuleCreateOptions&) override { + throw std::logic_error("unexpected engine load"); + } + std::unique_ptr + create_module_prebound(const void*, size_t, const trtmc::ModuleCreateOptions&, + const std::vector&) override { + throw std::logic_error("unexpected prebound engine load"); + } + trtmc::BackendDualProfileModules + create_dual_profile_modules(const void*, size_t, const trtmc::ModuleCreateOptions&) override { + throw std::logic_error("unexpected dual engine load"); + } + const char* name() const override { return "trt"; } +}; + +/// Write a deliberately incomplete bundle to observe which factory owns its error. +void write_bundle(const std::filesystem::path& path, bool edge) { + nlohmann::json sections = nlohmann::json::object(); + if (edge) + sections["edge_llm.json"] = {{"offset", 0}, {"length", 2}}; + const auto header = nlohmann::json{ + {"format", 1}, + {"family", "qwen3_8"}, + {"task", "text_generation"}, + {"backend", "trt"}, + {"sections", + sections}}.dump(); + std::ofstream out(path, std::ios::binary); + out.write("BUNDLE\x01\x00", 8); + const std::uint64_t size = header.size(); + for (int shift = 0; shift < 64; shift += 8) + out.put(static_cast((size >> shift) & 0xff)); + out << header; + if (edge) + out << "{}"; + if (!out) + throw std::runtime_error("Cannot write factory test bundle"); +} +} // namespace + +int main() { + char pattern[] = "/tmp/trtmc_qwen3_8_factory_XXXXXX"; + const auto* directory = mkdtemp(pattern); + if (!directory) + return 1; + const auto path = std::filesystem::path(directory) / "test.bundle"; + RejectBackend backend; + int failures = 0; + for (const bool edge : {false, true}) { + write_bundle(path, edge); + trtmc::BundleReader bundle(path.string()); + try { + std::unique_ptr task(trtmc_create_family({bundle, backend, 0})); + ++failures; + } catch (const std::exception& error) { + const std::string message = error.what(); + if (!edge) { + if (message.find("runtime.json") == std::string::npos) + ++failures; + } else { +#ifdef TRTMC_HAS_EDGE_LLM + // The malformed Edge marker must not reach native metadata parsing. + if (message.find("runtime.json") != std::string::npos || + message.find("unexpected") != std::string::npos) + ++failures; +#else + if (message.find("TRTMC_ENABLE_EDGELLM=ON") == std::string::npos) + ++failures; +#endif + } + } + } + std::filesystem::remove_all(directory); + if (failures) + std::cerr << "Family Edge/native factory routing failed" << std::endl; + return failures == 0 ? 0 : 1; +} diff --git a/families/qwen3_8/tests/cpp/test_qwen3_8_edge_inference.cpp b/families/qwen3_8/tests/cpp/test_qwen3_8_edge_inference.cpp new file mode 100644 index 0000000000..22d2e79a6c --- /dev/null +++ b/families/qwen3_8/tests/cpp/test_qwen3_8_edge_inference.cpp @@ -0,0 +1,142 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +#include "trtmc/bundle.h" +#include "trtmc/runtime/family_loader.h" +#include "trtmc/task.h" + +#include +#include +#include +#include +#include +#include + +namespace { +/// Apply only explicitly named public configuration fields; unknown test inputs are errors. +trtmc::TextGenerationConfig generation_config(const nlohmann::json& values) { + trtmc::TextGenerationConfig result; + if (!values.is_object()) + throw std::invalid_argument("config must be an object"); + for (const auto& [key, value] : values.items()) { + if (key == "max_new_tokens") + result.max_new_tokens = value.get(); + else if (key == "temperature") + result.temperature = value.get(); + else if (key == "top_k") + result.top_k = value.get(); + else if (key == "top_p") + result.top_p = value.get(); + else if (key == "min_p") + result.min_p = value.get(); + else if (key == "seed") + result.seed = value.get(); + else if (key == "eos_token_id") + result.eos_token_id = value.get(); + else if (key == "use_chat_template") + result.use_chat_template = value.get(); + else if (key == "enable_thinking") + result.enable_thinking = value.get(); + else if (key == "text_generation_mode") + result.text_generation_mode = value.get(); + else if (key == "stop_on_boxed_answer") + result.stop_on_boxed_answer = value.get(); + else if (key == "lora_adapter_id") + result.lora_adapter_id = value.get(); + else if (key == "repetition_penalty") + result.repetition_penalty = value.get(); + else if (key == "block_length") + result.block_length = value.get(); + else if (key == "confidence_threshold") + result.confidence_threshold = value.get(); + else if (key == "forced_bos_token_id") + result.forced_bos_token_id = value.get(); + else if (key == "source_language_token_id") + result.source_language_token_id = value.get(); + else + throw std::invalid_argument("Unknown public generation field: " + key); + } + return result; +} + +/// Parse required qualification inputs without an environment or runtime-root fallback. +std::map arguments(int argc, char** argv) { + std::map result; + for (int i = 1; i < argc; i += 2) { + if (i + 1 >= argc || !result.emplace(argv[i], argv[i + 1]).second) + throw std::invalid_argument("Expected unique --option VALUE pairs"); + } + for (const auto* name : {"--bundle", "--runtime-root", "--requests", "--output"}) + if (!result.count(name)) + throw std::invalid_argument(std::string("Missing ") + name); + if (result.size() != 4) + throw std::invalid_argument("Unknown qualification option"); + return result; +} +} // namespace + +/// Exercise the public load/generate path once, retaining one task across the complete case list. +int main(int argc, char** argv) { + try { + const auto args = arguments(argc, argv); + const trtmc::BundleReader bundle(args.at("--bundle")); + if (bundle.info().family != "qwen3_8" || !bundle.find_section("edge_llm.json")) + throw std::runtime_error("Expected a qwen3_8 Edge bundle, not native fallback"); + nlohmann::json report{{"backend", "native"}, {"results", nlohmann::json::array()}}; + if (bundle.find_section("edge_llm.json")) { + const auto bytes = bundle.read_section("edge_llm.json"); + const auto marker = nlohmann::json::parse(bytes.begin(), bytes.end()); + report["backend"] = "edge_llm"; + report["edge_revision"] = marker.at("edge_revision"); + } + std::ifstream request_file(args.at("--requests")); + const auto requests = nlohmann::json::parse(request_file); + if (!requests.is_array() || requests.empty()) + throw std::invalid_argument("Requests must be a nonempty JSON array"); + auto task = trtmc::load_task(args.at("--bundle"), args.at("--runtime-root")); + auto* generator = dynamic_cast(task.get()); + if (!generator) + throw std::runtime_error("Bundle does not expose ITextGeneration"); + bool passed = true; + std::map previous; + for (const auto& request : requests) { + nlohmann::json output{{"id", request.at("id")}}; + try { + const auto config = + generation_config(request.value("config", nlohmann::json::object())); + const auto result = + generator->generate(request.at("prompt").get(), config); + output["text"] = result.text; + output["token_ids"] = result.token_ids; + } catch (const std::exception& error) { + output["error"] = error.what(); + } + if (output.contains("error") != request.value("expect_error", false)) + passed = false; + if (request.contains("equal_to")) { + const auto earlier = previous.find(request.at("equal_to").get()); + if (earlier == previous.end() || output.contains("error") || + earlier->second.value("text", "") != output.value("text", "") || + earlier->second.value("token_ids", nlohmann::json::array()) != + output.value("token_ids", nlohmann::json::array())) + passed = false; + } + if (!output.contains("error") && output.at("token_ids").empty()) + passed = false; + previous[request.at("id").get()] = output; + report["results"].push_back(std::move(output)); + } + report["passed"] = passed; + task.reset(); + std::ofstream output(args.at("--output")); + output << report.dump(2) << '\n'; + output.close(); + if (!output) + throw std::runtime_error("Cannot write qualification output"); + return passed ? 0 : 1; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/families/qwen3_8/tests/cpp/test_qwen3_8_edge_request.cpp b/families/qwen3_8/tests/cpp/test_qwen3_8_edge_request.cpp new file mode 100644 index 0000000000..4e40b543e7 --- /dev/null +++ b/families/qwen3_8/tests/cpp/test_qwen3_8_edge_request.cpp @@ -0,0 +1,41 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +#include "families/qwen3_8/runtime/edge_llm/request.h" + +#include + +int main() { + trtmc::TextGenerationConfig config; + config.max_new_tokens = 27; + config.temperature = 0.7F; + config.top_k = 12; + config.top_p = 0.8F; + config.use_chat_template = true; + config.enable_thinking = false; + const auto request = trtmc::qwen3_8::edge_llm::make_request("Hello", config, 128); + if (request.requests.size() != 1 || request.requests[0].messages.size() != 1 || + request.requests[0].messages[0].role != "user" || + request.requests[0].messages[0].contents[0].content != "Hello" || + request.temperature != config.temperature || request.topK != config.top_k || + request.topP != config.top_p || request.maxGenerateLength != config.max_new_tokens || + !request.applyChatTemplate || request.enableThinking || !request.addGenerationPrompt || + request.saveSystemPromptKVCache || !request.loraWeightsName.empty()) { + std::cerr << "Model Connect arguments did not map to the pinned Edge request" << std::endl; + return 1; + } + config.max_new_tokens = 0; + config.temperature = 0; + config.use_chat_template = false; + const auto raw = trtmc::qwen3_8::edge_llm::make_request("raw", config, 32); + if (raw.maxGenerateLength != 32 || raw.temperature != 0 || raw.applyChatTemplate) + return 1; + config.seed = 42; + try { + trtmc::qwen3_8::edge_llm::make_request("unsupported", config, 32); + return 1; + } catch (const std::invalid_argument&) { + } + return 0; +} diff --git a/families/qwen3_8/tests/test_edge_llm.py b/families/qwen3_8/tests/test_edge_llm.py new file mode 100644 index 0000000000..7d1cd815f8 --- /dev/null +++ b/families/qwen3_8/tests/test_edge_llm.py @@ -0,0 +1,452 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Family-owned Edge selection, API mapping and fault-isolation contracts.""" + +from __future__ import annotations + +from dataclasses import replace +import importlib +import json +import logging +from pathlib import Path +import shutil +import struct +import subprocess +from unittest.mock import Mock + +import pytest + +from tensorrt_model_connect.build import BuildRequest +from tensorrt_model_connect.bundle_writer import BundleWriter +from families.qwen3_8 import dispatch, edge_llm + +FAMILY = "qwen3_8" +NEWER_VARIANT = True +TARGET = { + "os": "linux", "os_version": "24.04", "arch": "x86_64", "sm": 80, + "cuda_version": "13.3", "tensorrt_version": "11.1.0.106", +} +ENGINE_FILES = ("llm.engine", "config.json", "tokenizer.json", "tokenizer_config.json", + "processed_chat_template.json") + + +@pytest.fixture +def inputs(tmp_path): + checkpoint = tmp_path / "checkpoint" + checkpoint.mkdir() + text = {"linear_key_head_dim": 128, "linear_value_head_dim": 128, + "max_position_embeddings": 32768} + if NEWER_VARIANT: + text["output_gate_type"] = "sigmoid" + raw = {"model_type": "qwen3_5", "text_config": text} + (checkpoint / "config.json").write_text(json.dumps(raw)) + (checkpoint / "model.safetensors").write_bytes(b"original checkpoint contents") + (checkpoint / "tokenizer.json").write_text("{}") + (checkpoint / "tokenizer_config.json").write_text("{}") + (checkpoint / "unrelated.log").write_text("not part of deployment") + request = BuildRequest(checkpoint, tmp_path / "model.bundle", FAMILY, + "text_generation", "fp16", max_sequence_length=1024) + return request, raw + + +def route(monkeypatch, prepare): + monkeypatch.setattr(edge_llm, "local_target", lambda: dict(TARGET)) + monkeypatch.setattr(dispatch, "EDGE_DISPATCH", {("linux", "x86_64", 80, "fp16"): prepare}) + + +def test_supported_family_config_matches(inputs): + request, raw = inputs + assert dispatch.candidate(request, raw) + assert dispatch.EDGE_DISPATCH[("linux", "x86_64", 80, "fp16")] is edge_llm.prepare + + +@pytest.mark.parametrize("updates", [ + {"precision": "fp32"}, {"backend": "trt_rtx"}, {"task": "image_generation"}, + {"quantization": "fp8"}, {"max_batch_size": 2}, {"tensor_parallel_size": 2}, + {"context_parallel_size": 2}, {"dynamic_kv_cache": True}, {"fp32_layers": (0,)}, + {"graph_transform": lambda *args: None}, {"image_height": 32}, + {"image_width": 32}, {"video_num_frames": 1}, +]) +def test_unmapped_build_options_stay_native(inputs, updates): + request, raw = inputs + assert not dispatch.candidate(replace(request, **updates), raw) + + +@pytest.mark.parametrize("model_type", ["qwen2", "qwen3", "qwen3_8_moe", "llama"]) +def test_old_or_other_model_types_are_not_edge_candidates(inputs, model_type): + request, raw = inputs + raw["model_type"] = model_type + assert not dispatch.candidate(request, raw) + + +@pytest.mark.parametrize("updates", [ + {"linear_key_head_dim": 64}, {"linear_value_head_dim": 64}, {"num_experts": 8}, + {"quantization_config": {"quant_method": "awq"}}, +]) +def test_unsupported_checkpoint_profiles_do_not_dispatch(inputs, updates): + request, raw = inputs + raw["text_config"].update(updates) + assert not dispatch.candidate(request, raw) + + +def test_family_variant_marker_is_not_cross_owned(inputs): + request, raw = inputs + if NEWER_VARIANT: + raw["text_config"].pop("output_gate_type") + else: + raw["text_config"]["output_gate_type"] = "sigmoid" + assert not dispatch.candidate(request, raw) + + +def test_nonmatch_calls_native_once_even_when_native_fails(inputs, monkeypatch): + request, _ = inputs + request = replace(request, precision="fp32") + failure = RuntimeError("native failed independently") + native = Mock(side_effect=failure) + discover = Mock(side_effect=AssertionError("must not discover Edge")) + monkeypatch.setattr(edge_llm, "local_target", discover) + writer = object() + with pytest.raises(RuntimeError) as caught: + dispatch.build(request, writer, native) + assert caught.value is failure + assert caught.value.__cause__ is None + native.assert_called_once_with(request, writer) + discover.assert_not_called() + + +def test_gpu_nonmatch_is_silent_native_once(inputs, monkeypatch, caplog): + request, _ = inputs + monkeypatch.setattr(edge_llm, "local_target", lambda: {**TARGET, "sm": 75}) + native, writer = Mock(), object() + dispatch.build(request, writer, native) + native.assert_called_once_with(request, writer) + assert not caplog.records + assert not list(request.output_path.parent.glob(".*.edge-*.log")) + + +@pytest.mark.parametrize("invalid", [[], {"text_config": []}]) +def test_invalid_common_config_does_not_attempt_either_builder(inputs, invalid, monkeypatch): + request, _ = inputs + (request.model_dir / "config.json").write_text(json.dumps(invalid)) + native, discovery = Mock(), Mock() + monkeypatch.setattr(edge_llm, "local_target", discovery) + with pytest.raises(ValueError): + dispatch.build(request, object(), native) + native.assert_not_called() + discovery.assert_not_called() + + +@pytest.mark.parametrize("failure", [FileNotFoundError("Edge missing"), RuntimeError("Edge failed")]) +def test_edge_failure_warns_before_one_unchanged_native_retry(inputs, monkeypatch, caplog, failure): + request, _ = inputs + prepare = Mock(side_effect=failure) + route(monkeypatch, prepare) + writer = object() + calls = [] + + def native(actual_request, actual_writer): + assert actual_request is request and actual_writer is writer + assert any(record.levelno == logging.WARNING and "Retrying native once" in record.message + for record in caplog.records) + calls.append(actual_request) + + dispatch.build(request, writer, native) + assert calls == [request] + prepare.assert_called_once() + logs = list(request.output_path.parent.glob(".*.edge-*.log")) + assert len(logs) == 1 and str(failure) in logs[0].read_text() + + +def test_native_failure_preserves_original_edge_cause(inputs, monkeypatch): + request, _ = inputs + edge_failure = RuntimeError("upstream builder") + native_failure = ValueError("native builder") + route(monkeypatch, Mock(side_effect=edge_failure)) + native = Mock(side_effect=native_failure) + with pytest.raises(ValueError) as caught: + dispatch.build(request, object(), native) + assert caught.value is native_failure + assert caught.value.__cause__ is edge_failure + native.assert_called_once() + + +@pytest.mark.parametrize("cancel", [KeyboardInterrupt(), SystemExit(130)]) +def test_cancellation_never_falls_back(inputs, monkeypatch, cancel): + request, _ = inputs + route(monkeypatch, Mock(side_effect=cancel)) + native = Mock() + with pytest.raises(type(cancel)): + dispatch.build(request, object(), native) + native.assert_not_called() + + +def test_success_publishes_once_without_native(inputs, monkeypatch): + request, _ = inputs + files, marker = {}, {"version": 1} + route(monkeypatch, Mock(return_value=(files, marker))) + publish, native, writer = Mock(), Mock(), object() + monkeypatch.setattr(edge_llm, "publish", publish) + dispatch.build(request, writer, native) + publish.assert_called_once_with(request, writer, files, marker) + native.assert_not_called() + + +def test_publication_failure_does_not_retry_native(inputs, monkeypatch): + request, _ = inputs + route(monkeypatch, Mock(return_value=({}, {}))) + failure = OSError("disk full during publication") + monkeypatch.setattr(edge_llm, "publish", Mock(side_effect=failure)) + native = Mock() + with pytest.raises(OSError) as caught: + dispatch.build(request, object(), native) + assert caught.value is failure + native.assert_not_called() + + +@pytest.fixture +def package(tmp_path, monkeypatch): + prefix = tmp_path / "install" + (prefix / "share/trtmc").mkdir(parents=True) + (prefix / "bin").mkdir() + (prefix / "lib").mkdir() + (prefix / "bin/python").write_bytes(b"python") + (prefix / "lib/plugin.so").write_bytes(b"plugin") + metadata = {"schema_version": 1, "version": "0.10.1", "revision": edge_llm.EDGE_REVISION, + "arch": "x86_64", "architectures": [80], "cuda_version": "13.3", + "tensorrt_version": TARGET["tensorrt_version"], + "python": "bin/python", "plugin": "lib/plugin.so"} + manifest = prefix / "share/trtmc/edge-llm.json" + manifest.write_text(json.dumps(metadata)) + monkeypatch.setattr(edge_llm, "cmake_prefixes", lambda: [prefix]) + return prefix, manifest, metadata + + +def test_package_resolution_uses_pinned_contained_artifacts(package): + prefix, _, _ = package + found = edge_llm.installed_package(TARGET) + assert found["python"] == str(prefix / "bin/python") + assert found["plugin"] == str(prefix / "lib/plugin.so") + + +@pytest.mark.parametrize("updates", [ + {"schema_version": 2}, {"version": "0.10.0"}, {"revision": "main"}, + {"arch": "aarch64"}, {"architectures": [86]}, {"cuda_version": "13.2"}, + {"tensorrt_version": "11.1.0.105"}, {"python": "/usr/bin/python3"}, + {"plugin": "../outside.so"}, +]) +def test_package_pin_platform_and_path_mismatches_are_rejected(package, updates): + _, manifest, metadata = package + manifest.write_text(json.dumps({**metadata, **updates})) + with pytest.raises(ValueError): + edge_llm.installed_package(TARGET) + + +def test_package_symlink_escape_is_rejected(package, tmp_path): + prefix, _, _ = package + target = tmp_path / "outside" + target.write_bytes(b"outside") + plugin = prefix / "lib/plugin.so" + plugin.unlink() + plugin.symlink_to(target) + with pytest.raises(ValueError, match="contained"): + edge_llm.installed_package(TARGET) + + +def test_missing_package_and_artifact_fail_without_installing(package): + prefix, manifest, _ = package + (prefix / "lib/plugin.so").unlink() + with pytest.raises(FileNotFoundError, match="missing"): + edge_llm.installed_package(TARGET) + manifest.unlink() + with pytest.raises(FileNotFoundError, match="not installed"): + edge_llm.installed_package(TARGET) + + +def fake_upstream(command, *, check, stdout, stderr, cwd): + assert check is True and stderr is subprocess.STDOUT + assert command[1:4] == ["-I", "-c", "from experimental.builder.cli import main; main()"] + options = dict(zip(command[4::2], command[5::2])) + assert options["--components"] == "llm" + assert options["--max-input-len"] == options["--max-kv-cache-capacity"] == "1024" + assert options["--max-batch-size"] == "1" + assert options["--dense"] == "fp16" + assert options["--externalize-weights"] == "all" + checkpoint = Path(options["--model-dir"]) + assert checkpoint.is_relative_to(cwd) + assert (checkpoint / "model.safetensors").read_bytes() == b"original checkpoint contents" + engine = Path(options["--engine-dir"]) + engine.mkdir(parents=True) + for name in ENGINE_FILES: + (engine / name).write_bytes(b"upstream artifact") + stdout.write("upstream API called\n") + return subprocess.CompletedProcess(command, 0) + + +def test_prepare_maps_api_and_preserves_private_checkpoint(inputs, package, tmp_path, monkeypatch): + request, raw = inputs + upstream = Mock(side_effect=fake_upstream) + monkeypatch.setattr(edge_llm.subprocess, "run", upstream) + staging = tmp_path / "stage" + files, marker = edge_llm.prepare(request, raw, TARGET, staging, tmp_path / "edge.log") + upstream.assert_called_once() + assert upstream.call_args.args[0][0] == str(package[0] / "bin/python") + assert marker["target"] == TARGET and marker["edge_revision"] == edge_llm.EDGE_REVISION + assert marker["artifacts"] == list(files) + assert marker["precision"] == "fp16" and marker["max_sequence_length"] == 1024 + assert all(name.startswith("edge_llm/") for name in files) + assert "edge_llm/checkpoint/unrelated.log" not in files + shutil.rmtree(request.model_dir) + assert files["edge_llm/checkpoint/model.safetensors"].read_bytes() == b"original checkpoint contents" + + +@pytest.mark.parametrize("missing", ENGINE_FILES) +def test_incomplete_upstream_output_is_rejected(inputs, package, tmp_path, monkeypatch, missing): + request, raw = inputs + + def incomplete(command, **kwargs): + result = fake_upstream(command, **kwargs) + Path(command[command.index("--engine-dir") + 1], missing).unlink() + return result + + monkeypatch.setattr(edge_llm.subprocess, "run", incomplete) + with pytest.raises(ValueError, match="required artifact"): + edge_llm.prepare(request, raw, TARGET, tmp_path / "stage", tmp_path / "edge.log") + + +def test_upstream_process_failure_propagates(inputs, package, tmp_path, monkeypatch): + request, raw = inputs + failure = subprocess.CalledProcessError(1, ["edge"], output="compile failed") + monkeypatch.setattr(edge_llm.subprocess, "run", Mock(side_effect=failure)) + with pytest.raises(subprocess.CalledProcessError) as caught: + edge_llm.prepare(request, raw, TARGET, tmp_path / "stage", tmp_path / "edge.log") + assert caught.value is failure + + +def test_publish_streams_self_contained_bundle(inputs, package, tmp_path, monkeypatch): + request, raw = inputs + monkeypatch.setattr(edge_llm.subprocess, "run", fake_upstream) + files, marker = edge_llm.prepare(request, raw, TARGET, tmp_path / "stage", tmp_path / "edge.log") + writer = BundleWriter(request.output_path) + real_copy = shutil.copyfileobj + copies = [] + + def stream(source, destination, length): + assert length == 1024 * 1024 + copies.append(source.name) + real_copy(source, destination, length) + + monkeypatch.setattr(edge_llm.shutil, "copyfileobj", stream) + edge_llm.publish(request, writer, files, marker) + assert len(copies) == len(files) + monkeypatch.setattr(edge_llm.shutil, "copyfileobj", real_copy) + writer.finish() + shutil.rmtree(request.model_dir) + shutil.rmtree(tmp_path / "stage") + with request.output_path.open("rb") as bundle: + assert bundle.read(8) == b"BUNDLE\x01\x00" + header = json.loads(bundle.read(struct.unpack(" Date: Thu, 10 Sep 2026 23:03:52 +0000 Subject: [PATCH 2/2] test(architecture): follow family build delegation Keep the closed CMake inventory explicit for Edge provisioning. Follow bounded family-local request and invoked callback handling without crediting dead, shadowed, foreign or unrelated helpers. Preserve the mandatory field set and cover positive and negative delegation cases. Signed-off-by: Joshua Calafato --- tools/tests/test_architecture.py | 384 +++++++++++++++++++++++++++++-- 1 file changed, 369 insertions(+), 15 deletions(-) diff --git a/tools/tests/test_architecture.py b/tools/tests/test_architecture.py index 98b6c95b7c..13e2808f97 100644 --- a/tools/tests/test_architecture.py +++ b/tools/tests/test_architecture.py @@ -7,6 +7,9 @@ import importlib.util import json import re +from textwrap import dedent + +import pytest from pathlib import Path @@ -349,7 +352,16 @@ def test_shared_python_and_native_trees_are_closed_minimal_sets() -> None: "tools/tests/test_pr_metadata.py", "tools/tests/test_public_source_hygiene.py", } - expected_cmake = {"cmake/trtmcConfig.cmake.in"} + expected_cmake = { + "cmake/trtmcConfig.cmake.in", + "cmake/EdgeLLM.cmake", + "cmake/edgellm/CheckNative.cmake", + "cmake/edgellm/EdgeLLMConfig.cmake.in", + "cmake/edgellm/Install.cmake.in", + "cmake/edgellm/Prepare.cmake.in", + "cmake/edgellm/README.md", + "cmake/edgellm/tests/package_contract.cmake", + } expected_third_party = { "third_party/stb/stb_image.h", "third_party/stb/stb_image_resize2.h", @@ -530,6 +542,361 @@ def test_builders_publish_the_explicit_task_without_guessing() -> None: assert violations == [] +def _handled_build_request_fields(family: Path) -> set[str]: + """Return fields read along explicit, owning-family build request calls. + + Args: + family: Family directory containing model.py and any delegated helpers. + + Returns: + Fields accessed on the original request, including called native callbacks. + Imports and renamed parameters are supported; dynamic lookup, reassigned + values and nested definitions receive no credit. This is a conservative + syntactic call graph, not proof of execution through every runtime branch. + + Raises: + OSError, SyntaxError: A visited family module cannot be read or parsed. + """ + prefix = f"families.{family.name}" + request_value = "" + modules: dict[str, tuple[dict[str, ast.FunctionDef | ast.AsyncFunctionDef], dict[str, str]]] = {} + + def imported_names(node: ast.AST, package: str) -> dict[str, str]: + """Resolve explicit imports in package to bound names; other nodes return empty.""" + if isinstance(node, ast.Import): + return { + alias.asname or alias.name.split(".")[0]: _canonical_family_module( + alias.name if alias.asname else alias.name.split(".")[0] + ) + for alias in node.names + } + if not isinstance(node, ast.ImportFrom): + return {} + base = node.module or "" + if node.level: + try: + base = importlib.util.resolve_name("." * node.level + base, package) + except (ImportError, ValueError): + return {} + return { + alias.asname or alias.name: f"{_canonical_family_module(base)}.{alias.name}" + for alias in node.names if alias.name != "*" + } + + def module_symbols(module: str): + """Cache functions/imports for module, returning empty for non-family code.""" + if module not in modules: + modules[module] = ({}, {}) + if not module.startswith(prefix + "."): + return modules[module] + path = family.joinpath(*module.removeprefix(prefix + ".").split(".")).with_suffix(".py") + if not path.is_file(): + return modules[module] + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + functions, names = modules[module] + for node in tree.body: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + functions[node.name] = node + names[node.name] = f"{module}.{node.name}" + elif isinstance(node, (ast.Assign, ast.AnnAssign, ast.AugAssign, ast.ClassDef)): + overwritten = ( + {node.name} if isinstance(node, ast.ClassDef) else + {item.id for item in ast.walk(node) + if isinstance(item, ast.Name) and isinstance(item.ctx, ast.Store)} + ) + for bound_name in overwritten: + names.pop(bound_name, None) + functions.pop(bound_name, None) + imported = imported_names(node, module.rpartition(".")[0]) + for bound_name in imported: + functions.pop(bound_name, None) + names.update(imported) + return modules[module] + + pending = [(f"{prefix}.model.build", (("request", request_value),))] + visited = set() + handled: set[str] = set() + while pending: + state = pending.pop() + if state in visited: + continue + visited.add(state) + target, bindings = state + module, _, name = target.rpartition(".") + functions, globals_ = module_symbols(module) + function = functions.get(name) + if function is None: + continue + names = dict(globals_) + for argument in [*function.args.posonlyargs, *function.args.args, *function.args.kwonlyargs]: + names.pop(argument.arg, None) + names.update(bindings) + + def value(node: ast.AST) -> str | None: + """Resolve an expression through current request, function or import bindings.""" + parts = _attribute_path(node) + if not parts or parts[0] not in names: + return None + return ".".join((names[parts[0]], *parts[1:])) + + def scan(node: ast.AST) -> None: + """Visit executable syntax in order, queuing calls and killing overwritten names.""" + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + names.pop(node.name, None) + return + if isinstance(node, ast.Lambda): + return + if isinstance(node, ast.Attribute) and value(node.value) == request_value: + if isinstance(node.ctx, ast.Load): + handled.add(node.attr) + if isinstance(node, ast.Call) and (callee := value(node.func)): + called_module, _, called_name = callee.rpartition(".") + called = module_symbols(called_module)[0].get(called_name) + if called is not None: + arguments = [*called.args.posonlyargs, *called.args.args] + supplied = {arg.arg: value(expr) for arg, expr in zip(arguments, node.args)} + keyword_names = { + arg.arg for arg in [*called.args.args, *called.args.kwonlyargs] + } + supplied.update({ + kw.arg: value(kw.value) for kw in node.keywords if kw.arg in keyword_names + }) + # Only request identity and known callbacks flow onward; + # attributes are not requests and cannot grow recursive states. + bound = {} + for key, item in supplied.items(): + if item is None: + continue + owner, _, symbol = item.rpartition(".") + if item == request_value or symbol in module_symbols(owner)[0]: + bound[key] = item + if request_value in bound.values(): + pending.append((callee, tuple(sorted(bound.items())))) + names.update(imported_names(node, module.rpartition(".")[0])) + # Evaluate assignment RHS before invalidating request identity. No + # credit is given to a new object merely because it is named request. + if isinstance(node, (ast.Assign, ast.AnnAssign, ast.NamedExpr)): + if node.value is not None: + scan(node.value) + targets = node.targets if isinstance(node, ast.Assign) else [node.target] + for target_node in targets: + scan(target_node) + return + if isinstance(node, ast.Name) and isinstance(node.ctx, (ast.Store, ast.Del)): + names.pop(node.id, None) + for child in ast.iter_child_nodes(node): + scan(child) + + for statement in function.body: + scan(statement) + return handled + + +@pytest.mark.parametrize( + ("model", "helper", "expected"), + [ + ("def build(request, writer): return request.precision", "", {"precision"}), + ("async def build(request, writer): return request.precision", "", {"precision"}), + ( + """ + def native(original, writer): return original.precision + def build(request, writer): + from .helper import dispatch as route + route(request, writer, native) + """, + """ + def dispatch(incoming, writer, fallback): + incoming.task + fallback(original=incoming, writer=writer) + """, + {"precision", "task"}, + ), + ( + """ + from . import helper as adapter + def build(request, writer): adapter.check(request) + """, + "def check(config): return config.precision", + {"precision"}, + ), + ( + """ + from tensorrt_model_connect.families.owned.helper import check as validate + def build(request, writer): validate(config=request) + """, + "def check(*, config): return config.precision", + {"precision"}, + ), + ( + """ + from .helper import check + def build(request, writer): check(request) + """, + """ + def check(config): + config.precision + check(config) + """, + {"precision"}, + ), + ( + """ + def unused(request): return request.precision + def build(request, writer): return request.task + """, + "", + {"task"}, + ), + ( + """ + def build(request, writer): + def unused(request): return request.precision + return request.task + """, + "", + {"task"}, + ), + ( + """ + from .helper import check + def build(request, writer): check(writer) + """, + "def check(request): return request.precision", + set(), + ), + ( + """ + from families.other.helper import check + def build(request, writer): check(request) + """, + "def check(request): return request.precision", + set(), + ), + ( + """ + from tensorrt_model_connect.build import check + def build(request, writer): check(request) + """, + "def check(request): return request.precision", + set(), + ), + ( + """ + def native(request, writer): return request.precision + def build(request, writer): + from .helper import dispatch + dispatch(request, writer, native) + """, + "def dispatch(request, writer, native): return request.task", + {"task"}, + ), + ( + """ + def build(request, writer): + request.task + request = writer + return request.precision + """, + "", + {"task"}, + ), + ( + """ + from .helper import check + def build(request, writer): check(request) + """, + """ + def check(original): + request = object() + return request.precision + """, + set(), + ), + ], + ids=[ + "direct", "async-direct", "native-callback-and-renamed-parameters", "module-alias", + "absolute-import-and-keyword", "recursive-helper", "dead-helper", + "nested-dead-helper", "call-without-request", "other-family", "shared-helper", + "unused-callback", "reassigned-request", "unrelated-request-object", + ], +) +def test_build_request_field_reachability(tmp_path, model, helper, expected) -> None: + """Credit only fields reached with the original request inside its owning family.""" + family = tmp_path / "owned" + family.mkdir() + (family / "model.py").write_text(dedent(model), encoding="utf-8") + (family / "helper.py").write_text(dedent(helper), encoding="utf-8") + handled = _handled_build_request_fields(family) + assert handled == expected + + +@pytest.mark.parametrize("shadow", [ + "def check(request): pass", "class check: pass", "check = object()", + "from tensorrt_model_connect.build import check", +]) +@pytest.mark.parametrize("scope", ["local", "module", "imported-module"]) +def test_build_request_fields_ignore_shadowed_helpers(tmp_path, shadow, scope) -> None: + """Overwriting a callable must not credit the former callable or nested dead body.""" + family = tmp_path / "owned" + family.mkdir() + definition = "def check(request): return request.precision\n" + if scope == "imported-module": + model = "from .helper import check\ndef build(request, writer): check(request)\n" + helper = definition + shadow + "\n" + elif scope == "module": + model = definition + shadow + "\ndef build(request, writer): check(request)\n" + helper = "" + else: + model = definition + "def build(request, writer):\n " + shadow + "\n check(request)\n" + helper = "" + (family / "model.py").write_text(model, encoding="utf-8") + (family / "helper.py").write_text(helper, encoding="utf-8") + assert _handled_build_request_fields(family) == set() + + +def test_build_request_fields_ignore_undeclared_keyword(tmp_path) -> None: + """An undeclared keyword cannot bind a global request-looking object in the callee.""" + family = tmp_path / "owned" + family.mkdir() + (family / "model.py").write_text(dedent(""" + request = object() + def check(writer): return request.precision + def build(request, writer): check(request=request) + """), encoding="utf-8") + assert _handled_build_request_fields(family) == set() + + +def test_build_request_field_recursion_has_finite_bindings(tmp_path) -> None: + """Recursive attribute chains cannot generate unbounded symbolic call states.""" + family = tmp_path / "owned" + family.mkdir() + (family / "model.py").write_text(dedent(""" + def recurse(original, item): + original.precision + recurse(original, item.next) + def build(request, writer): recurse(request, request) + """), encoding="utf-8") + assert _handled_build_request_fields(family) == {"precision", "next"} + + +def test_build_request_contract_rejects_missing_delegated_field(tmp_path, monkeypatch) -> None: + """The mandatory-field gate still rejects missing handling, even with a dead helper.""" + family = tmp_path / "owned" + family.mkdir() + (family / "model.py").write_text(dedent(""" + def unused(request): return request.task + def native(original): return original.precision + def build(request, writer): native(request) + """), encoding="utf-8") + api = tmp_path / "core/builder/tensorrt_model_connect/build.py" + api.parent.mkdir(parents=True) + api.write_text("class BuildRequest:\n precision: str\n task: str\n", encoding="utf-8") + monkeypatch.setitem(globals(), "REPO", tmp_path) + monkeypatch.setitem(globals(), "family_dirs", lambda: [family]) + with pytest.raises(AssertionError, match="owned:task"): + test_every_builder_handles_every_family_owned_request_field() + + def test_every_builder_handles_every_family_owned_request_field() -> None: build_api = REPO / "core/builder/tensorrt_model_connect/build.py" build_api_tree = ast.parse(build_api.read_text(encoding="utf-8"), filename=str(build_api)) @@ -553,20 +920,7 @@ def test_every_builder_handles_every_family_owned_request_field() -> None: violations: list[str] = [] for family in family_dirs(): - model = family / "model.py" - tree = ast.parse(model.read_text(encoding="utf-8"), filename=str(model)) - build = next( - node - for node in tree.body - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == "build" - ) - handled = { - node.attr - for node in ast.walk(build) - if isinstance(node, ast.Attribute) - and isinstance(node.value, ast.Name) - and node.value.id == "request" - } + handled = _handled_build_request_fields(family) for field in sorted(family_owned_fields - handled): violations.append(f"{family.name}:{field}") assert violations == []