Add vision-only Edge-LLM ExecuTorch delegate integration

cpp/BUILD

@@ -119,6 +119,17 @@ cc_library(
     strip_include_prefix = "include",
 )
 
+cc_library(
+    name = "edge_llm_executorch_blob_header",
+    srcs = [
+        "src/torch_tensorrt/executorch/EdgeLLMBlobHeader.cpp",
+    ],
+    hdrs = [
+        "include/torch_tensorrt/executorch/EdgeLLMBlobHeader.h",
+    ],
+    strip_include_prefix = "include",
+)
+
 cc_library(
     name = "tensorrt_executorch_weight_streaming_budget",
     srcs = [
@@ -153,7 +164,6 @@ cc_library(
     srcs = [
         "src/torch_tensorrt/executorch/RegisterCudaDeviceAllocator.cpp",
     ],
-    alwayslink = True,
     target_compatible_with = select({
         ":linux_x86_64": [],
         ":sbsa": [],
@@ -170,6 +180,7 @@ cc_library(
         ],
         "//conditions:default": [],
     }),
+    alwayslink = True,
 )
 
 # Registration only, for the same reason as the allocator above: the kernel
@@ -184,11 +195,11 @@ cc_library(
     srcs = [
         "src/torch_tensorrt/executorch/RegisterDeviceCopyKernels.cpp",
     ],
-    alwayslink = True,
     deps = [
         "@executorch//:executorch_device_copy_kernels",
         "@executorch//:executorch_headers",
     ],
+    alwayslink = True,
 )
 
 cc_library(
@@ -214,15 +225,15 @@ cc_library(
         ":tensorrt_executorch_weight_streaming_budget",
     ] + select({
         ":linux_x86_64": [
+            "@cuda//:cudart",
             "@executorch//:executorch_headers",
             "@executorch//:extension_cuda",
-            "@cuda//:cudart",
             "@tensorrt//:nvinfer",
         ],
         ":sbsa": [
+            "@cuda//:cudart",
             "@executorch//:executorch_headers",
             "@executorch//:extension_cuda",
-            "@cuda//:cudart",
             "@tensorrt_sbsa//:nvinfer",
         ],
         "//conditions:default": [],
@@ -234,6 +245,8 @@ filegroup(
     name = "executorch_backend_source_files",
     srcs = [
         "src/torch_tensorrt/executorch/CMakeLists.txt",
+        "src/torch_tensorrt/executorch/EdgeLLMBackend.cpp",
+        "src/torch_tensorrt/executorch/EdgeLLMBlobHeader.cpp",
         "src/torch_tensorrt/executorch/README.md",
         "src/torch_tensorrt/executorch/TensorRTBackend.cpp",
         "src/torch_tensorrt/executorch/TensorRTBlobHeader.cpp",
@@ -254,6 +267,8 @@ filegroup(
 filegroup(
     name = "executorch_api_headers",
     srcs = [
+        "include/torch_tensorrt/executorch/EdgeLLMBackend.h",
+        "include/torch_tensorrt/executorch/EdgeLLMBlobHeader.h",
         "include/torch_tensorrt/executorch/TensorRTBackend.h",
         "include/torch_tensorrt/executorch/TensorRTBindingNames.h",
         "include/torch_tensorrt/executorch/TensorRTBlobHeader.h",

cpp/include/torch_tensorrt/executorch/EdgeLLMBackend.h

@@ -0,0 +1,42 @@
+#pragma once
+
+#include <cuda_runtime.h>
+#include <executorch/runtime/backend/interface.h>
+
+#include <executorch/vitExecutorchAdapter.h>
+
+#include <memory>
+#include <mutex>
+
+namespace torch_tensorrt {
+namespace executorch_backend {
+
+struct EdgeLLMHandle {
+  int device_id = 0;
+  std::unique_ptr<trt_edgellm::executorch::VitExecutorchAdapter> vision_runner;
+  std::mutex mu;
+  cudaEvent_t inflight_event = nullptr;
+  bool inflight_pending = false;
+
+  ~EdgeLLMHandle();
+};
+
+class EdgeLLMBackend final : public ::executorch::runtime::BackendInterface {
+ public:
+  bool is_available() const override;
+
+  ::executorch::runtime::Result<::executorch::runtime::DelegateHandle*> init(
+      ::executorch::runtime::BackendInitContext& context,
+      ::executorch::runtime::FreeableBuffer* processed,
+      ::executorch::runtime::ArrayRef<::executorch::runtime::CompileSpec> compile_specs) const override;
+
+  ::executorch::runtime::Error execute(
+      ::executorch::runtime::BackendExecutionContext& context,
+      ::executorch::runtime::DelegateHandle* handle,
+      ::executorch::runtime::Span<::executorch::runtime::EValue*> args) const override;
+
+  void destroy(::executorch::runtime::DelegateHandle* handle) const override;
+};
+
+} // namespace executorch_backend
+} // namespace torch_tensorrt

cpp/include/torch_tensorrt/executorch/EdgeLLMBlobHeader.h

@@ -0,0 +1,26 @@
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <string>
+
+namespace torch_tensorrt {
+namespace executorch_backend {
+
+struct EdgeLLMBlobHeader {
+  uint32_t metadata_offset = 0;
+  uint32_t metadata_size = 0;
+  uint32_t blob_offset = 0;
+  uint64_t blob_size = 0;
+  int abi_version = 0;
+  std::string component;
+  std::string runner;
+  std::string metadata_json;
+  std::string runner_config_json;
+
+  static const void* nested_blob_data(const void* payload, const EdgeLLMBlobHeader& h);
+  static bool parse(const void* data, std::size_t size, EdgeLLMBlobHeader& out);
+};
+
+} // namespace executorch_backend
+} // namespace torch_tensorrt

cpp/src/torch_tensorrt/executorch/CMakeLists.txt

@@ -23,6 +23,7 @@ find_package(Threads REQUIRED)
 set(_torchtrt_executorch_sources
     "${CMAKE_CURRENT_LIST_DIR}/TensorRTBackend.cpp"
     "${CMAKE_CURRENT_LIST_DIR}/TensorRTBlobHeader.cpp"
+    "${CMAKE_CURRENT_LIST_DIR}/EdgeLLMBlobHeader.cpp"
     "${CMAKE_CURRENT_LIST_DIR}/WeightStreamingBudget.cpp"
 )
 
@@ -189,6 +190,57 @@ else()
     )
 endif()
 
+option(
+    TORCHTRT_BUILD_EDGE_LLM_EXECUTORCH_BACKEND
+    "Build the optional TensorRT-Edge-LLM ExecuTorch backend"
+    OFF
+)
+if(TORCHTRT_BUILD_EDGE_LLM_EXECUTORCH_BACKEND)
+    set(EDGELLM_INCLUDE_DIR "" CACHE PATH "TensorRT-Edge-LLM cpp include root")
+    set(EDGELLM_EXECUTORCH_LIBRARY "" CACHE FILEPATH "Path to libedgellmExecutorch.so")
+    foreach(_required IN ITEMS EDGELLM_INCLUDE_DIR EDGELLM_EXECUTORCH_LIBRARY)
+        if("${${_required}}" STREQUAL "" OR NOT EXISTS "${${_required}}")
+            message(FATAL_ERROR "${_required} must name an existing TensorRT-Edge-LLM artifact")
+        endif()
+    endforeach()
+
+    add_library(edgellm_executorch SHARED IMPORTED)
+    set_target_properties(edgellm_executorch PROPERTIES IMPORTED_LOCATION "${EDGELLM_EXECUTORCH_LIBRARY}")
+
+    add_library(executorch_edge_llm_backend STATIC
+        "${CMAKE_CURRENT_LIST_DIR}/EdgeLLMBackend.cpp"
+    )
+    target_include_directories(executorch_edge_llm_backend
+        PUBLIC
+            "${CMAKE_CURRENT_LIST_DIR}/../../../include"
+            "${EDGELLM_INCLUDE_DIR}"
+    )
+    target_compile_definitions(executorch_edge_llm_backend
+        PUBLIC
+            C10_USING_CUSTOM_GENERATED_MACROS
+    )
+    target_link_libraries(executorch_edge_llm_backend
+        PUBLIC
+            executorch_trt_backend
+            edgellm_executorch
+            ${_torchtrt_executorch_link_libraries}
+            ${CMAKE_DL_LIBS}
+    )
+
+    add_library(torchtrt_edge_llm_executorch_backend INTERFACE)
+    add_library(torchtrt::edge_llm_executorch_backend ALIAS torchtrt_edge_llm_executorch_backend)
+    if(MSVC)
+        target_link_libraries(torchtrt_edge_llm_executorch_backend
+            INTERFACE executorch_edge_llm_backend edgellm_executorch)
+    else()
+        target_link_libraries(torchtrt_edge_llm_executorch_backend
+            INTERFACE
+                "-Wl,--whole-archive,$<TARGET_FILE:executorch_edge_llm_backend>,--no-whole-archive"
+                edgellm_executorch
+        )
+    endif()
+endif()
+
 install(
     TARGETS executorch_trt_backend
     ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"