feat(serve): add local serving foundation

.coderabbit.yaml

@@ -138,6 +138,20 @@ reviews:
         outputs, owns family validation policy, or becomes a dependency of core
         or family implementation.
 
+    - path: "server/**"
+      instructions: |
+        Treat the local server as an optional application over public runtime
+        contracts. Preserve the one-way server-to-library dependency, fixed
+        bounded execution lanes, sanitized public errors, and source-build and
+        installed-wheel parity. Flag model-specific policy, reverse dependencies
+        from core or families, hidden queues, or claims of distributed scaling,
+        automatic placement, or worker self-healing that are not implemented.
+
+    - path: "server/tests/**"
+      instructions: |
+        Do not suggest weakening assertions, lifecycle and security boundaries,
+        expected values, or acceptance criteria merely to make tests pass.
+
     - path: "apps/benchmark/**"
       instructions: |
         Check semantic equivalence of timed regions, synchronization, warmup,

CMakeLists.txt

@@ -263,6 +263,8 @@ foreach(_trtmc_runtime_cmake IN LISTS _trtmc_family_runtime_cmake)
   )
 endforeach()
 
+add_subdirectory(server)
+
 add_library(trtmc_cli STATIC
   apps/cli/cli.cpp
   apps/cli/io.cpp
@@ -288,7 +290,8 @@ set_source_files_properties(apps/cli/io.cpp PROPERTIES
 
 add_executable(trtmc apps/cli/main.cpp)
 target_include_directories(trtmc PRIVATE ${PROJECT_SOURCE_DIR}/apps)
-target_link_libraries(trtmc PRIVATE trtmc_cli)
+target_link_libraries(trtmc PRIVATE trtmc_cli trtmc_server_native)
+add_dependencies(trtmc trtmc_server_python)
 target_compile_options(trtmc PRIVATE -Wall -Wextra -Wpedantic)
 set_target_properties(trtmc PROPERTIES
   BUILD_RPATH "\$ORIGIN"

Dockerfile

@@ -78,6 +78,16 @@ RUN python3.12 -m venv "$VIRTUAL_ENV" \
       --index-url https://download.pytorch.org/whl/cu130 \
     && pip install "setuptools>=80,<82"
 
+# Server control-plane and CPU contract-test dependencies. Keep these bounds
+# aligned with the serve and test extras in pyproject.toml.
+RUN pip install --target /opt/trtmc-server-test-deps \
+      "fastapi>=0.115,<0.142" \
+      "httpx>=0.27,<0.29" \
+      "pydantic>=2.11,<3" \
+      "python-multipart>=0.0.9,<1" \
+      "uvicorn>=0.30,<0.53" \
+      "websockets>=13,<17"
+
 ENV TRT_LIB_DIR=/opt/venv/lib/python3.12/site-packages/tensorrt_libs
 ENV NCCL_LIB_DIR=/opt/venv/lib/python3.12/site-packages/nvidia/nccl/lib
 ENV TVM_FFI_LIB_DIR=/opt/venv/lib/python3.12/site-packages/tvm_ffi/lib

apps/cli/cli.cpp

@@ -1280,6 +1280,7 @@ void print_usage(std::ostream& output) {
     output << "Usage:\n"
               "  trtmc version\n"
               "  trtmc inspect BUNDLE\n"
+              "  trtmc serve --runtime-root DIR [SERVER OPTIONS]\n"
               "  trtmc COMMAND BUNDLE --runtime-root DIR [OPTIONS]\n\n"
               "Execution commands:\n"
               "  run, encode, embed, rerank, classify, detect, extract-features,\n"

apps/cli/main.cpp

@@ -4,9 +4,15 @@
  */
 
 #include "cli/cli.h"
+#include "native/entrypoint.h"
 
 #include <iostream>
+#include <string>
 
 int main(int argc, char** argv) {
+    if (argc >= 2 && std::string(argv[1]) == "serve")
+        return trtmc::server::run_server_frontend(argc - 2, argv + 2);
+    if (argc >= 2 && std::string(argv[1]) == "_serve-worker")
+        return trtmc::server::run_native_worker(argc - 2, argv + 2);
     return trtmc::cli::run(argc, argv, std::cout, std::cerr);
 }

apps/cli/tests/test_cli.cpp

@@ -732,7 +732,8 @@ int main() {
 
     std::ostringstream usage;
     trtmc::cli::print_usage(usage);
-    check(usage.str().find("--source-language-token-id") != std::string::npos &&
+    check(usage.str().find("trtmc serve --runtime-root DIR") != std::string::npos &&
+              usage.str().find("--source-language-token-id") != std::string::npos &&
               usage.str().find("--segment-overlap-seconds") != std::string::npos &&
               usage.str().find("--runtime-cache") != std::string::npos &&
               usage.str().find("--cuda-graphs") != std::string::npos,

pyproject.toml

@@ -35,14 +35,32 @@ dependencies = [
 trtmc-bench = "trtmc_benchmark.cli:main"
 
 [project.optional-dependencies]
-test = ["pytest>=7.0", "torch>=2.0", "jsonschema>=4.23,<5"]
+serve = [
+    "fastapi>=0.115,<0.142",
+    "pydantic>=2.11,<3",
+    "python-multipart>=0.0.9,<1",
+    "uvicorn>=0.30,<0.53",
+    "websockets>=13,<17",
+]
+test = [
+    "pytest>=7.0",
+    "torch>=2.0",
+    "jsonschema>=4.23,<5",
+    "fastapi>=0.115,<0.142",
+    "httpx>=0.27,<0.29",
+    "pydantic>=2.11,<3",
+    "python-multipart>=0.0.9,<1",
+    "uvicorn>=0.30,<0.53",
+    "websockets>=13,<17",
+]
 cutedsl = ["nvidia-cutlass-dsl==4.7.1"]
 
 [tool.conan-py-build.wheel]
 packages = [
     "core/builder/tensorrt_model_connect",
     "apps/benchmark/trtmc_benchmark",
     "families",
+    "server/python/trtmc_server",
 ]
 
 [tool.conan-py-build.sdist]
@@ -57,6 +75,7 @@ include = [
     "apps",
     "families",
     "requirements",
+    "server",
     "third_party",
     "examples",
     "tools/perf_matrix.py",
@@ -66,11 +85,13 @@ exclude = [
     "*.pyc",
     "core/builder/tensorrt_model_connect/build",
     "core/builder/tensorrt_model_connect/*.egg-info",
+    "server/python/trtmc_server/*.egg-info",
 ]
 
 [tool.pytest.ini_options]
 enable_assertion_pass_hook = true
 testpaths = [
+    "server/tests",
     "families",
     "core/builder/tests",
     "apps/benchmark/trtmc_benchmark/tests",

requirements/community-ci.txt

@@ -2,9 +2,14 @@
 # SPDX-License-Identifier: Apache-2.0
 
 # Keep local source checks and the public Community CPU workflow on the same
-# tool versions. The development images consume this file as well.
+# tool and server-test dependencies. The development images consume this file
+# as well.
 pre-commit==4.6.2
 apache-tvm-ffi==0.1.12
+fastapi>=0.115,<0.142
+httpx>=0.27,<0.29
+pydantic>=2.11,<3
+python-multipart>=0.0.9,<1
 ruff==0.16.4
 clang-format==22.1.8
 lizard==1.21.2
@@ -13,3 +18,5 @@ pytest-xdist==3.8.0
 jsonschema==4.26.0
 Pillow==12.2.0
 PyYAML==6.0.2
+uvicorn>=0.30,<0.53
+websockets>=13,<17

server/CMakeLists.txt

@@ -0,0 +1,58 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+add_library(trtmc_server_native STATIC
+  native/entrypoint.cpp
+  native/worker.cpp
+)
+target_include_directories(trtmc_server_native
+  PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}
+)
+target_link_libraries(trtmc_server_native
+  PRIVATE
+    trtmc_runtime
+    nlohmann_json::nlohmann_json
+)
+target_compile_options(trtmc_server_native PRIVATE -Wall -Wextra -Wpedantic)
+
+# Keep source-build CLI smoke tests independent of the caller's working directory
+# without embedding checkout paths in the native executable.
+add_custom_target(trtmc_server_python ALL
+  COMMAND ${CMAKE_COMMAND} -E remove_directory
+    "${PROJECT_BINARY_DIR}/server/python/trtmc_server"
+  COMMAND ${CMAKE_COMMAND} -E copy_directory
+    "${CMAKE_CURRENT_SOURCE_DIR}/python/trtmc_server"
+    "${PROJECT_BINARY_DIR}/server/python/trtmc_server"
+  COMMENT "Copying the local server Python module"
+)
+
+if(TRTMC_BUILD_TESTS)
+  add_executable(test_serve_worker tests/test_serve_worker.cpp)
+  target_include_directories(test_serve_worker PRIVATE
+    ${PROJECT_SOURCE_DIR}/core/runtime/include
+  )
+  target_link_libraries(test_serve_worker PRIVATE
+    trtmc_server_native
+    nlohmann_json::nlohmann_json
+  )
+  target_compile_options(test_serve_worker PRIVATE -Wall -Wextra -Wpedantic)
+  add_test(NAME serve_worker COMMAND test_serve_worker)
+  set_tests_properties(serve_worker PROPERTIES LABELS cpu)
+
+  set(_trtmc_serve_shadow_dir "${PROJECT_BINARY_DIR}/server/tests/cwd-shadow")
+  file(MAKE_DIRECTORY "${_trtmc_serve_shadow_dir}/trtmc_server")
+  file(WRITE "${_trtmc_serve_shadow_dir}/trtmc_server/__init__.py" "")
+  file(WRITE "${_trtmc_serve_shadow_dir}/trtmc_server/__main__.py"
+    "print('TRTMC_CWD_SHADOW_EXECUTED')\n"
+  )
+  add_test(
+    NAME serve_cli_ignores_cwd_shadow
+    COMMAND $<TARGET_FILE:trtmc> serve --help
+  )
+  set_tests_properties(serve_cli_ignores_cwd_shadow PROPERTIES
+    WORKING_DIRECTORY "${_trtmc_serve_shadow_dir}"
+    PASS_REGULAR_EXPRESSION "Serve TensorRT-Model-Connect bundles"
+    FAIL_REGULAR_EXPRESSION "TRTMC_CWD_SHADOW_EXECUTED"
+    LABELS cpu
+  )
+endif()

server/README.md

@@ -0,0 +1,31 @@
+<!--
+SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-License-Identifier: Apache-2.0
+-->
+
+# Local server
+
+This directory owns the optional process behind `trtmc serve`:
+
+- `python/trtmc_server/` provides the HTTP and WebSocket control plane.
+- `native/` adapts the public `load_task()` and `ITask` contracts to a private
+  JSONL worker process.
+- `tests/` owns server API, process-lifecycle, protocol, and dependency checks.
+
+The dependency is one-way: the server may use public library contracts, while
+core and model families never depend on server implementation. Applications
+consume the `trtmc serve` process and its HTTP/WebSocket APIs; they do not
+import `trtmc_server`.
+
+Concurrency is a fixed set of serial worker lanes configured at startup. The
+server has no waiting queue, dynamic placement, continuous batching, cluster
+scheduler, or worker restart. Saturation fails immediately, failed lanes leave
+the model degraded while another lane remains healthy, and recovery belongs to
+an external supervisor. One server process is one local placement domain, not
+a generic distributed serving system.
+
+Replica counts above one apply only to independently loadable single-process
+bundles. MPI/NCCL distributed bundles are not supported by `trtmc serve`.
+For multiple GPUs, run independent single-process server instances and pin
+each instance with `CUDA_VISIBLE_DEVICES`; external routing remains the
+supervisor's responsibility.