Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 14 additions & 13 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ ENV VIRTUAL_ENV=/opt/venv
ENV PATH=/opt/venv/bin:$PATH
COPY requirements/base.txt /tmp/trtmc-base-requirements.txt
RUN python3.12 -m venv "$VIRTUAL_ENV" \
&& pip install --upgrade pip \
&& pip install --upgrade "pip==26.2.1" \
&& pip install \
"torch==2.12.0+cu130" \
"torchvision==0.27.0+cu130" \
Expand All @@ -53,29 +53,30 @@ RUN python3.12 -m venv "$VIRTUAL_ENV" \
"huggingface_hub==1.21.0" \
"jsonschema==4.26.0" \
"lizard==1.21.2" \
"ml_dtypes>=0.4" \
"numpy>=1.24,<2.5" \
"onnx>=1.16" \
"Pillow" \
"protobuf" \
"ml_dtypes==0.5.4" \
"numpy==1.26.4" \
"onnx==1.21.0" \
"packaging==26.2" \
"Pillow==12.2.0" \
"protobuf==7.35.0" \
"pybind11==2.13.6" \
"pybind11-stubgen==2.4.2" \
"pytest<9" \
"PyYAML>=6.0" \
"pytest==8.4.2" \
"PyYAML==6.0.3" \
"ruff==0.16.4" \
"safetensors>=0.4" \
"safetensors==0.8.0" \
"scikit-build-core==0.8.2" \
"sentencepiece>=0.1.99" \
"setuptools>=80,<82" \
"sentencepiece==0.2.2" \
"setuptools==81.0.0" \
"tensorrt==11.1.0.106" \
"tokenizers" \
"tokenizers==0.22.2" \
"transformers==5.2.0" \
&& pip install --force-reinstall \
"torch==2.12.0+cu130" \
"torchvision==0.27.0+cu130" \
"torchaudio==2.11.0+cu130" \
--index-url https://download.pytorch.org/whl/cu130 \
&& pip install "setuptools>=80,<82"
&& pip install "setuptools==81.0.0"

ENV TRT_LIB_DIR=/opt/venv/lib/python3.12/site-packages/tensorrt_libs
ENV NCCL_LIB_DIR=/opt/venv/lib/python3.12/site-packages/nvidia/nccl/lib
Expand Down
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@

```bash
python -m tensorrt_model_connect build Qwen/Qwen3-0.6B \
--revision c1899de289a04d12100db370d81485cdf75e47ca \
--max-sequence-length 16384 \
--output qwen3-0.6b.bundle
trtmc run ./qwen3-0.6b.bundle \
Expand Down
85 changes: 70 additions & 15 deletions apps/benchmark/trtmc_benchmark/builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
from pathlib import Path
from typing import Any, Iterable, Mapping, Sequence

from tensorrt_model_connect import read_bundle_provenance, resolve_source_revision
from tensorrt_model_connect.build_cli import _resolve_model

from .types import BenchmarkError, ModelDescriptor, ResolvedCase
Expand Down Expand Up @@ -119,7 +120,12 @@ def _prepare_group(
model = cases[0].model
managed = _is_relative_to(requested, self.cache_root)
if requested.is_file() and not rebuild:
return requested, BundlePreparation(model.name, "reused", requested)
if _bundle_matches_model(requested, model, cases):
return requested, BundlePreparation(model.name, "reused", requested)
if not allow_build:
raise BenchmarkError(
f"bundle provenance does not match {model.name}: {requested}"
)
if requested.is_file() and not managed:
raise BenchmarkError(
f"--rebuild cannot overwrite explicit bundle {requested}; "
Expand Down Expand Up @@ -151,6 +157,16 @@ def _plan(self, model: ModelDescriptor, cases: Sequence[ResolvedCase]) -> _Build
raise BenchmarkError(f"model directory does not exist: {explicit}")
if explicit is None and not model.hf_id:
raise BenchmarkError(f"{model.name} has no hf_id; pass --model-dir")
if explicit is not None and model.hf_id:
resolved_explicit = explicit.resolve()
if (
resolved_explicit.parent.name != "snapshots"
or resolved_explicit.name.lower() != model.hf_revision.lower()
):
raise BenchmarkError(
f"explicit model directory for {model.name} must be the exact Hugging Face "
f"snapshot ending in /snapshots/{model.hf_revision}"
)
try:
model_dir = _resolve_model(
str(explicit) if explicit is not None else model.hf_id,
Expand Down Expand Up @@ -238,7 +254,7 @@ def _build_command(
bundle: Path,
cases: Sequence[ResolvedCase],
) -> tuple[str, ...]:
settings = model.build_settings
settings = _build_request_options(model, cases)
command = [
sys.executable,
"-m",
Expand All @@ -252,6 +268,8 @@ def _build_command(
"--precision",
model.precision,
]
command.extend(("--checkpoint-id", model.checkpoint_id))
command.extend(("--revision", model.checkpoint_revision))
flags = (
("max_sequence_length", "--max-sequence-length"),
("image_height", "--image-height"),
Expand All @@ -272,24 +290,30 @@ def _build_command(
if settings.get("dynamic_kv_cache", False):
command.append("--dynamic-kv-cache")

return tuple(command)


def _build_request_options(
model: ModelDescriptor, cases: Sequence[ResolvedCase]
) -> dict[str, object]:
settings: dict[str, object] = dict(model.build_settings)
settings.setdefault("backend", "trt")
settings.setdefault("max_batch_size", 1)
settings.setdefault("tensor_parallel_size", 1)
settings.setdefault("context_parallel_size", 1)
settings.setdefault("dynamic_kv_cache", False)

image_cases = [case for case in cases if case.operation == "generate_image"]
if image_cases:
heights = [int(case.request.get("height", 0)) for case in image_cases]
widths = [int(case.request.get("width", 0)) for case in image_cases]
batches = [int(case.request.get("batch_size", 1)) for case in image_cases]
_replace_value(command, "--image-height", max(heights, default=0))
_replace_value(command, "--image-width", max(widths, default=0))
_replace_value(command, "--max-batch-size", max(batches, default=1))
return tuple(command)


def _replace_value(command: list[str], flag: str, value: int) -> None:
if value <= 0:
return
if flag in command:
command[command.index(flag) + 1] = str(value)
else:
command.extend((flag, str(value)))
if max(heights, default=0) > 0:
settings["image_height"] = max(heights)
if max(widths, default=0) > 0:
settings["image_width"] = max(widths)
settings["max_batch_size"] = max(batches, default=1)
return settings


def _write(path: Path, value: str) -> None:
Expand All @@ -308,3 +332,34 @@ def _is_relative_to(path: Path, parent: Path) -> bool:
except ValueError:
return False
return True


def _bundle_matches_model(
bundle: Path, model: ModelDescriptor, cases: Sequence[ResolvedCase]
) -> bool:
try:
source_revision = resolve_source_revision()
except ValueError:
return False
try:
provenance = read_bundle_provenance(bundle)
except (OSError, UnicodeDecodeError, ValueError):
return False
if not isinstance(provenance, dict) or provenance.get("format") != 1:
return False
expected_request = {
"family": model.family,
"task": model.task,
"precision": model.precision,
**_build_request_options(model, cases),
}
if not expected_request.get("fp32_layers"):
expected_request.pop("fp32_layers", None)
elif isinstance(expected_request["fp32_layers"], tuple):
expected_request["fp32_layers"] = list(expected_request["fp32_layers"])
return provenance.get("checkpoint") == {
"id": model.checkpoint_id,
"revision": model.checkpoint_revision,
} and provenance.get("build") == {
"source_revision": source_revision
} and provenance.get("request") == expected_request
26 changes: 24 additions & 2 deletions apps/benchmark/trtmc_benchmark/catalog.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@
from pathlib import Path
from typing import Any, Iterable, Mapping

from tensorrt_model_connect import validate_checkpoint_revision

from .task_adapters import default_operation, resolve_task_case, supported_tasks
from .types import BenchmarkError, MeasurementSpec, ModelDescriptor, ResolvedCase

Expand Down Expand Up @@ -166,10 +168,30 @@ def _load(path: Path) -> ModelDescriptor:
settings.setdefault("max_batch_size", 1)
settings.setdefault("tensor_parallel_size", 1)
settings.setdefault("context_parallel_size", 1)
hf_id = _optional_string(raw.get("hf_id", ""), "hf_id", path)
hf_revision = _optional_string(raw.get("hf_revision", ""), "hf_revision", path)
checkpoint_id = _optional_string(
raw.get("checkpoint_id", hf_id), "checkpoint_id", path
)
checkpoint_revision = _optional_string(
raw.get("checkpoint_revision", hf_revision), "checkpoint_revision", path
)
if not checkpoint_id or not checkpoint_revision:
raise BenchmarkError(
f"model manifest must declare an immutable checkpoint identity: {path}"
)
Comment thread
coderabbitai[bot] marked this conversation as resolved.
try:
validate_checkpoint_revision(checkpoint_revision)
except ValueError as error:
raise BenchmarkError(
f"model manifest has a mutable checkpoint_revision in {path}: {error}"
) from error
return ModelDescriptor(
name=_string(raw["name"], "name", path),
hf_id=_optional_string(raw.get("hf_id", ""), "hf_id", path),
hf_revision=_optional_string(raw.get("hf_revision", ""), "hf_revision", path),
hf_id=hf_id,
hf_revision=hf_revision,
checkpoint_id=checkpoint_id,
checkpoint_revision=checkpoint_revision,
bundle_name=_string(raw["bundle"], "bundle", path),
family=_string(raw["family"], "family", path),
task=_string(raw["task"], "task", path),
Expand Down
Loading
Loading