From 48d11fcf74892b0d89e9327a49e6ac8967b45590 Mon Sep 17 00:00:00 2001 From: pos-ei-don <1822533+pos-ei-don@users.noreply.github.com> Date: Fri, 23 Oct 2026 23:58:46 +0200 Subject: [PATCH 1/2] fix(vllm): build SamplingParams once so `stop` reaches the engine _build_sampling_params constructed SamplingParams and then assigned the request values with setattr. protobuf repeated fields (StopPrompts) stay RepeatedScalarContainer that way, and __post_init__ never sees them: - vLLM's argument checks assert `isinstance(self.stop, list)` and the msgpack encoder that ships the params to the engine core cannot serialise the container, so every request with `stop` failed (HTTP 500, EngineGenerateError), streaming or not. - the stop-string hold-back buffer (output_text_buffer_length) was never set, so the start of a multi-character stop string could be streamed; seed -1 was not mapped to None. Collect the values, turn repeated fields into lists and construct SamplingParams once, as vLLM's OpenAI server does. Values vLLM rejects (e.g. a penalty out of range, an empty stop string) now fail with vLLM's validation error instead of being set unchecked. Signed-off-by: pos-ei-don <1822533+pos-ei-don@users.noreply.github.com> --- backend/python/vllm/backend.py | 15 ++++++-- backend/python/vllm/test.py | 70 +++++++++++++++++++++++++++++++++- 2 files changed, 80 insertions(+), 5 deletions(-) diff --git a/backend/python/vllm/backend.py b/backend/python/vllm/backend.py index 6dabbf4bf687..15301a0b7da6 100644 --- a/backend/python/vllm/backend.py +++ b/backend/python/vllm/backend.py @@ -555,17 +555,24 @@ def _build_sampling_params(self, request): "TruncatePromptTokens": "truncate_prompt_tokens", } - sampling_params = SamplingParams(top_p=0.9, max_tokens=200) - + # Construct SamplingParams from the collected values, as vLLM's own + # OpenAI server does. Assigning them with setattr after construction + # skipped __post_init__ and left protobuf repeated fields (StopPrompts) + # as RepeatedScalarContainer, which vLLM's argument checks and the + # msgpack encoder to the engine core reject: every request with `stop` + # failed. Invalid values now raise vLLM's validation error here. + kwargs = {"top_p": 0.9, "max_tokens": 200} for request_field, param_field in request_to_sampling_params.items(): if hasattr(request, request_field): value = getattr(request, request_field) # See _EXPLICIT_ZERO_FIELDS: temperature 0 is greedy decoding # and 0 is a valid seed, so neither may be filtered out. if request_field in _EXPLICIT_ZERO_FIELDS or value not in (None, 0, [], False): - setattr(sampling_params, param_field, value) + if not isinstance(value, (str, bytes, int, float, bool)): + value = list(value) # protobuf repeated field -> list + kwargs[param_field] = value - return sampling_params + return SamplingParams(**kwargs) def _new_reasoning_parser(self, chat_template_kwargs): """Build the reasoning parser, telling it whether thinking is on. diff --git a/backend/python/vllm/test.py b/backend/python/vllm/test.py index 27a846f0918d..6317955c0348 100644 --- a/backend/python/vllm/test.py +++ b/backend/python/vllm/test.py @@ -133,8 +133,13 @@ def test_explicit_zero_temperature_and_seed_are_preserved(self): sampling_params = servicer._build_sampling_params(request) self.assertEqual(sampling_params.temperature, 0) self.assertEqual(sampling_params.seed, 0) - # Other protobuf-default scalar fields must remain filtered. + # Other protobuf-default scalar fields must remain filtered. (With + # temperature 0, vLLM's __post_init__ switches to greedy and sets + # top_p to 1.0, so check the default on a sampled request.) + request = backend_pb2.PredictOptions(Prompt="hello", Temperature=0.7, Seed=0) + sampling_params = servicer._build_sampling_params(request) self.assertEqual(sampling_params.top_p, 0.9) + self.assertEqual(sampling_params.seed, 0) def test_messages_to_dicts(self): @@ -296,6 +301,69 @@ def test_embedding(self): self.tearDown() +class TestSamplingParamsConstruction(unittest.TestCase): + """Server-less tests for BackendServicer._build_sampling_params. + + protobuf repeated fields (StopPrompts) are RepeatedScalarContainer, not + list. Assigned with setattr after construction they reached the engine + as-is: vLLM's argument checks assert a list, and the msgpack encoder that + ships the params to the engine core cannot serialise them, so every + request with `stop` failed. + """ + + def _build(self, **fields): + import sys, os + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + from backend import BackendServicer + servicer = BackendServicer.__new__(BackendServicer) + return servicer._build_sampling_params(backend_pb2.PredictOptions(**fields)) + + @staticmethod + def _engine_round_trip(sp): + """What happens to the params on their way to the engine core.""" + from vllm.v1.serial_utils import MsgpackEncoder + sp._verify_args() + sp.clone() + MsgpackEncoder().encode(sp) + + def test_stop_is_a_list_and_reaches_the_engine(self): + sp = self._build(Prompt="x", StopPrompts=["", "##"], Tokens=50, Temperature=0.7) + self.assertIs(type(sp.stop), list) + self.assertEqual(sp.stop, ["", "##"]) + self._engine_round_trip(sp) + + def test_stop_string_hold_back_buffer_is_set(self): + """Without it the start of a multi-character stop string is streamed + before the match completes.""" + sp = self._build(Prompt="x", StopPrompts=[""], Tokens=50, Temperature=0.7) + self.assertEqual(sp.output_text_buffer_length, len("") - 1) + + def test_request_values_are_applied(self): + sp = self._build(Prompt="x", Tokens=77, Temperature=0.6, TopP=0.95, TopK=20) + self.assertEqual(sp.max_tokens, 77) + self.assertEqual(sp.top_k, 20) + self.assertAlmostEqual(sp.top_p, 0.95, places=6) + self.assertEqual(sp.stop, []) + self._engine_round_trip(sp) + + def test_seed_minus_one_means_random(self): + sp = self._build(Prompt="x", Seed=-1, Temperature=0.7) + self.assertIsNone(sp.seed) + + def test_invalid_values_are_rejected(self): + """Out-of-range values fail with vLLM's validation error instead of + reaching the engine.""" + try: + from vllm.exceptions import VLLMValidationError as rejected + except ImportError: # older vLLM raises ValueError + rejected = ValueError + for fields in ({"PresencePenalty": 5.0, "Temperature": 0.7}, + {"StopPrompts": [""], "Temperature": 0.7}): + with self.subTest(fields=fields): + with self.assertRaises(rejected): + self._build(Prompt="x", Tokens=10, **fields) + + class TestStreamingToolParser(unittest.TestCase): """ Server-less unit tests for the streaming + tool-parser machinery in From 05af7b072c64a738178924e5a0dbd54c9f03e6d1 Mon Sep 17 00:00:00 2001 From: pos-ei-don <1822533+pos-ei-don@users.noreply.github.com> Date: Sat, 24 Oct 2026 19:30:35 +0000 Subject: [PATCH 2/2] docs(vllm): explain sampling validation Describe stop strings, greedy decoding, and seed handling after sampling parameters are validated during construction. Signed-off-by: pos-ei-don <1822533+pos-ei-don@users.noreply.github.com> --- docs/content/advanced/model-configuration.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/content/advanced/model-configuration.md b/docs/content/advanced/model-configuration.md index bea83b6f5abc..5fd89323c5ae 100644 --- a/docs/content/advanced/model-configuration.md +++ b/docs/content/advanced/model-configuration.md @@ -833,6 +833,12 @@ The fields below come from upstream llama.cpp's **CLI completion tool** and are These options apply when using the `vllm` backend: +The `vllm` backend validates sampling parameters before generation. Stop strings +work with streaming and non-streaming requests; an empty stop string is invalid. +Presence and frequency penalties must be between -2 and 2. A temperature of `0` +selects greedy decoding, which ignores `top_p` and `top_k`. A seed of `-1` uses +vLLM's default random seed behavior. + | Field | Type | Description | |-------|------|-------------| | `gpu_memory_utilization` | float32 | GPU memory utilization (0.0-1.0, default 0.9) |