diff --git a/backend/python/vllm/backend.py b/backend/python/vllm/backend.py index 6dabbf4bf687..15301a0b7da6 100644 --- a/backend/python/vllm/backend.py +++ b/backend/python/vllm/backend.py @@ -555,17 +555,24 @@ def _build_sampling_params(self, request): "TruncatePromptTokens": "truncate_prompt_tokens", } - sampling_params = SamplingParams(top_p=0.9, max_tokens=200) - + # Construct SamplingParams from the collected values, as vLLM's own + # OpenAI server does. Assigning them with setattr after construction + # skipped __post_init__ and left protobuf repeated fields (StopPrompts) + # as RepeatedScalarContainer, which vLLM's argument checks and the + # msgpack encoder to the engine core reject: every request with `stop` + # failed. Invalid values now raise vLLM's validation error here. + kwargs = {"top_p": 0.9, "max_tokens": 200} for request_field, param_field in request_to_sampling_params.items(): if hasattr(request, request_field): value = getattr(request, request_field) # See _EXPLICIT_ZERO_FIELDS: temperature 0 is greedy decoding # and 0 is a valid seed, so neither may be filtered out. if request_field in _EXPLICIT_ZERO_FIELDS or value not in (None, 0, [], False): - setattr(sampling_params, param_field, value) + if not isinstance(value, (str, bytes, int, float, bool)): + value = list(value) # protobuf repeated field -> list + kwargs[param_field] = value - return sampling_params + return SamplingParams(**kwargs) def _new_reasoning_parser(self, chat_template_kwargs): """Build the reasoning parser, telling it whether thinking is on. diff --git a/backend/python/vllm/test.py b/backend/python/vllm/test.py index 27a846f0918d..6317955c0348 100644 --- a/backend/python/vllm/test.py +++ b/backend/python/vllm/test.py @@ -133,8 +133,13 @@ def test_explicit_zero_temperature_and_seed_are_preserved(self): sampling_params = servicer._build_sampling_params(request) self.assertEqual(sampling_params.temperature, 0) self.assertEqual(sampling_params.seed, 0) - # Other protobuf-default scalar fields must remain filtered. + # Other protobuf-default scalar fields must remain filtered. (With + # temperature 0, vLLM's __post_init__ switches to greedy and sets + # top_p to 1.0, so check the default on a sampled request.) + request = backend_pb2.PredictOptions(Prompt="hello", Temperature=0.7, Seed=0) + sampling_params = servicer._build_sampling_params(request) self.assertEqual(sampling_params.top_p, 0.9) + self.assertEqual(sampling_params.seed, 0) def test_messages_to_dicts(self): @@ -296,6 +301,69 @@ def test_embedding(self): self.tearDown() +class TestSamplingParamsConstruction(unittest.TestCase): + """Server-less tests for BackendServicer._build_sampling_params. + + protobuf repeated fields (StopPrompts) are RepeatedScalarContainer, not + list. Assigned with setattr after construction they reached the engine + as-is: vLLM's argument checks assert a list, and the msgpack encoder that + ships the params to the engine core cannot serialise them, so every + request with `stop` failed. + """ + + def _build(self, **fields): + import sys, os + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + from backend import BackendServicer + servicer = BackendServicer.__new__(BackendServicer) + return servicer._build_sampling_params(backend_pb2.PredictOptions(**fields)) + + @staticmethod + def _engine_round_trip(sp): + """What happens to the params on their way to the engine core.""" + from vllm.v1.serial_utils import MsgpackEncoder + sp._verify_args() + sp.clone() + MsgpackEncoder().encode(sp) + + def test_stop_is_a_list_and_reaches_the_engine(self): + sp = self._build(Prompt="x", StopPrompts=["", "##"], Tokens=50, Temperature=0.7) + self.assertIs(type(sp.stop), list) + self.assertEqual(sp.stop, ["", "##"]) + self._engine_round_trip(sp) + + def test_stop_string_hold_back_buffer_is_set(self): + """Without it the start of a multi-character stop string is streamed + before the match completes.""" + sp = self._build(Prompt="x", StopPrompts=[""], Tokens=50, Temperature=0.7) + self.assertEqual(sp.output_text_buffer_length, len("") - 1) + + def test_request_values_are_applied(self): + sp = self._build(Prompt="x", Tokens=77, Temperature=0.6, TopP=0.95, TopK=20) + self.assertEqual(sp.max_tokens, 77) + self.assertEqual(sp.top_k, 20) + self.assertAlmostEqual(sp.top_p, 0.95, places=6) + self.assertEqual(sp.stop, []) + self._engine_round_trip(sp) + + def test_seed_minus_one_means_random(self): + sp = self._build(Prompt="x", Seed=-1, Temperature=0.7) + self.assertIsNone(sp.seed) + + def test_invalid_values_are_rejected(self): + """Out-of-range values fail with vLLM's validation error instead of + reaching the engine.""" + try: + from vllm.exceptions import VLLMValidationError as rejected + except ImportError: # older vLLM raises ValueError + rejected = ValueError + for fields in ({"PresencePenalty": 5.0, "Temperature": 0.7}, + {"StopPrompts": [""], "Temperature": 0.7}): + with self.subTest(fields=fields): + with self.assertRaises(rejected): + self._build(Prompt="x", Tokens=10, **fields) + + class TestStreamingToolParser(unittest.TestCase): """ Server-less unit tests for the streaming + tool-parser machinery in diff --git a/docs/content/advanced/model-configuration.md b/docs/content/advanced/model-configuration.md index bea83b6f5abc..5fd89323c5ae 100644 --- a/docs/content/advanced/model-configuration.md +++ b/docs/content/advanced/model-configuration.md @@ -833,6 +833,12 @@ The fields below come from upstream llama.cpp's **CLI completion tool** and are These options apply when using the `vllm` backend: +The `vllm` backend validates sampling parameters before generation. Stop strings +work with streaming and non-streaming requests; an empty stop string is invalid. +Presence and frequency penalties must be between -2 and 2. A temperature of `0` +selects greedy decoding, which ignores `top_p` and `top_k`. A seed of `-1` uses +vLLM's default random seed behavior. + | Field | Type | Description | |-------|------|-------------| | `gpu_memory_utilization` | float32 | GPU memory utilization (0.0-1.0, default 0.9) |