From 2e6fa838132f47670a5e83016791d0a40944077c Mon Sep 17 00:00:00 2001 From: Brad Hilton Date: Wed, 21 Oct 2026 22:09:20 +0000 Subject: [PATCH 1/3] fix: support vLLM 0.31 Responses history rendering --- src/art_inference/vllm.py | 24 +++++++- tests/unit/test_inference_history.py | 86 +++++++++++++++++++++++++--- 2 files changed, 101 insertions(+), 9 deletions(-) diff --git a/src/art_inference/vllm.py b/src/art_inference/vllm.py index 8bdb57e4b..5244aa357 100644 --- a/src/art_inference/vllm.py +++ b/src/art_inference/vllm.py @@ -208,6 +208,15 @@ def input_messages(*, prev_response_output=None, request_input, **kwargs): setattr(responses_utils, "construct_input_messages", input_messages) setattr(responses_serving, "construct_input_messages", input_messages) + if hasattr( + responses_serving.OpenAIServingResponses, "_render_resolved_response_inputs" + ): + # vLLM 0.31 moved Responses rendering (and this imported alias) here. + setattr( + importer("vllm.renderers.online_renderer"), + "construct_input_messages", + input_messages, + ) engine = importer("vllm.v1.engine.async_llm").AsyncLLM generate = engine.generate @@ -349,7 +358,20 @@ async def observe_response(response): if request.previous_response_id: async with self.response_store_lock: previous = self.response_store.get(request.previous_response_id) - conversation, inputs = await self._make_request(request, previous) + if hasattr(self, "_make_request"): # vLLM 0.24 + conversation, inputs = await self._make_request(request, previous) + else: # vLLM 0.31 + rendered = await self._render_resolved_response_inputs( + request, previous + ) + if isinstance( + rendered, + importer( + "vllm.entrypoints.serve.engine.protocol" + ).ErrorResponse, + ): + return + conversation, inputs = rendered.messages, [rendered.engine_input] if len(inputs) != 1: return protocol = importer("vllm.entrypoints.openai.chat_completion.protocol") diff --git a/tests/unit/test_inference_history.py b/tests/unit/test_inference_history.py index 895784782..904afc921 100644 --- a/tests/unit/test_inference_history.py +++ b/tests/unit/test_inference_history.py @@ -48,8 +48,8 @@ class Message(BaseModel): tool_calls: list[dict] = [] -@pytest.fixture -def serving(monkeypatch): +@pytest.fixture(params=["0.24", "0.31"]) +def serving(monkeypatch, request): monkeypatch.setattr(vllm, "_PREFIXES", TokenPrefixStore()) class Tokenizer: @@ -201,7 +201,26 @@ def construct_input_messages(**kwargs): construct_chat_messages_with_tool_call=lambda items: items, construct_tool_dicts=lambda *_: None, ) + online_renderer = SimpleNamespace(construct_input_messages=construct_input_messages) + + class ErrorResponse: + pass + + if request.param == "0.31": + make_request = Serving._make_request + + async def render_resolved(self, request, previous): + messages, inputs = await make_request(self, request, previous) + return SimpleNamespace(messages=messages, engine_input=inputs[0]) + + Serving._render_resolved_response_inputs = render_resolved + del Serving._make_request + modules = { + "vllm.renderers.online_renderer": online_renderer, + "vllm.entrypoints.serve.engine.protocol": SimpleNamespace( + ErrorResponse=ErrorResponse + ), "vllm.renderers.base": SimpleNamespace(BaseRenderer=Renderer), "vllm.renderers.params": SimpleNamespace(ChatParams=Params), "vllm.v1.engine.async_llm": SimpleNamespace(AsyncLLM=Engine), @@ -496,9 +515,13 @@ async def run(): def test_previous_responses_keep_reasoning_and_tool_calls(serving): - _, modules = serving + server, modules = serving previous = [{"type": "reasoning"}, {"type": "function_call"}] - utils = modules["vllm.entrypoints.openai.responses.utils"] + utils = modules[ + "vllm.renderers.online_renderer" + if hasattr(server, "_render_resolved_response_inputs") + else "vllm.entrypoints.openai.responses.utils" + ] assert utils.construct_input_messages( prev_response_output=previous, request_input="next" )["request_input"] == [*previous, {"role": "user", "content": "next"}] @@ -718,12 +741,56 @@ async def run(): asyncio.run(run()) +@pytest.mark.parametrize("serving", ["0.31"], indirect=True) +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.parametrize("render_error", [False, True]) +def test_responses_resolved_inputs_keep_previous_response_and_render_errors( + serving, stream, render_error +): + server, modules = serving + native_render = server._render_resolved_response_inputs + previous = SimpleNamespace(id="previous", output=[]) + seen = [] + + async def render(request, prev_response): + seen.append(prev_response) + if render_error: + return modules["vllm.entrypoints.serve.engine.protocol"].ErrorResponse() + return await native_render(request, prev_response) + + server._render_resolved_response_inputs = render + + async def run(): + server.response_store_lock = asyncio.Lock() + server.response_store = {previous.id: previous} + response = await server.create_responses( + Request( + messages=[{"role": "user", "content": "question"}], + previous_response_id=previous.id, + stream=stream, + ) + ) + if stream: + events = [event async for event in response] + response = events[0].response + assert response.output[0]["content"] == "action" + assert seen == [previous] + assert vllm._CURRENT.get() is None + + asyncio.run(run()) + + @pytest.mark.parametrize("stream", [False, True]) def test_responses_observe_rendered_tool_history_without_rejecting_result( serving, stream ): server, modules = serving - original_make = server._make_request + method = ( + "_make_request" + if hasattr(server, "_make_request") + else "_render_resolved_response_inputs" + ) + original_make = getattr(server, method) rendered = [] class StrictRequest(Request): @@ -737,15 +804,18 @@ def validate_tool_arguments(self): return self async def make_request(request, previous): - conversation, inputs = await original_make(request, previous) + result = await original_make(request, previous) + conversation = result[0] if isinstance(result, tuple) else result.messages conversation = copy.deepcopy(conversation) # vLLM's renderer converts historical JSON argument strings to mappings. function = conversation[1]["tool_calls"][0]["function"] function["arguments"] = json.loads(function["arguments"]) rendered.append(conversation) - return conversation, inputs + if isinstance(result, tuple): + return conversation, result[1] + return SimpleNamespace(messages=conversation, engine_input=result.engine_input) - server._make_request = make_request + setattr(server, method, make_request) modules[ "vllm.entrypoints.openai.chat_completion.protocol" ].ChatCompletionRequest = StrictRequest From 446a22709ee4fca2c126021e73ab08e0215d6ea9 Mon Sep 17 00:00:00 2001 From: Brad Hilton Date: Wed, 21 Oct 2026 22:14:56 +0000 Subject: [PATCH 2/3] test: keep vLLM compatibility fixture type safe --- tests/unit/test_inference_history.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_inference_history.py b/tests/unit/test_inference_history.py index 904afc921..7f47c8442 100644 --- a/tests/unit/test_inference_history.py +++ b/tests/unit/test_inference_history.py @@ -213,7 +213,7 @@ async def render_resolved(self, request, previous): messages, inputs = await make_request(self, request, previous) return SimpleNamespace(messages=messages, engine_input=inputs[0]) - Serving._render_resolved_response_inputs = render_resolved + setattr(Serving, "_render_resolved_response_inputs", render_resolved) del Serving._make_request modules = { From e605b1ef22661298847f2b9fd2b8d2f6b98a5a1b Mon Sep 17 00:00:00 2001 From: Brad Hilton Date: Wed, 21 Oct 2026 22:20:03 +0000 Subject: [PATCH 3/3] fix: preserve DeepSeek history with vLLM 0.31 --- src/art_inference/append_only.py | 2 ++ tests/unit/test_append_only.py | 24 ++++++++++++++++++++---- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/art_inference/append_only.py b/src/art_inference/append_only.py index ef733eccd..733d423c5 100644 --- a/src/art_inference/append_only.py +++ b/src/art_inference/append_only.py @@ -91,6 +91,8 @@ def render_message(index, messages, thinking_mode, *args, **kwargs): # user in the full chat, even with drop_thinking=False. if prefix_only: messages = messages[: index + 1] + # vLLM 0.31 caches the full chat's index; this prefix has its own. + kwargs.pop("last_user_idx", None) message = messages[index] if ( message.get("role") == "assistant" diff --git a/tests/unit/test_append_only.py b/tests/unit/test_append_only.py index 3cde7b2ed..236d6e692 100644 --- a/tests/unit/test_append_only.py +++ b/tests/unit/test_append_only.py @@ -17,9 +17,16 @@ @pytest.mark.parametrize("reasoning", [None, "", "\nthought\n"]) -def test_dsv32_earlier_turn_framing_survives_a_new_user_turn(reasoning): - def render(index, messages, thinking_mode): - last_user = max(i for i, m in enumerate(messages) if m["role"] == "user") +@pytest.mark.parametrize("cached_last_user", [False, True]) +def test_dsv32_earlier_turn_framing_survives_a_new_user_turn( + reasoning, cached_last_user +): + def render(index, messages, thinking_mode, last_user_idx=None): + last_user = ( + max(i for i, m in enumerate(messages) if m["role"] == "user") + if last_user_idx is None + else last_user_idx + ) thinking = thinking_mode == "thinking" if messages[index]["role"] == "user": return "USER" + ( @@ -39,8 +46,17 @@ def render(index, messages, thinking_mode): ] def encode(messages): + kwargs = ( + { + "last_user_idx": max( + i for i, m in enumerate(messages) if m["role"] == "user" + ) + } + if cached_last_user + else {} + ) return "".join( - encoding.render_message(i, messages, "thinking") + encoding.render_message(i, messages, "thinking", **kwargs) for i in range(len(messages)) )