diff --git a/src/llm/apis/openai_completions.cpp b/src/llm/apis/openai_completions.cpp index 810fcdc50a..20ccfe372f 100644 --- a/src/llm/apis/openai_completions.cpp +++ b/src/llm/apis/openai_completions.cpp @@ -409,17 +409,22 @@ std::string OpenAIChatCompletionsHandler::serializeUnaryResponse(ov::genai::Enco // choices: array of size N, where N is related to n request parameter jsonResponse.StartArray("choices"); - int index = 0; - for (int i = 0; i < results.tokens.size(); i++) { + if (results.finish_reasons.empty()) { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Missing finish reason in unary LM generation result, defaulting to STOP for all choices"); + } else if (results.finish_reasons.size() != results.tokens.size()) { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Finish reasons size ({}) does not match tokens size ({}) in unary LM generation result, defaulting missing entries to STOP", + results.finish_reasons.size(), results.tokens.size()); + } + for (size_t i = 0; i < results.tokens.size(); ++i) { const std::vector& tokens = results.tokens[i]; SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Generated tokens: {}", tokens); ParsedOutput parsedOutput = parseOutputIfNeeded(tokens); jsonResponse.StartObject(); - // finish_reason: "stop" in regular scenario, "tool_calls" if output contains tool calls - auto finishReason = mapFinishReason(ov::genai::GenerationFinishReason::STOP, !parsedOutput.toolCalls.empty()); + const ov::genai::GenerationFinishReason finishReasonRaw = i < results.finish_reasons.size() ? results.finish_reasons[i] : ov::genai::GenerationFinishReason::STOP; + auto finishReason = mapFinishReason(finishReasonRaw, !parsedOutput.toolCalls.empty()); jsonResponse.FinishReason(finishReason.value_or("unknown")); // index: integer; Choice index, only n=1 supported anyway - jsonResponse.Index(index++); + jsonResponse.Index(static_cast(i)); if (endpoint == Endpoint::CHAT_COMPLETIONS) { jsonResponse.MessageObject(parsedOutput); @@ -480,8 +485,12 @@ std::string OpenAIChatCompletionsHandler::serializeUnaryResponse(ov::genai::VLMD SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Generated tokens: {}", generatedTokens); ParsedOutput parsedOutput = parseOutputIfNeeded(generatedTokens); jsonResponse.StartObject(); - // finish_reason: "stop" in regular scenario, "tool_calls" if output contains tool calls - auto finishReason = mapFinishReason(ov::genai::GenerationFinishReason::STOP, !parsedOutput.toolCalls.empty()); + if (results.finish_reasons.empty()) { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Missing finish reason in unary VLM generation result, defaulting to STOP"); + } + // Current generation flow uses batch=1, so only finish_reasons[0] is expected here. + const ov::genai::GenerationFinishReason finishReasonRaw = results.finish_reasons.empty() ? ov::genai::GenerationFinishReason::STOP : results.finish_reasons[0]; + auto finishReason = mapFinishReason(finishReasonRaw, !parsedOutput.toolCalls.empty()); jsonResponse.FinishReason(finishReason.value_or("unknown")); // index: integer; Choice index, only n=1 supported anyway jsonResponse.Index(index++); diff --git a/src/llm/apis/openai_responses.cpp b/src/llm/apis/openai_responses.cpp index 60ec1c4f08..89b897dc4a 100644 --- a/src/llm/apis/openai_responses.cpp +++ b/src/llm/apis/openai_responses.cpp @@ -652,17 +652,30 @@ std::string OpenAIResponsesHandler::serializeUnaryResponse(ov::genai::EncodedRes OVMS_PROFILE_FUNCTION(); usage.promptTokens = results.perf_metrics.get_num_input_tokens(); usage.completionTokens = results.perf_metrics.get_num_generated_tokens(); + if (results.finish_reasons.empty()) { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Missing finish reason in unary LM responses generation result, defaulting to STOP"); + } std::vector parsedOutputs; + ov::genai::GenerationFinishReason responsesFinishReason = ov::genai::GenerationFinishReason::STOP; for (const auto& tokens : results.tokens) { parsedOutputs.push_back(parseOutputIfNeeded(tokens)); } - return serializeUnaryResponseImpl(parsedOutputs); + for (const auto& finishReason : results.finish_reasons) { + if (finishReason == ov::genai::GenerationFinishReason::LENGTH) { + responsesFinishReason = ov::genai::GenerationFinishReason::LENGTH; + break; + } + } + return serializeUnaryResponseImpl(parsedOutputs, responsesFinishReason); } std::string OpenAIResponsesHandler::serializeUnaryResponse(ov::genai::VLMDecodedResults& results, const std::string& textResponse) { OVMS_PROFILE_FUNCTION(); usage.promptTokens = results.perf_metrics.get_num_input_tokens(); usage.completionTokens = results.perf_metrics.get_num_generated_tokens(); + if (results.finish_reasons.empty()) { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Missing finish reason in unary VLM responses generation result, defaulting to STOP"); + } // Usage is already correctly set from perf_metrics above — no need for updateUsage. std::vector parsedOutputs; if (!textResponse.empty()) { @@ -677,7 +690,14 @@ std::string OpenAIResponsesHandler::serializeUnaryResponse(ov::genai::VLMDecoded parsedOutputs.push_back(std::move(output)); } } - return serializeUnaryResponseImpl(parsedOutputs); + ov::genai::GenerationFinishReason responsesFinishReason = ov::genai::GenerationFinishReason::STOP; + for (const auto& finishReason : results.finish_reasons) { + if (finishReason == ov::genai::GenerationFinishReason::LENGTH) { + responsesFinishReason = ov::genai::GenerationFinishReason::LENGTH; + break; + } + } + return serializeUnaryResponseImpl(parsedOutputs, responsesFinishReason); } // --- Streaming event building blocks --- diff --git a/src/llm/language_model/legacy/servable.cpp b/src/llm/language_model/legacy/servable.cpp index 4234088a2a..8e244df219 100644 --- a/src/llm/language_model/legacy/servable.cpp +++ b/src/llm/language_model/legacy/servable.cpp @@ -229,7 +229,12 @@ absl::Status LegacyServable::preparePartialResponse(std::shared_ptrlastStreamerCallbackOutput.empty()) { lastTextChunk = lastTextChunk + executionContext->lastStreamerCallbackOutput; } - std::string serializedChunk = executionContext->apiHandler->serializeStreamingChunk(lastTextChunk, ov::genai::GenerationFinishReason::STOP); + if (legacyExecutionContext->results.finish_reasons.empty()) { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Missing finish reason in legacy LM streaming generation result, defaulting to STOP"); + } + // Legacy generation path always runs with batch=1, so we read the single finish reason at index 0. + ov::genai::GenerationFinishReason finishReason = legacyExecutionContext->results.finish_reasons.empty() ? ov::genai::GenerationFinishReason::STOP : legacyExecutionContext->results.finish_reasons[0]; + std::string serializedChunk = executionContext->apiHandler->serializeStreamingChunk(lastTextChunk, finishReason); if (!serializedChunk.empty()) { executionContext->response = wrapTextInServerSideEventMessage(serializedChunk); } diff --git a/src/llm/visual_language_model/legacy/servable.cpp b/src/llm/visual_language_model/legacy/servable.cpp index 1bb2367001..a40dee296e 100644 --- a/src/llm/visual_language_model/legacy/servable.cpp +++ b/src/llm/visual_language_model/legacy/servable.cpp @@ -245,7 +245,12 @@ absl::Status VisualLanguageModelLegacyServable::preparePartialResponse(std::shar if (!executionContext->lastStreamerCallbackOutput.empty()) { lastTextChunk = lastTextChunk + executionContext->lastStreamerCallbackOutput; } - std::string serializedChunk = executionContext->apiHandler->serializeStreamingChunk(lastTextChunk, ov::genai::GenerationFinishReason::STOP); + if (legacyExecutionContext->results.finish_reasons.empty()) { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Missing finish reason in legacy VLM streaming generation result, defaulting to STOP"); + } + // Legacy generation path always runs with batch=1, so we read the single finish reason at index 0. + ov::genai::GenerationFinishReason finishReason = legacyExecutionContext->results.finish_reasons.empty() ? ov::genai::GenerationFinishReason::STOP : legacyExecutionContext->results.finish_reasons[0]; + std::string serializedChunk = executionContext->apiHandler->serializeStreamingChunk(lastTextChunk, finishReason); if (!serializedChunk.empty()) { executionContext->response = wrapTextInServerSideEventMessage(serializedChunk); } diff --git a/src/test/http_openai_handler_test.cpp b/src/test/http_openai_handler_test.cpp index c3a40cba3c..3505379522 100644 --- a/src/test/http_openai_handler_test.cpp +++ b/src/test/http_openai_handler_test.cpp @@ -1713,6 +1713,173 @@ TEST_F(HttpOpenAIHandlerParsingTest, serializeUnaryResponseForResponsesCompleted ASSERT_NE(serialized.find("\"metadata\":{}"), std::string::npos) << serialized; } +TEST_F(HttpOpenAIHandlerParsingTest, serializeUnaryResponseForResponsesEncodedResultsIncompleteOnLength) { + std::string json = R"({ + "model": "llama", + "input": "What is OpenVINO?", + "max_output_tokens": 5 + })"; + doc.Parse(json.c_str()); + ASSERT_FALSE(doc.HasParseError()); + + auto apiHandler = std::make_shared(doc, ovms::Endpoint::RESPONSES, std::chrono::system_clock::now(), *tokenizer); + std::optional maxTokensLimit; + uint32_t bestOfLimit = 0; + std::optional maxModelLength; + ASSERT_EQ(apiHandler->parseRequest(maxTokensLimit, bestOfLimit, maxModelLength), absl::OkStatus()); + + ov::genai::EncodedResults results; + ov::Tensor outputIds = tokenizer->encode("OVMS", ov::genai::add_special_tokens(false)).input_ids; + const auto& shape = outputIds.get_shape(); + ASSERT_EQ(shape.size(), 2); + ASSERT_EQ(shape[0], 1); + ASSERT_EQ(outputIds.get_element_type(), ov::element::i64); + int64_t* outputIdsData = reinterpret_cast(outputIds.data()); + results.tokens = {std::vector(outputIdsData, outputIdsData + shape[1])}; + results.finish_reasons = {ov::genai::GenerationFinishReason::LENGTH}; + + std::string serialized = apiHandler->serializeUnaryResponse(results); + + ASSERT_NE(serialized.find("\"status\":\"incomplete\""), std::string::npos) << serialized; + ASSERT_NE(serialized.find("\"incomplete_details\""), std::string::npos) << serialized; + ASSERT_NE(serialized.find("\"reason\":\"max_tokens\""), std::string::npos) << serialized; + ASSERT_EQ(serialized.find("\"completed_at\""), std::string::npos) << serialized; + ASSERT_EQ(serialized.find("\"status\":\"completed\""), std::string::npos) << serialized; +} + +TEST_F(HttpOpenAIHandlerParsingTest, serializeUnaryResponseForResponsesEncodedResultsCompletedOnStop) { + std::string json = R"({ + "model": "llama", + "input": "What is OpenVINO?", + "max_output_tokens": 5 + })"; + doc.Parse(json.c_str()); + ASSERT_FALSE(doc.HasParseError()); + + auto apiHandler = std::make_shared(doc, ovms::Endpoint::RESPONSES, std::chrono::system_clock::now(), *tokenizer); + std::optional maxTokensLimit; + uint32_t bestOfLimit = 0; + std::optional maxModelLength; + ASSERT_EQ(apiHandler->parseRequest(maxTokensLimit, bestOfLimit, maxModelLength), absl::OkStatus()); + + ov::genai::EncodedResults results; + ov::Tensor outputIds = tokenizer->encode("OVMS", ov::genai::add_special_tokens(false)).input_ids; + int64_t* outputIdsData = reinterpret_cast(outputIds.data()); + results.tokens = {std::vector(outputIdsData, outputIdsData + outputIds.get_shape()[1])}; + results.finish_reasons = {ov::genai::GenerationFinishReason::STOP}; + + std::string serialized = apiHandler->serializeUnaryResponse(results); + + ASSERT_NE(serialized.find("\"status\":\"completed\""), std::string::npos) << serialized; + ASSERT_NE(serialized.find("\"completed_at\""), std::string::npos) << serialized; + ASSERT_EQ(serialized.find("\"incomplete_details\""), std::string::npos) << serialized; +} + +TEST_F(HttpOpenAIHandlerParsingTest, serializeUnaryResponseForResponsesVLMDecodedResultsIncompleteOnLength) { + std::string json = R"({ + "model": "llama", + "input": "What is OpenVINO?", + "max_output_tokens": 5 + })"; + doc.Parse(json.c_str()); + ASSERT_FALSE(doc.HasParseError()); + + auto apiHandler = std::make_shared(doc, ovms::Endpoint::RESPONSES, std::chrono::system_clock::now(), *tokenizer); + std::optional maxTokensLimit; + uint32_t bestOfLimit = 0; + std::optional maxModelLength; + ASSERT_EQ(apiHandler->parseRequest(maxTokensLimit, bestOfLimit, maxModelLength), absl::OkStatus()); + + ov::genai::VLMDecodedResults results; + std::string text = "OVMS"; + results.texts = {text}; + results.finish_reasons = {ov::genai::GenerationFinishReason::LENGTH}; + + std::string serialized = apiHandler->serializeUnaryResponse(results, text); + + ASSERT_NE(serialized.find("\"status\":\"incomplete\""), std::string::npos) << serialized; + ASSERT_NE(serialized.find("\"incomplete_details\""), std::string::npos) << serialized; + ASSERT_NE(serialized.find("\"reason\":\"max_tokens\""), std::string::npos) << serialized; + ASSERT_EQ(serialized.find("\"completed_at\""), std::string::npos) << serialized; + ASSERT_EQ(serialized.find("\"status\":\"completed\""), std::string::npos) << serialized; +} + +TEST_F(HttpOpenAIHandlerParsingTest, serializeUnaryResponseForResponsesVLMDecodedResultsCompletedOnStop) { + std::string json = R"({ + "model": "llama", + "input": "What is OpenVINO?", + "max_output_tokens": 5 + })"; + doc.Parse(json.c_str()); + ASSERT_FALSE(doc.HasParseError()); + + auto apiHandler = std::make_shared(doc, ovms::Endpoint::RESPONSES, std::chrono::system_clock::now(), *tokenizer); + std::optional maxTokensLimit; + uint32_t bestOfLimit = 0; + std::optional maxModelLength; + ASSERT_EQ(apiHandler->parseRequest(maxTokensLimit, bestOfLimit, maxModelLength), absl::OkStatus()); + + ov::genai::VLMDecodedResults results; + std::string text = "OVMS"; + results.texts = {text}; + results.finish_reasons = {ov::genai::GenerationFinishReason::STOP}; + + std::string serialized = apiHandler->serializeUnaryResponse(results, text); + + ASSERT_NE(serialized.find("\"status\":\"completed\""), std::string::npos) << serialized; + ASSERT_NE(serialized.find("\"completed_at\""), std::string::npos) << serialized; + ASSERT_EQ(serialized.find("\"incomplete_details\""), std::string::npos) << serialized; +} + +TEST_F(HttpOpenAIHandlerParsingTest, serializeUnaryResponseChatCompletionsEncodedResultsLengthFinishReason) { + std::string json = R"({ + "model": "llama", + "stream": false, + "messages": [{"role": "user", "content": "What is OpenVINO?"}] + })"; + doc.Parse(json.c_str()); + ASSERT_FALSE(doc.HasParseError()); + + auto apiHandler = std::make_shared(doc, ovms::Endpoint::CHAT_COMPLETIONS, std::chrono::system_clock::now(), *tokenizer); + uint32_t maxTokensLimit = 100; + uint32_t bestOfLimit = 0; + std::optional maxModelLength; + ASSERT_EQ(apiHandler->parseRequest(maxTokensLimit, bestOfLimit, maxModelLength), absl::OkStatus()); + + ov::genai::EncodedResults results; + ov::Tensor outputIds = tokenizer->encode("OVMS", ov::genai::add_special_tokens(false)).input_ids; + int64_t* outputIdsData = reinterpret_cast(outputIds.data()); + results.tokens = {std::vector(outputIdsData, outputIdsData + outputIds.get_shape()[1])}; + results.finish_reasons = {ov::genai::GenerationFinishReason::LENGTH}; + + std::string serialized = apiHandler->serializeUnaryResponse(results); + ASSERT_NE(serialized.find("\"finish_reason\":\"length\""), std::string::npos) << serialized; +} + +TEST_F(HttpOpenAIHandlerParsingTest, serializeUnaryResponseChatCompletionsVLMDecodedResultsLengthFinishReason) { + std::string json = R"({ + "model": "llama", + "stream": false, + "messages": [{"role": "user", "content": "What is OpenVINO?"}] + })"; + doc.Parse(json.c_str()); + ASSERT_FALSE(doc.HasParseError()); + + auto apiHandler = std::make_shared(doc, ovms::Endpoint::CHAT_COMPLETIONS, std::chrono::system_clock::now(), *tokenizer); + uint32_t maxTokensLimit = 100; + uint32_t bestOfLimit = 0; + std::optional maxModelLength; + ASSERT_EQ(apiHandler->parseRequest(maxTokensLimit, bestOfLimit, maxModelLength), absl::OkStatus()); + + ov::genai::VLMDecodedResults results; + std::string text = "OVMS"; + results.texts = {text}; + results.finish_reasons = {ov::genai::GenerationFinishReason::LENGTH}; + + std::string serialized = apiHandler->serializeUnaryResponse(results, text); + ASSERT_NE(serialized.find("\"finish_reason\":\"length\""), std::string::npos) << serialized; +} + TEST_F(HttpOpenAIHandlerParsingTest, ParsingMessagesSucceedsBase64) { std::string json = R"({ "model": "llama", diff --git a/src/test/llm/llmnode_test.cpp b/src/test/llm/llmnode_test.cpp index 99d9e8743c..0e52c1bd1f 100644 --- a/src/test/llm/llmnode_test.cpp +++ b/src/test/llm/llmnode_test.cpp @@ -2685,9 +2685,9 @@ INSTANTIATE_TEST_SUITE_P( ::testing::Values( // params: model name, generate expected output, check logprobs, check finish reason, test speculative decoding, supports empty handshake msg TestParameters{"lm_cb_regular", true, true, true, false, true}, - TestParameters{"lm_legacy_regular", false, false, false, false, false}, + TestParameters{"lm_legacy_regular", false, false, true, false, false}, TestParameters{"vlm_cb_regular", false, true, true, false, true}, - TestParameters{"vlm_legacy_regular", false, false, false, false, false})); + TestParameters{"vlm_legacy_regular", false, false, true, false, false})); const std::string validRequestBodyWithParameter(const std::string& modelName, const std::string& parameter, const std::string& value) { std::string requestBody = R"( @@ -3611,7 +3611,7 @@ INSTANTIATE_TEST_SUITE_P( TestParameters{"lm_cb_regular", true, true, true, false, true}, TestParameters{"lm_legacy_regular", false, false, false, false, false}, TestParameters{"vlm_cb_regular", false, true, true, false, true}, - TestParameters{"vlm_legacy_regular", false, false, false, false, false})); + TestParameters{"vlm_legacy_regular", false, false, true, false, false})); // Common tests for all pipeline types (testing logic executed prior pipeline type selection) class LLMConfigHttpTest : public ::testing::Test {};