From dd7ac8de89fe76cfe00d04d4f69620e00e5aa37e Mon Sep 17 00:00:00 2001 From: DassaultFalconKing Date: Tue, 15 Sep 2026 00:59:53 +0200 Subject: [PATCH 1/2] feat(gemma4): transfer RC tool-calling stack for 2026.4 upstream review --- docs/gemma4/.gitattributes | 3 + docs/gemma4/UPSTREAM-HANDOFF.md | 55 + .../genai-gemma4-bounded-whitespace.patch | 51 + docs/gemma4/evidence/benchmark/aggregate.json | 16 + .../gpu-512-InvokeWebRequest-request.json | 1 + .../gpu-512-InvokeWebRequest-response.json | 1 + .../benchmark/gpu-512-curl-request.json | 1 + .../benchmark/gpu-512-curl-response.json | 1 + .../long-1024-InvokeWebRequest-request.json | 1 + .../long-1024-InvokeWebRequest-response.json | 1 + .../benchmark/long-1024-curl-request.json | 1 + .../benchmark/long-1024-curl-response.json | 1 + .../long-2048-InvokeWebRequest-request.json | 1 + .../long-2048-InvokeWebRequest-response.json | 1 + .../benchmark/long-2048-curl-request.json | 1 + .../benchmark/long-2048-curl-response.json | 1 + .../evidence/benchmark/models-response.json | 1 + .../gemma4/evidence/benchmark/requests.ndjson | 14 + docs/gemma4/evidence/benchmark/run.json | 15 + ...hort-128-101-InvokeWebRequest-request.json | 1 + ...ort-128-101-InvokeWebRequest-response.json | 1 + .../benchmark/short-128-101-curl-request.json | 1 + .../short-128-101-curl-response.json | 1 + ...hort-128-102-InvokeWebRequest-request.json | 1 + ...ort-128-102-InvokeWebRequest-response.json | 1 + .../benchmark/short-128-102-curl-request.json | 1 + .../short-128-102-curl-response.json | 1 + ...hort-128-103-InvokeWebRequest-request.json | 1 + ...ort-128-103-InvokeWebRequest-response.json | 1 + .../benchmark/short-128-103-curl-request.json | 1 + .../short-128-103-curl-response.json | 1 + docs/gemma4/evidence/benchmark/summary.csv | 15 + .../warmup-InvokeWebRequest-request.json | 1 + .../warmup-InvokeWebRequest-response.json | 1 + .../benchmark/warmup-curl-request.json | 1 + .../benchmark/warmup-curl-response.json | 1 + .../evidence/tools/parallel-request.json | 1 + .../evidence/tools/parallel-response.txt | 1 + docs/gemma4/evidence/tools/results.json | 36 + .../gemma4/evidence/tools/single-request.json | 1 + .../gemma4/evidence/tools/single-response.txt | 1 + .../gemma4/evidence/tools/stream-request.json | 1 + .../gemma4/evidence/tools/stream-response.txt | 9 + .../evidence/transfer-source-inventory.json | 182 +++ src/llm/apis/openai_api_handler.hpp | 42 +- src/llm/apis/openai_completions.hpp | 10 + src/llm/apis/openai_request.hpp | 3 + src/llm/apis/openai_responses.cpp | 2 +- src/llm/apis/openai_responses.hpp | 10 + .../base_generation_config_builder.hpp | 7 + src/llm/io_processing/base_output_parser.hpp | 8 + .../io_processing/chat_template/analyzer.cpp | 23 +- src/llm/io_processing/chat_template/caps.hpp | 10 +- .../gemma4/gemma4_reasoning_parser.cpp | 42 +- .../gemma4/gemma4_reasoning_parser.hpp | 59 +- .../gemma4/gemma4_tool_parser.cpp | 1087 ++++++++++++----- .../gemma4/gemma4_tool_parser.hpp | 117 +- .../generation_config_builder.hpp | 206 +++- .../chat_template_adapter.cpp | 28 +- .../chat_template_adapter.hpp | 5 + .../chat_template_processor.cpp | 90 ++ .../chat_template_processor.hpp | 9 + src/llm/io_processing/output_parser.cpp | 190 +-- src/llm/io_processing/output_parser.hpp | 11 +- .../io_processing/output_parsing_config.hpp | 17 +- src/llm/ovms_text_streamer.cpp | 69 +- src/llm/ovms_text_streamer.hpp | 5 +- src/llm/servable.cpp | 512 +++++++- src/llm/servable.hpp | 50 + src/test/llm/gemma4_fast/BUILD | 23 + .../gemma4_parser_contract_test.cpp | 316 +++++ .../gemma4_reasoning_semantic_refit_test.cpp | 186 +++ .../gemma4_recovery_contract_test.cpp | 229 ++++ src/test/llm/gemma4_overlay/BUILD | 29 + ...a4_chat_template_overlay_contract_test.cpp | 128 ++ .../gemma4_google_jinja_contract_test.cpp | 83 ++ src/test/llm/generation_config/BUILD | 54 + .../gemma4_generation_contract_test.cpp | 465 +++++++ ..._prompt_state_generation_contract_test.cpp | 82 ++ ...enai_parallel_tool_calls_contract_test.cpp | 187 +++ 80 files changed, 4332 insertions(+), 492 deletions(-) create mode 100644 docs/gemma4/.gitattributes create mode 100644 docs/gemma4/UPSTREAM-HANDOFF.md create mode 100644 docs/gemma4/dependencies/genai-gemma4-bounded-whitespace.patch create mode 100644 docs/gemma4/evidence/benchmark/aggregate.json create mode 100644 docs/gemma4/evidence/benchmark/gpu-512-InvokeWebRequest-request.json create mode 100644 docs/gemma4/evidence/benchmark/gpu-512-InvokeWebRequest-response.json create mode 100644 docs/gemma4/evidence/benchmark/gpu-512-curl-request.json create mode 100644 docs/gemma4/evidence/benchmark/gpu-512-curl-response.json create mode 100644 docs/gemma4/evidence/benchmark/long-1024-InvokeWebRequest-request.json create mode 100644 docs/gemma4/evidence/benchmark/long-1024-InvokeWebRequest-response.json create mode 100644 docs/gemma4/evidence/benchmark/long-1024-curl-request.json create mode 100644 docs/gemma4/evidence/benchmark/long-1024-curl-response.json create mode 100644 docs/gemma4/evidence/benchmark/long-2048-InvokeWebRequest-request.json create mode 100644 docs/gemma4/evidence/benchmark/long-2048-InvokeWebRequest-response.json create mode 100644 docs/gemma4/evidence/benchmark/long-2048-curl-request.json create mode 100644 docs/gemma4/evidence/benchmark/long-2048-curl-response.json create mode 100644 docs/gemma4/evidence/benchmark/models-response.json create mode 100644 docs/gemma4/evidence/benchmark/requests.ndjson create mode 100644 docs/gemma4/evidence/benchmark/run.json create mode 100644 docs/gemma4/evidence/benchmark/short-128-101-InvokeWebRequest-request.json create mode 100644 docs/gemma4/evidence/benchmark/short-128-101-InvokeWebRequest-response.json create mode 100644 docs/gemma4/evidence/benchmark/short-128-101-curl-request.json create mode 100644 docs/gemma4/evidence/benchmark/short-128-101-curl-response.json create mode 100644 docs/gemma4/evidence/benchmark/short-128-102-InvokeWebRequest-request.json create mode 100644 docs/gemma4/evidence/benchmark/short-128-102-InvokeWebRequest-response.json create mode 100644 docs/gemma4/evidence/benchmark/short-128-102-curl-request.json create mode 100644 docs/gemma4/evidence/benchmark/short-128-102-curl-response.json create mode 100644 docs/gemma4/evidence/benchmark/short-128-103-InvokeWebRequest-request.json create mode 100644 docs/gemma4/evidence/benchmark/short-128-103-InvokeWebRequest-response.json create mode 100644 docs/gemma4/evidence/benchmark/short-128-103-curl-request.json create mode 100644 docs/gemma4/evidence/benchmark/short-128-103-curl-response.json create mode 100644 docs/gemma4/evidence/benchmark/summary.csv create mode 100644 docs/gemma4/evidence/benchmark/warmup-InvokeWebRequest-request.json create mode 100644 docs/gemma4/evidence/benchmark/warmup-InvokeWebRequest-response.json create mode 100644 docs/gemma4/evidence/benchmark/warmup-curl-request.json create mode 100644 docs/gemma4/evidence/benchmark/warmup-curl-response.json create mode 100644 docs/gemma4/evidence/tools/parallel-request.json create mode 100644 docs/gemma4/evidence/tools/parallel-response.txt create mode 100644 docs/gemma4/evidence/tools/results.json create mode 100644 docs/gemma4/evidence/tools/single-request.json create mode 100644 docs/gemma4/evidence/tools/single-response.txt create mode 100644 docs/gemma4/evidence/tools/stream-request.json create mode 100644 docs/gemma4/evidence/tools/stream-response.txt create mode 100644 docs/gemma4/evidence/transfer-source-inventory.json create mode 100644 src/test/llm/gemma4_fast/BUILD create mode 100644 src/test/llm/gemma4_fast/gemma4_parser_contract_test.cpp create mode 100644 src/test/llm/gemma4_fast/gemma4_reasoning_semantic_refit_test.cpp create mode 100644 src/test/llm/gemma4_fast/gemma4_recovery_contract_test.cpp create mode 100644 src/test/llm/gemma4_overlay/BUILD create mode 100644 src/test/llm/gemma4_overlay/gemma4_chat_template_overlay_contract_test.cpp create mode 100644 src/test/llm/gemma4_overlay/gemma4_google_jinja_contract_test.cpp create mode 100644 src/test/llm/generation_config/BUILD create mode 100644 src/test/llm/generation_config/gemma4_generation_contract_test.cpp create mode 100644 src/test/llm/generation_config/gemma4_prompt_state_generation_contract_test.cpp create mode 100644 src/test/llm/generation_config/openai_parallel_tool_calls_contract_test.cpp diff --git a/docs/gemma4/.gitattributes b/docs/gemma4/.gitattributes new file mode 100644 index 0000000000..e6692865b2 --- /dev/null +++ b/docs/gemma4/.gitattributes @@ -0,0 +1,3 @@ +# Preserve upstream patch context and raw SSE framing without whitespace rewriting. +dependencies/*.patch -text whitespace=-blank-at-eol,-blank-at-eof +evidence/tools/*-response.txt -text whitespace=-blank-at-eol,-blank-at-eof diff --git a/docs/gemma4/UPSTREAM-HANDOFF.md b/docs/gemma4/UPSTREAM-HANDOFF.md new file mode 100644 index 0000000000..f49d2afbdb --- /dev/null +++ b/docs/gemma4/UPSTREAM-HANDOFF.md @@ -0,0 +1,55 @@ +# Gemma4 2026.4 stack transfer — draft + +Source: downstream RC `170644006a5334cb971b05824e4a8c95b495c4e2`. Target: upstream releases/2026/4 at `869b2186a004c6d7eba654db1b03b701bd80757f`. + +## Scope + +This transfers the custom registry-aware native/JSON tool parser, quoted call boundaries and nested arguments, independent reasoning parser, reasoning-to-tool routing, Google-template/history and rendered-prompt adaptation, auto/required/named guided grammars, parallel_tool_calls validation, opt-in session journal/seed state, and actual terminal streamer finish reasons/incomplete-frame diagnostics. Tool schemas carry max_whitespace_cnt=2 through the typed GenAI API. + +Session persistence activates through OVMS_SESSION_STORE_DIR plus X-OVMS-Session-ID. Bodies are journaled on disk with bounded size/cache limits. This storage/API addition and endpoint-wide hard-choice validation need separate upstream design/security review and may need separate PRs. + +The upstream logprob fix and all existing upstream Gemma4 parser tests are retained. Windows build-policy changes, other modalities, fork branding and later cache diagnostics/cache-off experiments are excluded. Generic utility helpers remain available for upstream users. + +## Dependency blocker + +The target GenAI dependency lacks JSONSchema(schema, optional whitespace_bound). The frozen GenAI patch is attached under dependencies for review; it is NOT applied by the upstream build. A companion GenAI API/serialization/matcher change and dependency update are required before this draft can compile. An unbounded fallback would invalidate the repair. + +Downstream runtime tuple: OpenVINO `227c33757d1ef95d4da506d00686f923fdd2a535`, GenAI base `7ea2546852a382cd16bd22dea0cfad2db70ed744` plus attached patch, Tokenizers `a04accf6282d9b304214b492694b18c3979f667a`, XGrammar `9aa840b6d16abf094f3e8e2ac9c10465b77656c9`. + +[Frozen package and provenance](https://github.com/DassaultFalconKing/gemmamonster_model_server_OVMS/releases/tag/gemmamonster-2026.4-rc-whitespace-17064400). + +## Downstream live evidence + +2026-09-14 22:44–22:49 UTC; already-running Windows RC, Intel Arc 140V 16GB, driver 32.0.101.8991, GPU/VLM_CB, Gemma4 26B A4B heretic INT4, prefix caching and DEBUG enabled. Concurrent client traffic was not controlled. + +| Check on frozen RC | Result | +|---|---| +| Non-stream benchmark | 14/14 HTTP 200 | +| Long requests, curl | 1797 actual tokens / 72.979 s = 24.624 tok/s | +| Long requests, Invoke-WebRequest | 2186 actual tokens / 86.639 s = 25.231 tok/s | +| Named single echo | PASS, exact arguments, 1 call, tool_calls finish | +| Two same-name parallel echo calls | PASS, exact arguments, 2 calls, tool_calls finish | +| Named SSE echo | PASS, reconstructed arguments, 1 call, tool_calls finish | + +Raw synthetic requests/responses and summaries are under evidence. Tool cases: temperature=0, seed=170644, max_tokens=256. Benchmark: temperature=1, top_k=64, top_p=0.95, preserved seeds/prompts; timing includes prefill/HTTP/possible queueing. Long outputs stopped before max_tokens and metrics use actual usage. Cold startup, TTFT, concurrency, factual accuracy of free benchmark texts, multi-turn/real-tool execution and session persistence were NOT RUN in this campaign. + +These results belong to the frozen downstream RC, NOT the assembled upstream head. The original candidate manifest's historical live acceptance NOT_RUN is not overwritten. + +## Merge gates + +- Review/land the companion GenAI API and update dependencies coherently. +- Build product and run the six targets below on this exact PR head. +- Run every retained upstream parser regression and generic parser/streamer coverage plus Linux/Windows CI; no unsupported exclusions. +- Review journal filesystem/seed/API behavior and hard-choice policy; split scope if maintainers prefer. +- Run packaged repeated, streaming, multi-turn/tool-result semantic acceptance with raw evidence on this head. + +```text +//src/test/llm/gemma4_fast:gemma4_parser_contract_test +//src/test/llm/generation_config:gemma4_generation_contract_test +//src/test/llm/generation_config:gemma4_prompt_state_generation_contract_test +//src/test/llm/generation_config:openai_parallel_tool_calls_contract_test +//src/test/llm/gemma4_overlay:gemma4_chat_template_overlay_contract_test +//src/test/llm/gemma4_overlay:gemma4_google_jinja_contract_test +``` + +Build, executable tests and broad CI on the transferred head: NOT RUN. This remains draft while gates are open. diff --git a/docs/gemma4/dependencies/genai-gemma4-bounded-whitespace.patch b/docs/gemma4/dependencies/genai-gemma4-bounded-whitespace.patch new file mode 100644 index 0000000000..9d6b0e40c8 --- /dev/null +++ b/docs/gemma4/dependencies/genai-gemma4-bounded-whitespace.patch @@ -0,0 +1,51 @@ +diff --git a/src/cpp/CMakeLists.txt b/src/cpp/CMakeLists.txt +index 463542c9..42bbe6a3 100644 +--- a/src/cpp/CMakeLists.txt ++++ b/src/cpp/CMakeLists.txt +@@ -152,7 +152,7 @@ if(ANDROID) + endif() + + if(ENABLE_XGRAMMAR) +- set(XGRAMMAR_VERSION v0.1.31) ++ set(XGRAMMAR_VERSION 9aa840b6d16abf094f3e8e2ac9c10465b77656c9) + set(XGRAMMAR_DIR ${CMAKE_BINARY_DIR}/xgrammar) + + FetchContent_Declare( +diff --git a/src/cpp/include/openvino/genai/generation_config.hpp b/src/cpp/include/openvino/genai/generation_config.hpp +index 26b230d3..9724c1a7 100644 +--- a/src/cpp/include/openvino/genai/generation_config.hpp ++++ b/src/cpp/include/openvino/genai/generation_config.hpp +@@ -5,6 +5,7 @@ + + #include + #include ++#include + #include + #include + #include +@@ -157,17 +158,21 @@ public: + */ + struct JSONSchema { + std::string value; ++ std::optional max_whitespace_cnt; + + JSONSchema() = default; +- JSONSchema(const std::string& schema) : value(schema) {} ++ JSONSchema(const std::string& schema, std::optional whitespace_bound = std::nullopt) ++ : value(schema), max_whitespace_cnt(whitespace_bound) {} + std::string to_string() const { +- return "JSONSchema(\"" + value + "\")"; ++ return "JSONSchema(\"" + value + "\"" + ++ (max_whitespace_cnt ? ", max_whitespace_cnt=" + std::to_string(*max_whitespace_cnt) : "") + ")"; + } + std::string to_json() const { +- return std::string("{\"type\": \"json_schema\", \"json_schema\": ") + value + "}"; ++ return std::string("{\"type\": \"json_schema\", \"json_schema\": ") + value + ++ (max_whitespace_cnt ? ", \"max_whitespace_cnt\": " + std::to_string(*max_whitespace_cnt) : "") + "}"; + } + bool operator==(const JSONSchema& other) const { +- return value == other.value; ++ return value == other.value && max_whitespace_cnt == other.max_whitespace_cnt; + } + }; + diff --git a/docs/gemma4/evidence/benchmark/aggregate.json b/docs/gemma4/evidence/benchmark/aggregate.json new file mode 100644 index 0000000000..25de546d12 --- /dev/null +++ b/docs/gemma4/evidence/benchmark/aggregate.json @@ -0,0 +1,16 @@ +[ + { + "transport": "curl", + "cases": 2, + "completion_tokens": 1797, + "elapsed_s": 72.979, + "weighted_tokens_per_s": 24.624 + }, + { + "transport": "InvokeWebRequest", + "cases": 2, + "completion_tokens": 2186, + "elapsed_s": 86.639, + "weighted_tokens_per_s": 25.231 + } +] diff --git a/docs/gemma4/evidence/benchmark/gpu-512-InvokeWebRequest-request.json b/docs/gemma4/evidence/benchmark/gpu-512-InvokeWebRequest-request.json new file mode 100644 index 0000000000..9b380d5563 --- /dev/null +++ b/docs/gemma4/evidence/benchmark/gpu-512-InvokeWebRequest-request.json @@ -0,0 +1 @@ +{"model":"gemma4-26-heretic","messages":[{"role":"user","content":"Write a continuous detailed technical explanation of GPU transformer inference optimization. Do not use tools, headings, lists, or a conclusion. Keep expanding the explanation until the token limit."}],"max_tokens":512,"temperature":1.0,"top_k":64,"top_p":0.95,"seed":777} diff --git a/docs/gemma4/evidence/benchmark/gpu-512-InvokeWebRequest-response.json b/docs/gemma4/evidence/benchmark/gpu-512-InvokeWebRequest-response.json new file mode 100644 index 0000000000..3271f3f3ac --- /dev/null +++ b/docs/gemma4/evidence/benchmark/gpu-512-InvokeWebRequest-response.json @@ -0,0 +1 @@ +{"choices":[{"finish_reason":"length","index":0,"logprobs":null,"message":{"content":"The optimization of transformer inference on a GPU is a multifaceted engineering challenge that necessitates a deep understanding of the interplay between memory bandwidth, compute throughput, and the fundamental mathematical structures of the attention mechanism. At the core of the transformer architecture lies the self-attention mechanism, which scales quadratically with sequence length, creating a significant computational bottleneck. To optimize this, practitioners employ techniques such as FlashAttention, which addresses the memory-bound nature of the attention computation by utilizing tiling and recomputation to minimize the movement of data between the high-bandwidth memory (HBM) and the on-chip SRAM. By restructuring the computation into blocks that fit within the fast SRAM, FlashAttention reduces the number of read/write operations to the slower HBM, effectively transforming a memory-bound operation into a compute-bound one through better utilization of the GPU's streaming multiprocessors. Furthermore, the inference process is divided into two distinct phases: the prefill phase and the decoding phase. The prefill phase involves processing the initial input prompt, which is highly parallelizable and compute-intensive, while the decoding phase involves generating tokens one by one, which is inherently sequential and memory-bound due to the low arithmetic intensity of loading the entire Key-Value (KV) cache for each new token. To mitigate the overhead of the decoding phase, techniques like PagedAttention are employed, which manage the KV cache using a memory management scheme inspired by operating systems, utilizing non-contiguous memory blocks to reduce fragmentation and allow for higher batch sizes. This prevents the \"out of memory\" errors often encountered when using static memory allocation for long sequences. As the model moves from prefill to decoding, the bottleneck shifts from compute-bound to memory-bandwidth-bound, necessitating optimizations like continuous batching, where new requests are interleaved with ongoing generation to maximize GPU utilization without waiting for a full batch to complete. Quantization also plays a critical role in increasing throughput by reducing the precision of the weights and activations. By converting 16-bit floating-point (FP16 or BF16) weights to 8-bit (INT8) or even 4-bit (INT4) integers, one can significantly reduce the memory footprint and the amount of data transferred from HBM to the registers. This reduction in data movement is crucial because, in many modern LLM inference scenarios, the GPU's performance is limited by how fast it can pull data from memory rather than how many floating-point operations it can perform per second. However, low-precision arithmetic requires careful","role":"assistant","tool_calls":[]}}],"created":1789426100,"model":"gemma4-26-heretic","object":"chat.completion","usage":{"prompt_tokens":47,"completion_tokens":512,"total_tokens":559}} diff --git a/docs/gemma4/evidence/benchmark/gpu-512-curl-request.json b/docs/gemma4/evidence/benchmark/gpu-512-curl-request.json new file mode 100644 index 0000000000..9b380d5563 --- /dev/null +++ b/docs/gemma4/evidence/benchmark/gpu-512-curl-request.json @@ -0,0 +1 @@ +{"model":"gemma4-26-heretic","messages":[{"role":"user","content":"Write a continuous detailed technical explanation of GPU transformer inference optimization. Do not use tools, headings, lists, or a conclusion. Keep expanding the explanation until the token limit."}],"max_tokens":512,"temperature":1.0,"top_k":64,"top_p":0.95,"seed":777} diff --git a/docs/gemma4/evidence/benchmark/gpu-512-curl-response.json b/docs/gemma4/evidence/benchmark/gpu-512-curl-response.json new file mode 100644 index 0000000000..204ab0a267 --- /dev/null +++ b/docs/gemma4/evidence/benchmark/gpu-512-curl-response.json @@ -0,0 +1 @@ +{"choices":[{"finish_reason":"length","index":0,"logprobs":null,"message":{"content":"The optimization of transformer inference on modern graphics processing units (GPUs) necessitates a multi-faceted approach that addresses the fundamental bottlenecks of memory bandwidth and compute throughput, specifically focusing on the distinct phases of the auto-regressive generation process. During the prefill phase, where the entire input prompt is processed in parallel to compute the initial key and value states, the operation is compute-bound, meaning the primary objective is maximizing the utilization of tensor cores through high-performance matrix multiplications. As the model transitions into the decoding phase, where tokens are generated one by one, the computational profile shifts dramatically toward being memory-bandwidth bound due to the repetitive loading of the existing key-value cache from high-bandwidth memory (HBM) to the much faster but smaller on-chip SRAM. To mitigate this bottleneck, techniques like PagedAttention revolutionize memory management by implementing a virtual memory paging system for the KV cache, allowing for non-contiguous physical memory allocation which eliminates the fragmentation problems inherent in traditional static allocation methods. By utilizing a block-based memory management strategy similar to operating system paging, the system can allocate memory dynamically for the key-value pairs of each sequence, significantly increasing the effective batch size and reducing the waste of VRAM. Parallel to memory management, the optimization of kernel execution involves fusing multiple operations into a single GPU kernel to reduce the overhead of memory round-trips between the global memory and the register files. Kernel fusion is critical when dealing with the activation functions and normalization layers following a projection layer; instead of writing the intermediate result of a linear transformation back to HBM and then reading it back for a subsequent Softmax or LayerNorm, the operations are fused into a single computational unit that keeps the data in the L1 cache or shared memory. Another critical component is the implementation of quantization techniques such as FP8, INT8, or even INT4 precision for the weights and activations. Quantization reduces the memory footprint of the model, allowing larger models to fit within the constraints of a single GPU or enabling larger batch sizes, while simultaneously increasing throughput by reducing the total amount of data that must be moved from HBM to the streaming multiprocessors. However, quantization introduces the risk of precision loss, which is managed through sophisticated scaling factors and techniques like SmoothQuant, which balances the dynamic range between weights and activations to ensure that quantization errors do not accumulate across the deep layers of the transformer architecture. Furthermore, the deployment of speculative decoding introduces a computational strategy where a smaller, faster \"draft\" model predicts a sequence of tokens that are subsequently verified by the","role":"assistant","tool_calls":[]}}],"created":1789426080,"model":"gemma4-26-heretic","object":"chat.completion","usage":{"prompt_tokens":47,"completion_tokens":512,"total_tokens":559}} \ No newline at end of file diff --git a/docs/gemma4/evidence/benchmark/long-1024-InvokeWebRequest-request.json b/docs/gemma4/evidence/benchmark/long-1024-InvokeWebRequest-request.json new file mode 100644 index 0000000000..d432fa2871 --- /dev/null +++ b/docs/gemma4/evidence/benchmark/long-1024-InvokeWebRequest-request.json @@ -0,0 +1 @@ +{"model":"gemma4-26-heretic","messages":[{"role":"user","content":"Write a detailed continuous technical discussion of local large-language-model inference, memory bandwidth, KV caching, batching, and latency. Do not use tools, headings, bullet lists, or conclude early. Keep expanding naturally until the 1024 token output limit."}],"max_tokens":1024,"temperature":1.0,"top_k":64,"top_p":0.95,"seed":8024} diff --git a/docs/gemma4/evidence/benchmark/long-1024-InvokeWebRequest-response.json b/docs/gemma4/evidence/benchmark/long-1024-InvokeWebRequest-response.json new file mode 100644 index 0000000000..1d18f66b72 --- /dev/null +++ b/docs/gemma4/evidence/benchmark/long-1024-InvokeWebRequest-response.json @@ -0,0 +1 @@ +{"choices":[{"finish_reason":"stop","index":0,"logprobs":null,"message":{"content":"The process of local large language model inference is fundamentally a struggle against the physical limitations of hardware, specifically the dichotomy between computational throughput and memory bandwidth. When running a transformer-based architecture locally, the primary bottleneck is rarely the raw arithmetic capability of the GPU cores—measured in TFLOPS—but rather the speed at which data can be moved from the VRAM to the registers, a metric known as memory bandwidth. During the generation phase of inference, the model operates in an autoregressive fashion, meaning it predicts one token at a time. For each new token produced, the entire set of model weights must be read from the memory and loaded into the processor to perform the matrix-vector multiplications required for the forward pass. Because the arithmetic intensity—the ratio of computations to memory accesses—is relatively low during the generation phase, the GPU spends most of its time waiting for the weights to arrive from the memory bus, making the inference process \"memory-bandwidth bound.\"\n\nAs the sequence length grows, a new phenomenon emerges that complicates this landscape: the KV cache. To avoid the redundant computation of recalculating the attention mechanism for all previous tokens every time a new token is generated, the system stores the Key and Value matrices of all preceding tokens in a dedicated memory buffer known as the KV cache. While this drastically reduces the computational load by turning an $O(n^2)$ problem into something more manageable, it introduces a severe memory footprint issue. The KV cache scales linearly with the sequence length, the batch size, and the number of attention heads. In a local environment with limited VRAM, the KV cache can quickly consume the majority of the available memory, leaving little room for the actual model weights. This creates a direct trade-off between the maximum context window a user can support and the batch size of the inference.\n\nTo increase throughput in a server-side or multi-user context, batching is employed, where multiple independent queries are processed simultaneously in a single forward pass. This increases the arithmetic intensity because a single weight loading operation can be reused for multiple queries, effectively amortizing the cost of memory access across many tokens. However, batching is the enemy of latency. As the batch size increases, the time required to process a single batch increases, leading to higher per-token latency. For a local user, high latency is the primary metric of quality, as it dictates the \"perceived\" speed of the model; if the time between one token being generated and the next exceeds the human reading speed, the experience degrades. This creates a complex optimization problem where one must balance the throughput gains of batching against the latency requirements of a single user.\n\nThe interaction between memory bandwidth, KV cache, and batching reaches a breaking point at high context lengths. When the KV cache grows so large that it spills out of high-speed VRAM into slower system RAM, the performance collapses due to the massive latency penalty of moving data across the PCIe bus. Techniques like PagedAttention, which manages the KV cache in non-contiguous memory blocks similar to virtual memory in an operating system, have emerged to mitigate this by reducing fragmentation and allowing more efficient use of available memory. Furthermore, quantization techniques like 4-bit or 8-bit integer arithmetic serve as a proxy for bandwidth relief. By reducing the precision of the weights and the KV cache, we effectively increase the \"virtual\" bandwidth, allowing more data to be moved per clock cycle and enabling larger batch sizes or longer context windows within the same hardware constraints. The ultimate goal in local inference is finding the \"sweet spot\" where the quantization level is low enough to fit the model and cache into memory, the batch size is high enough to saturate the compute units, but the latency remains low enough to maintain a fluid, real-time interaction. This delicate balance of memory management, precision, and parallelization defines the current state of local LLM deployment.","role":"assistant","tool_calls":[]}}],"created":1789425921,"model":"gemma4-26-heretic","object":"chat.completion","usage":{"prompt_tokens":68,"completion_tokens":789,"total_tokens":857}} diff --git a/docs/gemma4/evidence/benchmark/long-1024-curl-request.json b/docs/gemma4/evidence/benchmark/long-1024-curl-request.json new file mode 100644 index 0000000000..d432fa2871 --- /dev/null +++ b/docs/gemma4/evidence/benchmark/long-1024-curl-request.json @@ -0,0 +1 @@ +{"model":"gemma4-26-heretic","messages":[{"role":"user","content":"Write a detailed continuous technical discussion of local large-language-model inference, memory bandwidth, KV caching, batching, and latency. Do not use tools, headings, bullet lists, or conclude early. Keep expanding naturally until the 1024 token output limit."}],"max_tokens":1024,"temperature":1.0,"top_k":64,"top_p":0.95,"seed":8024} diff --git a/docs/gemma4/evidence/benchmark/long-1024-curl-response.json b/docs/gemma4/evidence/benchmark/long-1024-curl-response.json new file mode 100644 index 0000000000..9201dd398b --- /dev/null +++ b/docs/gemma4/evidence/benchmark/long-1024-curl-response.json @@ -0,0 +1 @@ +{"choices":[{"finish_reason":"stop","index":0,"logprobs":null,"message":{"content":"The computational architecture of local large language model inference is a complex interplay between compute-bound and memory-bound phases, where the fundamental bottleneck shifts depending on the generation phase of the sequence. During the initial prompt processing, known as the prefill phase, the system functions in a compute-bound regime where the hardware's floating-point arithmetic capabilities are heavily utilized to process the input tokens in parallel. However, as the model enters the auto-regressive decoding phase, where it generates tokens one by one, the process transitions into a memory-bandwidth-bound regime. In this state, the time taken to generate a single token is not determined by how fast the GPU can perform matrix multiplications, but rather by how quickly the system can move the massive weight matrices of the model from the VRAM into the processing cores. Because every single token generation requires reading the entire model's weights, the memory bandwidth becomes the primary constraint on latency. This is where the concept of KV caching becomes essential for maintaining performance. During inference, the model does not re-calculate the mathematical representations of all previous tokens in a sequence every time a new token is generated; instead, it stores the Key and Value projections of previous tokens in a dedicated cache within the GPU memory. This mechanism significantly reduces redundant computations, effectively transforming the problem from a computational complexity issue to a memory capacity and bandwidth issue. As the context window grows, the KV cache grows proportionally, eventually leading to a tension between the amount of memory required to hold these caches and the memory required to hold the model weights themselves. This tension is a central challenge in local deployment, as consumers often have limited VRAM. To optimize throughput, techniques such as batching are employed, allowing the hardware to process multiple sequences simultaneously. While batching increases the total throughput—the number of tokens generated per second across all users—it typically comes at the cost of increased latency per individual token. This occurs because larger batches require more memory bandwidth to shuffle different sets of KV caches and weights, potentially leading to more contention and slower response times for the end user. The relationship between batch size and latency is non-linear; as you increase batch size, you maximize the utilization of the GPU's parallel cores, but you eventually hit a \"memory wall\" where the overhead of managing large batches offsets the gains in throughput. Furthermore, modern quantization techniques like 4-bit or 8-bit weight compression are used to alleviate these memory constraints, allowing larger models to fit into consumer-grade hardware. While quantization reduces the memory footprint and increases effective bandwidth by reducing the amount of data moved per token, it introduces a trade-off in terms of precision and model intelligence. The deployment of local models ultimately requires a delicate balancing of these four pillars: minimizing latency for a responsive user experience, maximizing throughput for efficiency, optimizing KV cache management to handle long contexts, and managing memory bandwidth to ensure the hardware is not sitting idle while waiting for data to arrive from the VRAM. As context windows expand toward hundreds of thousands of tokens, the memory requirements for the KV cache can actually exceed the memory required for the model weights themselves, making sophisticated paging mechanisms, such as PagedAttention, critical for managing memory fragmentation and ensuring that local inference remains performant and scalable on limited hardware. The interaction between these components determines whether a model feels instantaneous or sluggish, as every millisecond spent moving data from VRAM to the registers is a millisecond where the execution units are effectively stalled. Consequently, the frontier of local LLM optimization lies in hardware-aware software engineering that aligns memory access patterns with the physical realities of the silicon.","role":"assistant","tool_calls":[]}}],"created":1789425891,"model":"gemma4-26-heretic","object":"chat.completion","usage":{"prompt_tokens":68,"completion_tokens":728,"total_tokens":796}} \ No newline at end of file diff --git a/docs/gemma4/evidence/benchmark/long-2048-InvokeWebRequest-request.json b/docs/gemma4/evidence/benchmark/long-2048-InvokeWebRequest-request.json new file mode 100644 index 0000000000..96463767f4 --- /dev/null +++ b/docs/gemma4/evidence/benchmark/long-2048-InvokeWebRequest-request.json @@ -0,0 +1 @@ +{"model":"gemma4-26-heretic","messages":[{"role":"user","content":"Write a detailed continuous technical discussion of local large-language-model inference, memory bandwidth, KV caching, batching, and latency. Do not use tools, headings, bullet lists, or conclude early. Keep expanding naturally until the 2048 token output limit."}],"max_tokens":2048,"temperature":1.0,"top_k":64,"top_p":0.95,"seed":9048} diff --git a/docs/gemma4/evidence/benchmark/long-2048-InvokeWebRequest-response.json b/docs/gemma4/evidence/benchmark/long-2048-InvokeWebRequest-response.json new file mode 100644 index 0000000000..0660ef1db9 --- /dev/null +++ b/docs/gemma4/evidence/benchmark/long-2048-InvokeWebRequest-response.json @@ -0,0 +1 @@ +{"choices":[{"finish_reason":"stop","index":0,"logprobs":null,"message":{"content":"The fundamental bottleneck in local large language model inference is rarely the raw computational throughput of the hardware, often referred to as peak FLOPS, but rather the memory bandwidth that determines how quickly weights and intermediate states can be moved from VRAM to the processing cores. When running a model locally, the computational profile shifts dramatically between the prefill stage and the decoding stage. During the prefill stage, the model processes the entire input prompt simultaneously, a task that is highly compute-intensive and benefits significantly from the parallel processing capabilities of a GPU. However, once the model begins generating tokens one by one during the decoding phase, the workload transforms into a memory-bandwidth-bound problem. This occurs because, for every single token generated, the entire parameter set of the model must be read from the memory and loaded into the registers of the GPU to compute the next token. If a model has 70 billion parameters, and each parameter is stored in 16-bit floating point format, the system must move 140 GB of data just to produce one token. If the memory bandwidth is 1 TB/s, the theoretical maximum generation speed is limited by the constant movement of these weights, regardless of how many cores are available to perform the arithmetic. This phenomenon highlights the critical tension between model size and inference speed; as models scale, the memory requirements grow linearly, while the memory bandwidth often fails to scale at the same rate, leading to diminishing returns in perceived speed for local users.\n\nTo mitigate the inefficiency of generating tokens one by one, techniques such as batching are employed to increase throughput, but batching introduces a complex trade-off with latency. Batching involves processing multiple sequences simultaneously, which allows the system to amortize the cost of loading the model weights over multiple tokens. In a batching scenario, the cost of loading the weights is shared across $N$ sequences, effectively increasing the total tokens per second the hardware can produce. However, increasing the batch size increases the memory footprint and can lead to higher latency for individual requests. In a local environment, where resources are constrained, the user typically wants low latency for a single stream of text, meaning a batch size of one is often used. If we move to more advanced serving architectures like continuous batching or iteration-level scheduling, the system attempts to insert new requests into the batch as soon as others reach an end-token state, but this requires sophisticated management of the GPU memory to avoid fragmentation and excessive overhead.\n\nOne of the most significant innovations in making large-scale inference viable is the implementation of KV (Key-Value) caching. During the generation process, the attention mechanism requires access to the Key and Value vectors of all previous tokens in the sequence to compute the probability of the next token. If we were to recompute these vectors from scratch every time a new token is generated, the computational cost would grow quadratically with the sequence length, making long-context generation practically impossible. KV caching solves this by storing the computed Key and Value tensors in memory so they can be reused in subsequent steps. While this drastically reduces the compute burden, it introduces a massive memory management challenge. The KV cache grows linearly with both the batch size and the sequence length. In a multi-user or multi-tasking environment, the VRAM required for the KV cache can quickly exceed the VRAM required for the model weights themselves, especially when dealing with long contexts of 32k, 64k, or even 128k tokens. This leads to the problem of memory fragmentation, where the VRAM is full of small, non-contiguous chunks of data that are difficult to allocate efficiently. Techniques like PagedAttention, which borrows concepts from operating system virtual memory, attempt to solve this by allocating KV cache in non-contiguous blocks, allowing for much more efficient memory utilization and enabling much larger batch sizes and longer contexts.\n\nThe interplay between memory bandwidth, KV cache size, and batching creates a complex optimization space for local LLM deployment. For a single-user local setup, the goal is usually minimizing the time-to-first-token (TTFT) and the inter-token latency (the time between each generated token). TTFT is primarily driven by the compute-heavy prefill stage and the bandwidth required to load the model weights for the initial prompt. Inter-token latency is driven by the memory bandwidth required to fetch weights and the KV cache for every single token generated. As the sequence grows longer, the KV cache becomes a larger consumer of bandwidth, and the sheer volume of data being moved through the memory bus starts to slow down the generation rate. This is why quantization becomes so vital. By reducing the precision of the model weights from 16-bit to 8-bit, 4-bit, or even lower, we effectively double or quadruple the memory bandwidth efficiency, allowing for faster token generation or larger batch sizes within the same hardware constraints. However, quantization is not a free lunch; reducing precision can lead to a degradation in the model's reasoning capabilities and coherence. Furthermore, the KV cache itself can be quantized, a technique that significantly extends the maximum sequence length a local consumer GPU can handle, though it requires careful implementation to maintain accuracy in the attention mechanism.\n\nThe architectural design of the hardware itself dictates the limits of these software optimizations. In the hierarchy of memory, the speed gap between HBM (High Bandwidth Memory) used in data-center GPUs like the H100 and the standard GDDR6 used in consumer cards like the RTX 4090 is profound. A consumer card might have 1 TB/s of bandwidth, whereas a data-center card can have several TB/s. This gap determines how many concurrent streams a local user can run or how long a single context can be before the system hits a hard wall of VRAM limitations. When we combine these factors, we see that the pursuit of \"faster\" local LLM inference is actually a multi-front war: one against the computational limits of the processor (compute-bound), one against the movement of weights (bandwidth-bound), and one against the capacity of the memory (capacity-bound). Advanced inference engines try to balance these by using techniques like speculative decoding, where a much smaller, faster model generates a draft of text that is then verified by a larger, more accurate model. This leverages the high-speed, low-latency nature of the small model to mask the high-latency, high-compute nature of the large model, effectively turning a compute-bound problem into a more efficient usage of the available throughput. Ultimately, the efficiency of local LLM inference is a function of how well the software can orchestrate the movement of data to keep the compute cores saturated while managing the memory-intensive KV cache and the bandwidth-heavy weight loading in a way that respects the physical limits of the hardware.","role":"assistant","tool_calls":[]}}],"created":1789425995,"model":"gemma4-26-heretic","object":"chat.completion","usage":{"prompt_tokens":68,"completion_tokens":1397,"total_tokens":1465}} diff --git a/docs/gemma4/evidence/benchmark/long-2048-curl-request.json b/docs/gemma4/evidence/benchmark/long-2048-curl-request.json new file mode 100644 index 0000000000..96463767f4 --- /dev/null +++ b/docs/gemma4/evidence/benchmark/long-2048-curl-request.json @@ -0,0 +1 @@ +{"model":"gemma4-26-heretic","messages":[{"role":"user","content":"Write a detailed continuous technical discussion of local large-language-model inference, memory bandwidth, KV caching, batching, and latency. Do not use tools, headings, bullet lists, or conclude early. Keep expanding naturally until the 2048 token output limit."}],"max_tokens":2048,"temperature":1.0,"top_k":64,"top_p":0.95,"seed":9048} diff --git a/docs/gemma4/evidence/benchmark/long-2048-curl-response.json b/docs/gemma4/evidence/benchmark/long-2048-curl-response.json new file mode 100644 index 0000000000..0e1efcf691 --- /dev/null +++ b/docs/gemma4/evidence/benchmark/long-2048-curl-response.json @@ -0,0 +1 @@ +{"choices":[{"finish_reason":"stop","index":0,"logprobs":null,"message":{"content":"The architecture of local large language model inference represents a fundamental shift in computational priorities, moving away from the raw floating-point operations (FLOPs) characteristic of model training toward a regime dominated by memory bandwidth limitations. When running a model locally, whether on a high-end consumer GPU or a dedicated NPU, the primary bottleneck is almost always the speed at which data can be moved from the VRAM to the compute cores. This is because the transformer architecture, which underpins modern LLMs, is fundamentally memory-bound during the generation phase. During the prefill stage, where the prompt is processed, the computation is typically compute-bound because the model processes all input tokens simultaneously, utilizing high levels of parallelism. However, once the model transitions into the autoregressive decoding phase, where it generates tokens one by one, the operational pattern changes significantly. In this phase, each single token generation requires loading the entire model weight set from the memory into the processing units, yet the actual arithmetic performed on those weights is relatively small compared to the volume of data transferred. This creates a massive discrepancy between the theoretical peak TFLOPS of a GPU and its practical performance, often referred to as the arithmetic intensity. If a model has 70 billion parameters and is running in 16-bit precision, the system must move 140 GB of data just to produce one token, and if the bandwidth of the hardware is only 500 GB/s, the absolute theoretical maximum speed is limited by this transfer rate regardless of how many teraflops the chip possesses. This memory bandwidth bottleneck becomes even more complex when we introduce the concept of the Key-Value (KV) cache. In a standard transformer decoder, the attention mechanism requires access to the keys and values of all previous tokens in the sequence to compute the probability of the next token. If we were to recompute these values every time a new token was generated, the computational cost would grow quadratically, making inference prohibitively slow. To solve this, we store the computed keys and values in a dedicated buffer in memory known as the KV cache. While this technique drastically reduces the computational overhead, it introduces a significant memory capacity problem. As the sequence length increases, the KV cache grows linearly, consuming more and more VRAM. In a local deployment, VRAM is a scarce resource, and a large KV cache competes directly with the model weights for the limited memory available on the GPU. If the cache becomes too large, the system must either offload parts of it to slower system RAM (which devastates performance) or utilize techniques like PagedAttention to manage memory more efficiently, similar to how an operating system manages virtual memory via paging. PagedAttention allows for non-contiguous memory allocation, which reduces fragmentation and allows for more efficient use of VRAM, but the underlying tension between memory capacity and sequence length remains. This brings us to the critical trade-off between batch size and latency. In a single-user scenario, latency is the primary metric of interest, specifically the time-to-first-token (TTFT) and the time-per-output-token (TPOT). To maximize throughput in a multi-user environment, one would ideally use large batch sizes to amortize the cost of loading model weights over many concurrent requests. When you increase the batch size, you are essentially performing more work for every byte of weights loaded, which improves the arithmetic intensity and makes better use of the GPU's compute capabilities. However, increasing the batch size increases the memory footprint of the KV cache linearly, and it also increases the latency for each individual user. This creates a multi-objective optimization problem: how much can you batch to maximize throughput without causing the latency to exceed a perceptible threshold for the user? One way to mitigate this is through continuous batching, or iteration-level scheduling, where instead of waiting for a whole batch to finish before starting a new one, the engine inserts new requests into the batch at different points in their lifecycle. This ensures that the compute units remain saturated even when some sequences are near completion. However, even with sophisticated scheduling, the fundamental constraint remains the memory bandwidth. Even if we use quantization techniques like 4-bit or 8-bit integer quantization to shrink the model weights and the KV cache, we are essentially just shifting the goalposts. Quantization increases the effective bandwidth by allowing more \"information\" to be moved per byte, effectively increasing the number of tokens per second, but it introduces a trade-off with the model's perplexity and reasoning capabilities. The interaction between these variables—bandwidth, cache size, batching, and quantization—defines the frontier of local LLM performance. If you are running a model on a Mac with unified memory, the bandwidth is high but the capacity is shared with the CPU and the OS, meaning large-scale batching is more difficult than on a discrete A100. If you are on a high-end RTX 4090, you have massive bandwidth but limited capacity, forcing a choice between large context windows or larger batch sizes. Ultimately, the local inference experience is a delicate balancing act of managing these hardware constraints to minimize the time it takes for a token to appear while ensuring the context window remains long enough to be useful.","role":"assistant","tool_calls":[]}}],"created":1789425953,"model":"gemma4-26-heretic","object":"chat.completion","usage":{"prompt_tokens":68,"completion_tokens":1069,"total_tokens":1137}} \ No newline at end of file diff --git a/docs/gemma4/evidence/benchmark/models-response.json b/docs/gemma4/evidence/benchmark/models-response.json new file mode 100644 index 0000000000..af73c19040 --- /dev/null +++ b/docs/gemma4/evidence/benchmark/models-response.json @@ -0,0 +1 @@ +{"data":[{"id":"gemma4-26-heretic","object":"model","created":1789425886,"owned_by":"OVMS"}],"object":"list"} diff --git a/docs/gemma4/evidence/benchmark/requests.ndjson b/docs/gemma4/evidence/benchmark/requests.ndjson new file mode 100644 index 0000000000..d77ebf670f --- /dev/null +++ b/docs/gemma4/evidence/benchmark/requests.ndjson @@ -0,0 +1,14 @@ +{"timestamp_utc":"2026-09-14T22:44:50.4495236Z","case":"warmup","transport":"curl","endpoint":"http://127.0.0.1:18091/v3/chat/completions","requested_model":"gemma4-26-heretic","response_model":"gemma4-26-heretic","max_tokens":16,"seed":1,"do_sample":true,"temperature":1.0,"top_k":64,"top_p":0.95,"http_status":200,"elapsed_s":4.165,"prompt_tokens":21,"completion_tokens":5,"total_tokens":26,"tokens_per_s":1.201,"finish_reason":"stop","request_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\warmup-curl-request.json","response_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\warmup-curl-response.json"} +{"timestamp_utc":"2026-09-14T22:44:51.0761088Z","case":"warmup","transport":"InvokeWebRequest","endpoint":"http://127.0.0.1:18091/v3/chat/completions","requested_model":"gemma4-26-heretic","response_model":"gemma4-26-heretic","max_tokens":16,"seed":1,"do_sample":true,"temperature":1.0,"top_k":64,"top_p":0.95,"http_status":200,"elapsed_s":0.588,"prompt_tokens":21,"completion_tokens":5,"total_tokens":26,"tokens_per_s":8.506,"finish_reason":"stop","request_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\warmup-InvokeWebRequest-request.json","response_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\warmup-InvokeWebRequest-response.json"} +{"timestamp_utc":"2026-09-14T22:45:21.2809181Z","case":"long-1024","transport":"curl","endpoint":"http://127.0.0.1:18091/v3/chat/completions","requested_model":"gemma4-26-heretic","response_model":"gemma4-26-heretic","max_tokens":1024,"seed":8024,"do_sample":true,"temperature":1.0,"top_k":64,"top_p":0.95,"http_status":200,"elapsed_s":30.188,"prompt_tokens":68,"completion_tokens":728,"total_tokens":796,"tokens_per_s":24.115,"finish_reason":"stop","request_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\long-1024-curl-request.json","response_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\long-1024-curl-response.json"} +{"timestamp_utc":"2026-09-14T22:45:53.1548996Z","case":"long-1024","transport":"InvokeWebRequest","endpoint":"http://127.0.0.1:18091/v3/chat/completions","requested_model":"gemma4-26-heretic","response_model":"gemma4-26-heretic","max_tokens":1024,"seed":8024,"do_sample":true,"temperature":1.0,"top_k":64,"top_p":0.95,"http_status":200,"elapsed_s":31.87,"prompt_tokens":68,"completion_tokens":789,"total_tokens":857,"tokens_per_s":24.757,"finish_reason":"stop","request_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\long-1024-InvokeWebRequest-request.json","response_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\long-1024-InvokeWebRequest-response.json"} +{"timestamp_utc":"2026-09-14T22:46:35.9477569Z","case":"long-2048","transport":"curl","endpoint":"http://127.0.0.1:18091/v3/chat/completions","requested_model":"gemma4-26-heretic","response_model":"gemma4-26-heretic","max_tokens":2048,"seed":9048,"do_sample":true,"temperature":1.0,"top_k":64,"top_p":0.95,"http_status":200,"elapsed_s":42.791,"prompt_tokens":68,"completion_tokens":1069,"total_tokens":1137,"tokens_per_s":24.982,"finish_reason":"stop","request_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\long-2048-curl-request.json","response_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\long-2048-curl-response.json"} +{"timestamp_utc":"2026-09-14T22:47:30.7191435Z","case":"long-2048","transport":"InvokeWebRequest","endpoint":"http://127.0.0.1:18091/v3/chat/completions","requested_model":"gemma4-26-heretic","response_model":"gemma4-26-heretic","max_tokens":2048,"seed":9048,"do_sample":true,"temperature":1.0,"top_k":64,"top_p":0.95,"http_status":200,"elapsed_s":54.769,"prompt_tokens":68,"completion_tokens":1397,"total_tokens":1465,"tokens_per_s":25.507,"finish_reason":"stop","request_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\long-2048-InvokeWebRequest-request.json","response_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\long-2048-InvokeWebRequest-response.json"} +{"timestamp_utc":"2026-09-14T22:47:35.9176438Z","case":"short-128-101","transport":"curl","endpoint":"http://127.0.0.1:18091/v3/chat/completions","requested_model":"gemma4-26-heretic","response_model":"gemma4-26-heretic","max_tokens":128,"seed":101,"do_sample":true,"temperature":1.0,"top_k":64,"top_p":0.95,"http_status":200,"elapsed_s":5.196,"prompt_tokens":43,"completion_tokens":128,"total_tokens":171,"tokens_per_s":24.634,"finish_reason":"length","request_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\short-128-101-curl-request.json","response_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\short-128-101-curl-response.json"} +{"timestamp_utc":"2026-09-14T22:47:40.8721180Z","case":"short-128-101","transport":"InvokeWebRequest","endpoint":"http://127.0.0.1:18091/v3/chat/completions","requested_model":"gemma4-26-heretic","response_model":"gemma4-26-heretic","max_tokens":128,"seed":101,"do_sample":true,"temperature":1.0,"top_k":64,"top_p":0.95,"http_status":200,"elapsed_s":4.952,"prompt_tokens":43,"completion_tokens":128,"total_tokens":171,"tokens_per_s":25.851,"finish_reason":"length","request_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\short-128-101-InvokeWebRequest-request.json","response_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\short-128-101-InvokeWebRequest-response.json"} +{"timestamp_utc":"2026-09-14T22:47:45.8146549Z","case":"short-128-102","transport":"curl","endpoint":"http://127.0.0.1:18091/v3/chat/completions","requested_model":"gemma4-26-heretic","response_model":"gemma4-26-heretic","max_tokens":128,"seed":102,"do_sample":true,"temperature":1.0,"top_k":64,"top_p":0.95,"http_status":200,"elapsed_s":4.94,"prompt_tokens":43,"completion_tokens":128,"total_tokens":171,"tokens_per_s":25.91,"finish_reason":"length","request_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\short-128-102-curl-request.json","response_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\short-128-102-curl-response.json"} +{"timestamp_utc":"2026-09-14T22:47:50.7255474Z","case":"short-128-102","transport":"InvokeWebRequest","endpoint":"http://127.0.0.1:18091/v3/chat/completions","requested_model":"gemma4-26-heretic","response_model":"gemma4-26-heretic","max_tokens":128,"seed":102,"do_sample":true,"temperature":1.0,"top_k":64,"top_p":0.95,"http_status":200,"elapsed_s":4.9,"prompt_tokens":43,"completion_tokens":128,"total_tokens":171,"tokens_per_s":26.12,"finish_reason":"length","request_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\short-128-102-InvokeWebRequest-request.json","response_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\short-128-102-InvokeWebRequest-response.json"} +{"timestamp_utc":"2026-09-14T22:47:55.6742442Z","case":"short-128-103","transport":"curl","endpoint":"http://127.0.0.1:18091/v3/chat/completions","requested_model":"gemma4-26-heretic","response_model":"gemma4-26-heretic","max_tokens":128,"seed":103,"do_sample":true,"temperature":1.0,"top_k":64,"top_p":0.95,"http_status":200,"elapsed_s":4.945,"prompt_tokens":43,"completion_tokens":128,"total_tokens":171,"tokens_per_s":25.884,"finish_reason":"length","request_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\short-128-103-curl-request.json","response_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\short-128-103-curl-response.json"} +{"timestamp_utc":"2026-09-14T22:48:00.8437842Z","case":"short-128-103","transport":"InvokeWebRequest","endpoint":"http://127.0.0.1:18091/v3/chat/completions","requested_model":"gemma4-26-heretic","response_model":"gemma4-26-heretic","max_tokens":128,"seed":103,"do_sample":true,"temperature":1.0,"top_k":64,"top_p":0.95,"http_status":200,"elapsed_s":5.166,"prompt_tokens":43,"completion_tokens":128,"total_tokens":171,"tokens_per_s":24.778,"finish_reason":"length","request_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\short-128-103-InvokeWebRequest-request.json","response_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\short-128-103-InvokeWebRequest-response.json"} +{"timestamp_utc":"2026-09-14T22:48:20.7626798Z","case":"gpu-512","transport":"curl","endpoint":"http://127.0.0.1:18091/v3/chat/completions","requested_model":"gemma4-26-heretic","response_model":"gemma4-26-heretic","max_tokens":512,"seed":777,"do_sample":true,"temperature":1.0,"top_k":64,"top_p":0.95,"http_status":200,"elapsed_s":19.917,"prompt_tokens":47,"completion_tokens":512,"total_tokens":559,"tokens_per_s":25.707,"finish_reason":"length","request_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\gpu-512-curl-request.json","response_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\gpu-512-curl-response.json"} +{"timestamp_utc":"2026-09-14T22:48:40.5059187Z","case":"gpu-512","transport":"InvokeWebRequest","endpoint":"http://127.0.0.1:18091/v3/chat/completions","requested_model":"gemma4-26-heretic","response_model":"gemma4-26-heretic","max_tokens":512,"seed":777,"do_sample":true,"temperature":1.0,"top_k":64,"top_p":0.95,"http_status":200,"elapsed_s":19.741,"prompt_tokens":47,"completion_tokens":512,"total_tokens":559,"tokens_per_s":25.935,"finish_reason":"length","request_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\gpu-512-InvokeWebRequest-request.json","response_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\gpu-512-InvokeWebRequest-response.json"} diff --git a/docs/gemma4/evidence/benchmark/run.json b/docs/gemma4/evidence/benchmark/run.json new file mode 100644 index 0000000000..021c26e77c --- /dev/null +++ b/docs/gemma4/evidence/benchmark/run.json @@ -0,0 +1,15 @@ +{ + "timestamp_utc": "20260914T224446Z", + "endpoint": "http://127.0.0.1:18091/v3", + "requested_model": "gemma4-26-heretic", + "published_models": [ + "gemma4-26-heretic" + ], + "sampling": { + "do_sample": true, + "temperature": 1.0, + "top_k": 64, + "top_p": 0.95 + }, + "throughput": "completion_tokens / client wall time seconds" +} diff --git a/docs/gemma4/evidence/benchmark/short-128-101-InvokeWebRequest-request.json b/docs/gemma4/evidence/benchmark/short-128-101-InvokeWebRequest-request.json new file mode 100644 index 0000000000..bdf1230662 --- /dev/null +++ b/docs/gemma4/evidence/benchmark/short-128-101-InvokeWebRequest-request.json @@ -0,0 +1 @@ +{"model":"gemma4-26-heretic","messages":[{"role":"user","content":"Generate a continuous technical explanation of transformer inference performance. Do not use tools, headings, lists, or an early conclusion. Continue until the token limit."}],"max_tokens":128,"temperature":1.0,"top_k":64,"top_p":0.95,"seed":101} diff --git a/docs/gemma4/evidence/benchmark/short-128-101-InvokeWebRequest-response.json b/docs/gemma4/evidence/benchmark/short-128-101-InvokeWebRequest-response.json new file mode 100644 index 0000000000..de66c3a86c --- /dev/null +++ b/docs/gemma4/evidence/benchmark/short-128-101-InvokeWebRequest-response.json @@ -0,0 +1 @@ +{"choices":[{"finish_reason":"length","index":0,"logprobs":null,"message":{"content":"The computational dynamics of transformer inference are fundamentally governed by the interplay between memory bandwidth and compute throughput, often characterized by the distinction between memory-bound and compute-bound regimes. During the initial phase of generating a sequence, known as the prefill stage, the model processes the entire input prompt simultaneously, allowing for high utilization of matrix multiplication kernels and floating-point operations per second because the operation is heavily compute-bound. This phase leverages massive parallelism to compute the causal attention mechanisms and feed-forward transformations for the entire input context at once. However, as the model transitions into the autoregressive decoding phase, the performance bottleneck shifts dramatically. In this","role":"assistant","tool_calls":[]}}],"created":1789426055,"model":"gemma4-26-heretic","object":"chat.completion","usage":{"prompt_tokens":43,"completion_tokens":128,"total_tokens":171}} diff --git a/docs/gemma4/evidence/benchmark/short-128-101-curl-request.json b/docs/gemma4/evidence/benchmark/short-128-101-curl-request.json new file mode 100644 index 0000000000..bdf1230662 --- /dev/null +++ b/docs/gemma4/evidence/benchmark/short-128-101-curl-request.json @@ -0,0 +1 @@ +{"model":"gemma4-26-heretic","messages":[{"role":"user","content":"Generate a continuous technical explanation of transformer inference performance. Do not use tools, headings, lists, or an early conclusion. Continue until the token limit."}],"max_tokens":128,"temperature":1.0,"top_k":64,"top_p":0.95,"seed":101} diff --git a/docs/gemma4/evidence/benchmark/short-128-101-curl-response.json b/docs/gemma4/evidence/benchmark/short-128-101-curl-response.json new file mode 100644 index 0000000000..bd37c6a848 --- /dev/null +++ b/docs/gemma4/evidence/benchmark/short-128-101-curl-response.json @@ -0,0 +1 @@ +{"choices":[{"finish_reason":"length","index":0,"logprobs":null,"message":{"content":"The performance of transformer inference is fundamentally governed by the computational complexity of the self-attention mechanism and the memory bandwidth constraints imposed by the auto-regressive nature of sequence generation. In the forward pass, the primary bottleneck shifts depending on the stage of generation, specifically transitioning from a compute-bound regime during the initial prefill phase to a memory-bound regime during the incremental decoding phase. During the prefill phase, the entire input sequence is processed in parallel to compute the initial Key-Value (KV) cache. This stage is characterized by high floating-point operations per second (FLOPS) requirements because the quadratic complexity of the attention mechanism","role":"assistant","tool_calls":[]}}],"created":1789426050,"model":"gemma4-26-heretic","object":"chat.completion","usage":{"prompt_tokens":43,"completion_tokens":128,"total_tokens":171}} \ No newline at end of file diff --git a/docs/gemma4/evidence/benchmark/short-128-102-InvokeWebRequest-request.json b/docs/gemma4/evidence/benchmark/short-128-102-InvokeWebRequest-request.json new file mode 100644 index 0000000000..a007cb23b2 --- /dev/null +++ b/docs/gemma4/evidence/benchmark/short-128-102-InvokeWebRequest-request.json @@ -0,0 +1 @@ +{"model":"gemma4-26-heretic","messages":[{"role":"user","content":"Generate a continuous technical explanation of transformer inference performance. Do not use tools, headings, lists, or an early conclusion. Continue until the token limit."}],"max_tokens":128,"temperature":1.0,"top_k":64,"top_p":0.95,"seed":102} diff --git a/docs/gemma4/evidence/benchmark/short-128-102-InvokeWebRequest-response.json b/docs/gemma4/evidence/benchmark/short-128-102-InvokeWebRequest-response.json new file mode 100644 index 0000000000..a482bb7f22 --- /dev/null +++ b/docs/gemma4/evidence/benchmark/short-128-102-InvokeWebRequest-response.json @@ -0,0 +1 @@ +{"choices":[{"finish_reason":"length","index":0,"logprobs":null,"message":{"content":"The performance of transformer inference is fundamentally governed by the interplay between computational complexity, memory bandwidth constraints, and the transition from parallelizable training dynamics to sequential autoregressive generation. During the training phase, transformers benefit from massive parallelism because the entire sequence of tokens is known, allowing for the calculation of all attention scores simultaneously via optimized matrix multiplications. However, during inference, the model operates autoregressively, generating one token at a time. This transition fundamentally shifts the bottleneck from compute-bound operations to memory-bound operations. In the context of large language models, the primary performance bottleneck is often the memory bandwidth of the hardware, specifically how fast the weights","role":"assistant","tool_calls":[]}}],"created":1789426065,"model":"gemma4-26-heretic","object":"chat.completion","usage":{"prompt_tokens":43,"completion_tokens":128,"total_tokens":171}} diff --git a/docs/gemma4/evidence/benchmark/short-128-102-curl-request.json b/docs/gemma4/evidence/benchmark/short-128-102-curl-request.json new file mode 100644 index 0000000000..a007cb23b2 --- /dev/null +++ b/docs/gemma4/evidence/benchmark/short-128-102-curl-request.json @@ -0,0 +1 @@ +{"model":"gemma4-26-heretic","messages":[{"role":"user","content":"Generate a continuous technical explanation of transformer inference performance. Do not use tools, headings, lists, or an early conclusion. Continue until the token limit."}],"max_tokens":128,"temperature":1.0,"top_k":64,"top_p":0.95,"seed":102} diff --git a/docs/gemma4/evidence/benchmark/short-128-102-curl-response.json b/docs/gemma4/evidence/benchmark/short-128-102-curl-response.json new file mode 100644 index 0000000000..7f1c795abe --- /dev/null +++ b/docs/gemma4/evidence/benchmark/short-128-102-curl-response.json @@ -0,0 +1 @@ +{"choices":[{"finish_reason":"length","index":0,"logprobs":null,"message":{"content":"The performance of transformer inference is fundamentally governed by the interplay between computational complexity, memory bandwidth constraints, and the transition from parallelizable training dynamics to sequential autoregressive generation. During the training phase, transformers benefit from massive parallelism because the entire sequence of tokens is known, allowing for the calculation of all attention scores simultaneously via optimized matrix multiplications. However, during inference, the model operates autoregressively, generating one token at a time. This transition fundamentally shifts the bottleneck from compute-bound operations to memory-bound operations. In the context of large language models, the primary performance bottleneck is often the memory bandwidth of the hardware, specifically how fast the weights","role":"assistant","tool_calls":[]}}],"created":1789426060,"model":"gemma4-26-heretic","object":"chat.completion","usage":{"prompt_tokens":43,"completion_tokens":128,"total_tokens":171}} \ No newline at end of file diff --git a/docs/gemma4/evidence/benchmark/short-128-103-InvokeWebRequest-request.json b/docs/gemma4/evidence/benchmark/short-128-103-InvokeWebRequest-request.json new file mode 100644 index 0000000000..b4a44e8356 --- /dev/null +++ b/docs/gemma4/evidence/benchmark/short-128-103-InvokeWebRequest-request.json @@ -0,0 +1 @@ +{"model":"gemma4-26-heretic","messages":[{"role":"user","content":"Generate a continuous technical explanation of transformer inference performance. Do not use tools, headings, lists, or an early conclusion. Continue until the token limit."}],"max_tokens":128,"temperature":1.0,"top_k":64,"top_p":0.95,"seed":103} diff --git a/docs/gemma4/evidence/benchmark/short-128-103-InvokeWebRequest-response.json b/docs/gemma4/evidence/benchmark/short-128-103-InvokeWebRequest-response.json new file mode 100644 index 0000000000..d1d2703bf4 --- /dev/null +++ b/docs/gemma4/evidence/benchmark/short-128-103-InvokeWebRequest-response.json @@ -0,0 +1 @@ +{"choices":[{"finish_reason":"length","index":0,"logprobs":null,"message":{"content":"The performance of transformer inference is governed by a complex interplay between memory bandwidth, compute intensity, and the structural mechanics of the attention mechanism. At its core, the inference process is a sequence of auto-regressive steps where each subsequent token is predicted based on the preceding context, a process that fundamentally changes the computational profile as the sequence length grows. During the prefill phase, which occurs when the entire input prompt is processed, the transformer operates in a compute-bound regime. Here, the GPU or specialized hardware utilizes massive parallelization to compute the hidden states for all input tokens simultaneously through high-throughput matrix multiplications. This phase is characterized by","role":"assistant","tool_calls":[]}}],"created":1789426075,"model":"gemma4-26-heretic","object":"chat.completion","usage":{"prompt_tokens":43,"completion_tokens":128,"total_tokens":171}} diff --git a/docs/gemma4/evidence/benchmark/short-128-103-curl-request.json b/docs/gemma4/evidence/benchmark/short-128-103-curl-request.json new file mode 100644 index 0000000000..b4a44e8356 --- /dev/null +++ b/docs/gemma4/evidence/benchmark/short-128-103-curl-request.json @@ -0,0 +1 @@ +{"model":"gemma4-26-heretic","messages":[{"role":"user","content":"Generate a continuous technical explanation of transformer inference performance. Do not use tools, headings, lists, or an early conclusion. Continue until the token limit."}],"max_tokens":128,"temperature":1.0,"top_k":64,"top_p":0.95,"seed":103} diff --git a/docs/gemma4/evidence/benchmark/short-128-103-curl-response.json b/docs/gemma4/evidence/benchmark/short-128-103-curl-response.json new file mode 100644 index 0000000000..fe4dc46f91 --- /dev/null +++ b/docs/gemma4/evidence/benchmark/short-128-103-curl-response.json @@ -0,0 +1 @@ +{"choices":[{"finish_reason":"length","index":0,"logprobs":null,"message":{"content":"The performance of transformer inference is governed by a complex interplay between memory bandwidth, compute intensity, and the structural mechanics of the attention mechanism. At its core, the inference process is a sequence of auto-regressive steps where each subsequent token is predicted based on the preceding context, a process that fundamentally changes the computational profile as the sequence length grows. During the prefill phase, which occurs when the entire input prompt is processed, the transformer operates in a compute-bound regime. Here, the GPU or specialized hardware utilizes massive parallelization to compute the hidden states for all input tokens simultaneously through high-throughput matrix multiplications. This phase is characterized by","role":"assistant","tool_calls":[]}}],"created":1789426070,"model":"gemma4-26-heretic","object":"chat.completion","usage":{"prompt_tokens":43,"completion_tokens":128,"total_tokens":171}} \ No newline at end of file diff --git a/docs/gemma4/evidence/benchmark/summary.csv b/docs/gemma4/evidence/benchmark/summary.csv new file mode 100644 index 0000000000..a6b8c0d6a1 --- /dev/null +++ b/docs/gemma4/evidence/benchmark/summary.csv @@ -0,0 +1,15 @@ +"timestamp_utc","case","transport","endpoint","requested_model","response_model","max_tokens","seed","do_sample","temperature","top_k","top_p","http_status","elapsed_s","prompt_tokens","completion_tokens","total_tokens","tokens_per_s","finish_reason","request_path","response_path" +"2026-09-14T22:44:50.4495236Z","warmup","curl","http://127.0.0.1:18091/v3/chat/completions","gemma4-26-heretic","gemma4-26-heretic","16","1","True","1","64","0.95","200","4.165","21","5","26","1.201","stop","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\warmup-curl-request.json","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\warmup-curl-response.json" +"2026-09-14T22:44:51.0761088Z","warmup","InvokeWebRequest","http://127.0.0.1:18091/v3/chat/completions","gemma4-26-heretic","gemma4-26-heretic","16","1","True","1","64","0.95","200","0.588","21","5","26","8.506","stop","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\warmup-InvokeWebRequest-request.json","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\warmup-InvokeWebRequest-response.json" +"2026-09-14T22:45:21.2809181Z","long-1024","curl","http://127.0.0.1:18091/v3/chat/completions","gemma4-26-heretic","gemma4-26-heretic","1024","8024","True","1","64","0.95","200","30.188","68","728","796","24.115","stop","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\long-1024-curl-request.json","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\long-1024-curl-response.json" +"2026-09-14T22:45:53.1548996Z","long-1024","InvokeWebRequest","http://127.0.0.1:18091/v3/chat/completions","gemma4-26-heretic","gemma4-26-heretic","1024","8024","True","1","64","0.95","200","31.87","68","789","857","24.757","stop","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\long-1024-InvokeWebRequest-request.json","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\long-1024-InvokeWebRequest-response.json" +"2026-09-14T22:46:35.9477569Z","long-2048","curl","http://127.0.0.1:18091/v3/chat/completions","gemma4-26-heretic","gemma4-26-heretic","2048","9048","True","1","64","0.95","200","42.791","68","1069","1137","24.982","stop","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\long-2048-curl-request.json","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\long-2048-curl-response.json" +"2026-09-14T22:47:30.7191435Z","long-2048","InvokeWebRequest","http://127.0.0.1:18091/v3/chat/completions","gemma4-26-heretic","gemma4-26-heretic","2048","9048","True","1","64","0.95","200","54.769","68","1397","1465","25.507","stop","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\long-2048-InvokeWebRequest-request.json","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\long-2048-InvokeWebRequest-response.json" +"2026-09-14T22:47:35.9176438Z","short-128-101","curl","http://127.0.0.1:18091/v3/chat/completions","gemma4-26-heretic","gemma4-26-heretic","128","101","True","1","64","0.95","200","5.196","43","128","171","24.634","length","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\short-128-101-curl-request.json","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\short-128-101-curl-response.json" +"2026-09-14T22:47:40.8721180Z","short-128-101","InvokeWebRequest","http://127.0.0.1:18091/v3/chat/completions","gemma4-26-heretic","gemma4-26-heretic","128","101","True","1","64","0.95","200","4.952","43","128","171","25.851","length","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\short-128-101-InvokeWebRequest-request.json","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\short-128-101-InvokeWebRequest-response.json" +"2026-09-14T22:47:45.8146549Z","short-128-102","curl","http://127.0.0.1:18091/v3/chat/completions","gemma4-26-heretic","gemma4-26-heretic","128","102","True","1","64","0.95","200","4.94","43","128","171","25.91","length","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\short-128-102-curl-request.json","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\short-128-102-curl-response.json" +"2026-09-14T22:47:50.7255474Z","short-128-102","InvokeWebRequest","http://127.0.0.1:18091/v3/chat/completions","gemma4-26-heretic","gemma4-26-heretic","128","102","True","1","64","0.95","200","4.9","43","128","171","26.12","length","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\short-128-102-InvokeWebRequest-request.json","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\short-128-102-InvokeWebRequest-response.json" +"2026-09-14T22:47:55.6742442Z","short-128-103","curl","http://127.0.0.1:18091/v3/chat/completions","gemma4-26-heretic","gemma4-26-heretic","128","103","True","1","64","0.95","200","4.945","43","128","171","25.884","length","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\short-128-103-curl-request.json","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\short-128-103-curl-response.json" +"2026-09-14T22:48:00.8437842Z","short-128-103","InvokeWebRequest","http://127.0.0.1:18091/v3/chat/completions","gemma4-26-heretic","gemma4-26-heretic","128","103","True","1","64","0.95","200","5.166","43","128","171","24.778","length","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\short-128-103-InvokeWebRequest-request.json","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\short-128-103-InvokeWebRequest-response.json" +"2026-09-14T22:48:20.7626798Z","gpu-512","curl","http://127.0.0.1:18091/v3/chat/completions","gemma4-26-heretic","gemma4-26-heretic","512","777","True","1","64","0.95","200","19.917","47","512","559","25.707","length","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\gpu-512-curl-request.json","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\gpu-512-curl-response.json" +"2026-09-14T22:48:40.5059187Z","gpu-512","InvokeWebRequest","http://127.0.0.1:18091/v3/chat/completions","gemma4-26-heretic","gemma4-26-heretic","512","777","True","1","64","0.95","200","19.741","47","512","559","25.935","length","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\gpu-512-InvokeWebRequest-request.json","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\gpu-512-InvokeWebRequest-response.json" diff --git a/docs/gemma4/evidence/benchmark/warmup-InvokeWebRequest-request.json b/docs/gemma4/evidence/benchmark/warmup-InvokeWebRequest-request.json new file mode 100644 index 0000000000..6ac1836800 --- /dev/null +++ b/docs/gemma4/evidence/benchmark/warmup-InvokeWebRequest-request.json @@ -0,0 +1 @@ +{"model":"gemma4-26-heretic","messages":[{"role":"user","content":"Reply with a short sentence confirming readiness."}],"max_tokens":16,"temperature":1.0,"top_k":64,"top_p":0.95,"seed":1} diff --git a/docs/gemma4/evidence/benchmark/warmup-InvokeWebRequest-response.json b/docs/gemma4/evidence/benchmark/warmup-InvokeWebRequest-response.json new file mode 100644 index 0000000000..a051007693 --- /dev/null +++ b/docs/gemma4/evidence/benchmark/warmup-InvokeWebRequest-response.json @@ -0,0 +1 @@ +{"choices":[{"finish_reason":"stop","index":0,"logprobs":null,"message":{"content":"I am ready.","role":"assistant","tool_calls":[]}}],"created":1789425890,"model":"gemma4-26-heretic","object":"chat.completion","usage":{"prompt_tokens":21,"completion_tokens":5,"total_tokens":26}} diff --git a/docs/gemma4/evidence/benchmark/warmup-curl-request.json b/docs/gemma4/evidence/benchmark/warmup-curl-request.json new file mode 100644 index 0000000000..6ac1836800 --- /dev/null +++ b/docs/gemma4/evidence/benchmark/warmup-curl-request.json @@ -0,0 +1 @@ +{"model":"gemma4-26-heretic","messages":[{"role":"user","content":"Reply with a short sentence confirming readiness."}],"max_tokens":16,"temperature":1.0,"top_k":64,"top_p":0.95,"seed":1} diff --git a/docs/gemma4/evidence/benchmark/warmup-curl-response.json b/docs/gemma4/evidence/benchmark/warmup-curl-response.json new file mode 100644 index 0000000000..b2128275ce --- /dev/null +++ b/docs/gemma4/evidence/benchmark/warmup-curl-response.json @@ -0,0 +1 @@ +{"choices":[{"finish_reason":"stop","index":0,"logprobs":null,"message":{"content":"I am ready.","role":"assistant","tool_calls":[]}}],"created":1789425886,"model":"gemma4-26-heretic","object":"chat.completion","usage":{"prompt_tokens":21,"completion_tokens":5,"total_tokens":26}} \ No newline at end of file diff --git a/docs/gemma4/evidence/tools/parallel-request.json b/docs/gemma4/evidence/tools/parallel-request.json new file mode 100644 index 0000000000..fa0fe5d13f --- /dev/null +++ b/docs/gemma4/evidence/tools/parallel-request.json @@ -0,0 +1 @@ +{"seed":170644,"stream":false,"max_tokens":256,"model":"gemma4-26-heretic","tools":[{"type":"function","function":{"name":"echo","description":"Echo the supplied text verbatim.","parameters":{"type":"object","properties":{"text":{"type":"string"}},"additionalProperties":false,"required":["text"]}}}],"tool_choice":{"type":"function","function":{"name":"echo"}},"temperature":0,"messages":[{"role":"user","content":"Call echo exactly twice, once with text RC17064400-first and once with text RC17064400-second. No explanation or other calls."}],"parallel_tool_calls":true} diff --git a/docs/gemma4/evidence/tools/parallel-response.txt b/docs/gemma4/evidence/tools/parallel-response.txt new file mode 100644 index 0000000000..701c024ae0 --- /dev/null +++ b/docs/gemma4/evidence/tools/parallel-response.txt @@ -0,0 +1 @@ +{"choices":[{"finish_reason":"tool_calls","index":0,"logprobs":null,"message":{"content":"","role":"assistant","tool_calls":[{"id":"UI7iFil3s","type":"function","function":{"name":"echo","arguments":"{\"text\":\"RC17064400-first\"}"}},{"id":"DbVAEmpEk","type":"function","function":{"name":"echo","arguments":"{\"text\":\"RC17064400-second\"}"}}]}}],"created":1789426160,"model":"gemma4-26-heretic","object":"chat.completion","usage":{"prompt_tokens":100,"completion_tokens":53,"total_tokens":153}} diff --git a/docs/gemma4/evidence/tools/results.json b/docs/gemma4/evidence/tools/results.json new file mode 100644 index 0000000000..22122d0b67 --- /dev/null +++ b/docs/gemma4/evidence/tools/results.json @@ -0,0 +1,36 @@ +[ + { + "case": "single", + "status": "PASS", + "http": 200, + "call_count": 1, + "finish_reason": "tool_calls", + "texts": [ + "RC17064400-single" + ], + "elapsed_s": 1.884 + }, + { + "case": "parallel", + "status": "PASS", + "http": 200, + "call_count": 2, + "finish_reason": "tool_calls", + "texts": [ + "RC17064400-first", + "RC17064400-second" + ], + "elapsed_s": 2.668 + }, + { + "case": "stream", + "status": "PASS", + "http": 200, + "call_count": 1, + "finish_reason": "tool_calls", + "texts": [ + "RC17064400-single" + ], + "elapsed_s": 1.23 + } +] diff --git a/docs/gemma4/evidence/tools/single-request.json b/docs/gemma4/evidence/tools/single-request.json new file mode 100644 index 0000000000..1ee1b2e300 --- /dev/null +++ b/docs/gemma4/evidence/tools/single-request.json @@ -0,0 +1 @@ +{"seed":170644,"stream":false,"max_tokens":256,"model":"gemma4-26-heretic","tools":[{"type":"function","function":{"name":"echo","description":"Echo the supplied text verbatim.","parameters":{"type":"object","properties":{"text":{"type":"string"}},"additionalProperties":false,"required":["text"]}}}],"tool_choice":{"type":"function","function":{"name":"echo"}},"temperature":0,"messages":[{"role":"user","content":"Call echo exactly once with text RC17064400-single. No explanation or other calls."}],"parallel_tool_calls":false} diff --git a/docs/gemma4/evidence/tools/single-response.txt b/docs/gemma4/evidence/tools/single-response.txt new file mode 100644 index 0000000000..4d17075f4a --- /dev/null +++ b/docs/gemma4/evidence/tools/single-response.txt @@ -0,0 +1 @@ +{"choices":[{"finish_reason":"tool_calls","index":0,"logprobs":null,"message":{"content":"","role":"assistant","tool_calls":[{"id":"SXNuQaxfM","type":"function","function":{"name":"echo","arguments":"{\"text\":\"RC17064400-single\"}"}}]}}],"created":1789426158,"model":"gemma4-26-heretic","object":"chat.completion","usage":{"prompt_tokens":83,"completion_tokens":26,"total_tokens":109}} diff --git a/docs/gemma4/evidence/tools/stream-request.json b/docs/gemma4/evidence/tools/stream-request.json new file mode 100644 index 0000000000..e439b81769 --- /dev/null +++ b/docs/gemma4/evidence/tools/stream-request.json @@ -0,0 +1 @@ +{"seed":170644,"stream":true,"max_tokens":256,"model":"gemma4-26-heretic","tools":[{"type":"function","function":{"name":"echo","description":"Echo the supplied text verbatim.","parameters":{"type":"object","properties":{"text":{"type":"string"}},"additionalProperties":false,"required":["text"]}}}],"tool_choice":{"type":"function","function":{"name":"echo"}},"temperature":0,"messages":[{"role":"user","content":"Call echo exactly once with text RC17064400-single. No explanation or other calls."}],"parallel_tool_calls":false} diff --git a/docs/gemma4/evidence/tools/stream-response.txt b/docs/gemma4/evidence/tools/stream-response.txt new file mode 100644 index 0000000000..357f25cbf1 --- /dev/null +++ b/docs/gemma4/evidence/tools/stream-response.txt @@ -0,0 +1,9 @@ +data: {"choices":[{"index":0,"delta":{"role":"assistant","content":null},"finish_reason":null}],"created":1789426163,"model":"gemma4-26-heretic","object":"chat.completion.chunk"} + +data: {"choices":[{"index":0,"logprobs":null,"delta":{"tool_calls":[{"id":"ylBXbMytA","type":"function","index":0,"function":{"name":"echo","arguments":"{\"text\":\"RC17064400-single\"}"}}]},"finish_reason":null}],"created":1789426163,"model":"gemma4-26-heretic","object":"chat.completion.chunk"} + +data: {"choices":[{"index":0,"logprobs":null,"delta":{},"finish_reason":"tool_calls"}],"created":1789426163,"model":"gemma4-26-heretic","object":"chat.completion.chunk"} + +data: [DONE] + + diff --git a/docs/gemma4/evidence/transfer-source-inventory.json b/docs/gemma4/evidence/transfer-source-inventory.json new file mode 100644 index 0000000000..31f4ea4aec --- /dev/null +++ b/docs/gemma4/evidence/transfer-source-inventory.json @@ -0,0 +1,182 @@ +[ + { + "path": "src/llm/apis/openai_api_handler.hpp", + "blob": "253927749af097478f8a1a067af6d48ce61993b7", + "source": "170644006a5334cb971b05824e4a8c95b495c4e2" + }, + { + "path": "src/llm/apis/openai_completions.hpp", + "blob": "f2493f32b3ba72095ff6cdb94faa28caf916d439", + "source": "170644006a5334cb971b05824e4a8c95b495c4e2" + }, + { + "path": "src/llm/apis/openai_request.hpp", + "blob": "2e829ed413f56b25876b5dcef9203b4ddcdcbcb3", + "source": "170644006a5334cb971b05824e4a8c95b495c4e2" + }, + { + "path": "src/llm/apis/openai_responses.cpp", + "blob": "f65a34d1fc5f562fe50e736ba8620b5d9d79e15f", + "source": "170644006a5334cb971b05824e4a8c95b495c4e2" + }, + { + "path": "src/llm/apis/openai_responses.hpp", + "blob": "8c36b2267bdf5028cdcc8c2a6c811db0354ee4cb", + "source": "170644006a5334cb971b05824e4a8c95b495c4e2" + }, + { + "path": "src/llm/io_processing/base_generation_config_builder.hpp", + "blob": "0e4c73cb98a76aaf69df0f34e7dff5c974601769", + "source": "170644006a5334cb971b05824e4a8c95b495c4e2" + }, + { + "path": "src/llm/io_processing/base_output_parser.hpp", + "blob": "1a7387e9fca60e1cc40b178b31decfd6053b47f4", + "source": "170644006a5334cb971b05824e4a8c95b495c4e2" + }, + { + "path": "src/llm/io_processing/chat_template/analyzer.cpp", + "blob": "050ac4b9d23274d6de0919f044e9d3883bf54c22", + "source": "170644006a5334cb971b05824e4a8c95b495c4e2" + }, + { + "path": "src/llm/io_processing/chat_template/caps.hpp", + "blob": "18860f7043222444678d02e61bfedfaa4976f518", + "source": "170644006a5334cb971b05824e4a8c95b495c4e2" + }, + { + "path": "src/llm/io_processing/gemma4/gemma4_reasoning_parser.cpp", + "blob": "c7e848098ca384fd960f4d7ae0f771fc5f0e14d3", + "source": "170644006a5334cb971b05824e4a8c95b495c4e2" + }, + { + "path": "src/llm/io_processing/gemma4/gemma4_reasoning_parser.hpp", + "blob": "3a574c3a2d4cc372578fa1420b8125c2b439c222", + "source": "170644006a5334cb971b05824e4a8c95b495c4e2" + }, + { + "path": "src/llm/io_processing/gemma4/gemma4_tool_parser.cpp", + "blob": "d7d766e032b47df0bb155140e6d26a8740af5217", + "source": "170644006a5334cb971b05824e4a8c95b495c4e2" + }, + { + "path": "src/llm/io_processing/gemma4/gemma4_tool_parser.hpp", + "blob": "31d0b820465e580911399e79b4b819185b067789", + "source": "170644006a5334cb971b05824e4a8c95b495c4e2" + }, + { + "path": "src/llm/io_processing/generation_config_builder.hpp", + "blob": "d78bb9c6c2083d60e49219dab32239d1231c0c15", + "source": "170644006a5334cb971b05824e4a8c95b495c4e2" + }, + { + "path": "src/llm/io_processing/input_processors/chat_template_adapter.cpp", + "blob": "92e66f2c69ab50470e563fc73afae620ca25c6c7", + "source": "170644006a5334cb971b05824e4a8c95b495c4e2" + }, + { + "path": "src/llm/io_processing/input_processors/chat_template_adapter.hpp", + "blob": "99e4efa0d73b144d2d95b969002636150886f466", + "source": "170644006a5334cb971b05824e4a8c95b495c4e2" + }, + { + "path": "src/llm/io_processing/input_processors/chat_template_processor.cpp", + "blob": "3549be639764815fee7c2869ab18d885401401dd", + "source": "170644006a5334cb971b05824e4a8c95b495c4e2" + }, + { + "path": "src/llm/io_processing/input_processors/chat_template_processor.hpp", + "blob": "79befb72577894d713ab762cdebffe154b078321", + "source": "170644006a5334cb971b05824e4a8c95b495c4e2" + }, + { + "path": "src/llm/io_processing/output_parser.cpp", + "blob": "4a8c0a299b348d117d75f957a72c03c3e21101c5", + "source": "170644006a5334cb971b05824e4a8c95b495c4e2" + }, + { + "path": "src/llm/io_processing/output_parser.hpp", + "blob": "503ed80fdb851bd814ea58d68212847bc439c649", + "source": "170644006a5334cb971b05824e4a8c95b495c4e2" + }, + { + "path": "src/llm/io_processing/output_parsing_config.hpp", + "blob": "26dd20cf6b1ecd997969216d82fc75b3b4655bc8", + "source": "170644006a5334cb971b05824e4a8c95b495c4e2" + }, + { + "path": "src/llm/ovms_text_streamer.cpp", + "blob": "f65d486c1bdacefc197a3ac4ba65a594fd703ffd", + "source": "170644006a5334cb971b05824e4a8c95b495c4e2" + }, + { + "path": "src/llm/ovms_text_streamer.hpp", + "blob": "ab966c5e4167a60244f77a29804d28d6d33a6212", + "source": "170644006a5334cb971b05824e4a8c95b495c4e2" + }, + { + "path": "src/llm/servable.cpp", + "blob": "2e1bd37f99944a47cd44bd4ffc4b4b2fa0485e6d", + "source": "170644006a5334cb971b05824e4a8c95b495c4e2" + }, + { + "path": "src/llm/servable.hpp", + "blob": "77e52a3206f6010fc344ed1e77d1cceb57bb7053", + "source": "170644006a5334cb971b05824e4a8c95b495c4e2" + }, + { + "path": "src/test/llm/gemma4_fast/BUILD", + "blob": "576f1e8fc03b6b31602fb37a4397b9e323f7c2b0", + "source": "170644006a5334cb971b05824e4a8c95b495c4e2" + }, + { + "path": "src/test/llm/gemma4_fast/gemma4_parser_contract_test.cpp", + "blob": "47a2edad2ff71ee3b60f83f01c8c3c11797b8fa6", + "source": "170644006a5334cb971b05824e4a8c95b495c4e2" + }, + { + "path": "src/test/llm/gemma4_fast/gemma4_reasoning_semantic_refit_test.cpp", + "blob": "8a8dd7bc46ac2dd53ea0aec3d4aac1889c34989a", + "source": "170644006a5334cb971b05824e4a8c95b495c4e2" + }, + { + "path": "src/test/llm/gemma4_fast/gemma4_recovery_contract_test.cpp", + "blob": "21d2307fa590b91002e0e34ce5559afc9e435e68", + "source": "170644006a5334cb971b05824e4a8c95b495c4e2" + }, + { + "path": "src/test/llm/gemma4_overlay/BUILD", + "blob": "30f7a3f99033137ebbbd876840b9ff1e71f901f3", + "source": "170644006a5334cb971b05824e4a8c95b495c4e2" + }, + { + "path": "src/test/llm/gemma4_overlay/gemma4_chat_template_overlay_contract_test.cpp", + "blob": "00194b65667251cf8855ccd7c714d6f0d052414c", + "source": "170644006a5334cb971b05824e4a8c95b495c4e2" + }, + { + "path": "src/test/llm/gemma4_overlay/gemma4_google_jinja_contract_test.cpp", + "blob": "a5e219ace08e3aab1a24416f0c9f242735dbb7c2", + "source": "170644006a5334cb971b05824e4a8c95b495c4e2" + }, + { + "path": "src/test/llm/generation_config/BUILD", + "blob": "e46ce9c3e9fac23b3ca21bbe997dc489dbfd48ba", + "source": "170644006a5334cb971b05824e4a8c95b495c4e2" + }, + { + "path": "src/test/llm/generation_config/gemma4_generation_contract_test.cpp", + "blob": "9905e0b12ccabe580d3a9395bb6303918ada9a0a", + "source": "170644006a5334cb971b05824e4a8c95b495c4e2" + }, + { + "path": "src/test/llm/generation_config/gemma4_prompt_state_generation_contract_test.cpp", + "blob": "b754ebf7d4c4eb45d76a1c74c91a4928d9c5fe95", + "source": "170644006a5334cb971b05824e4a8c95b495c4e2" + }, + { + "path": "src/test/llm/generation_config/openai_parallel_tool_calls_contract_test.cpp", + "blob": "89a8fea4fd193b19eb793bab6db9520820d0314a", + "source": "170644006a5334cb971b05824e4a8c95b495c4e2" + } +] diff --git a/src/llm/apis/openai_api_handler.hpp b/src/llm/apis/openai_api_handler.hpp index e1fd7bc8df..253927749a 100644 --- a/src/llm/apis/openai_api_handler.hpp +++ b/src/llm/apis/openai_api_handler.hpp @@ -138,6 +138,43 @@ class OpenAIApiHandler { // Assemble a ParsedOutput from a sequence of streaming Delta variants produced by OVMSTextStreamer. static ParsedOutput parsedOutputFromDeltas(const std::vector& deltas); + // OpenAI defaults parallel_tool_calls to true. Parse it before model-specific + // generation policy consumes request.parallelToolCalls. + absl::Status parseParallelToolCallsPolicy() { + auto it = doc.FindMember("parallel_tool_calls"); + if (it == doc.MemberEnd() || it->value.IsNull()) + return absl::OkStatus(); + if (!it->value.IsBool()) + return absl::InvalidArgumentError("parallel_tool_calls is not a bool"); + request.parallelToolCalls = it->value.GetBool(); + return absl::OkStatus(); + } + + // Keep hard/named tool choices fail-closed. The generic upstream parser turns + // a request with no tools into toolChoice=none; for required/named choices that + // silently converts a constrained request into unconstrained generation. + absl::Status validateHardToolChoiceHasTools() const { + auto choiceIt = doc.FindMember("tool_choice"); + if (choiceIt == doc.MemberEnd() || choiceIt->value.IsNull()) + return absl::OkStatus(); + + bool hardChoice = false; + if (choiceIt->value.IsString()) { + hardChoice = std::string(choiceIt->value.GetString()) == "required"; + } else if (choiceIt->value.IsObject()) { + hardChoice = true; + } + if (!hardChoice) + return absl::OkStatus(); + + auto toolsIt = doc.FindMember("tools"); + if (toolsIt == doc.MemberEnd() || toolsIt->value.IsNull() || + (toolsIt->value.IsArray() && toolsIt->value.Empty())) { + return absl::InvalidArgumentError("tool_choice requires at least one tool"); + } + return absl::OkStatus(); + } + public: OpenAIApiHandler(Document& doc, Endpoint endpoint, std::chrono::time_point creationTime, ov::genai::Tokenizer tokenizer, const std::string& toolParserName = "", const std::string& reasoningParserName = "") : @@ -160,8 +197,9 @@ class OpenAIApiHandler { absl::Status parseRequest(std::optional maxTokensLimit, uint32_t bestOfLimit, std::optional maxModelLength, std::optional allowedLocalMediaPath = std::nullopt, std::optional> allowedMediaDomains = std::nullopt); - // Shared parsing (non-virtual) - absl::Status parseTools(); + // Shared parsing. Endpoint handlers override parseTools only to establish + // endpoint-wide policy before delegating to the upstream schema parser. + virtual absl::Status parseTools(); absl::StatusOr> parseToolsToJsonContainer(); absl::StatusOr> parseChatTemplateKwargsToJsonContainer(); const bool areToolsAvailable() const; diff --git a/src/llm/apis/openai_completions.hpp b/src/llm/apis/openai_completions.hpp index af69a611f8..f2493f32b3 100644 --- a/src/llm/apis/openai_completions.hpp +++ b/src/llm/apis/openai_completions.hpp @@ -36,6 +36,16 @@ class OpenAIChatCompletionsHandler : public OpenAIApiHandler { public: using OpenAIApiHandler::OpenAIApiHandler; // Inherit constructors + absl::Status parseTools() override { + auto status = parseParallelToolCallsPolicy(); + if (!status.ok()) + return status; + status = validateHardToolChoiceHasTools(); + if (!status.ok()) + return status; + return OpenAIApiHandler::parseTools(); + } + absl::Status parseRequestImpl(std::optional maxTokensLimit, uint32_t bestOfLimit, std::optional maxModelLength, std::optional allowedLocalMediaPath, std::optional> allowedMediaDomains) override; absl::Status parseMessages(std::optional allowedLocalMediaPath = std::nullopt, std::optional> allowedMediaDomains = std::nullopt); diff --git a/src/llm/apis/openai_request.hpp b/src/llm/apis/openai_request.hpp index 50c01a6f46..2e829ed413 100644 --- a/src/llm/apis/openai_request.hpp +++ b/src/llm/apis/openai_request.hpp @@ -81,6 +81,9 @@ struct OpenAIRequest { ToolsSchemas_t toolNameSchemaMap; // Holds value for tool_choice field as described in https://platform.openai.com/docs/api-reference/chat/create#chat_create-tool_choice std::string toolChoice; + // Whether the assistant may emit more than one tool call in the same turn. + // OpenAI-compatible servers default this field to true when it is omitted. + bool parallelToolCalls{true}; bool skipSpecialTokens{true}; diff --git a/src/llm/apis/openai_responses.cpp b/src/llm/apis/openai_responses.cpp index 8cc08e49cb..f65a34d1fc 100644 --- a/src/llm/apis/openai_responses.cpp +++ b/src/llm/apis/openai_responses.cpp @@ -922,7 +922,7 @@ void OpenAIResponsesHandler::serializeCommonResponseParameters(Writer maxTokensLimit, uint32_t bestOfLimit, std::optional maxModelLength, std::optional allowedLocalMediaPath, std::optional> allowedMediaDomains) override; diff --git a/src/llm/io_processing/base_generation_config_builder.hpp b/src/llm/io_processing/base_generation_config_builder.hpp index 7b2793beb7..0e4c73cb98 100644 --- a/src/llm/io_processing/base_generation_config_builder.hpp +++ b/src/llm/io_processing/base_generation_config_builder.hpp @@ -96,6 +96,13 @@ class BaseGenerationConfigBuilder { */ void unsetStructuredOutputConfig(); + /* + * Model-specific policy for structured-output validation failures. The generic + * serving path historically falls back to unguided generation; builders that + * represent a hard API contract may override this to keep the grammar fail-closed. + */ + virtual bool shouldPreserveStructuredOutputOnValidationFailure() const { return false; } + /* * Fills generation config with values read from OpenAI request. * If extended, model specific implementation should call base class method first to fill in common configuration diff --git a/src/llm/io_processing/base_output_parser.hpp b/src/llm/io_processing/base_output_parser.hpp index 163a836260..1a7387e9fc 100644 --- a/src/llm/io_processing/base_output_parser.hpp +++ b/src/llm/io_processing/base_output_parser.hpp @@ -96,6 +96,14 @@ class BaseOutputParser { } public: + struct PendingToolFrameDiagnostic { + std::string phase; + size_t bufferedBytes; + std::string toolName; + }; + virtual std::optional pendingToolFrameDiagnostic() const { + return std::nullopt; + } BaseOutputParser() = delete; explicit BaseOutputParser(ov::genai::Tokenizer& tokenizer) : tokenizer(tokenizer) {} diff --git a/src/llm/io_processing/chat_template/analyzer.cpp b/src/llm/io_processing/chat_template/analyzer.cpp index 98b740a961..050ac4b9d2 100644 --- a/src/llm/io_processing/chat_template/analyzer.cpp +++ b/src/llm/io_processing/chat_template/analyzer.cpp @@ -45,12 +45,32 @@ ChatTemplateAnalysisResult ChatTemplateAnalyzer::analyze(const std::string& temp return result; } - // Gemma4 detection + // Gemma4 detection. Compose upstream 2026.5 response-field handling with the + // Gemmamonster adapters required by the exact template in use. Current Google + // templates require tool_calls[].function.arguments to be a mapping and reject + // stringified JSON, while some compatible/older templates accept both shapes. + // Detect the contract from template syntax instead of forcing one representation + // on every Gemma4-derived template. if (contains(templateSource, "'<|tool_call>call:'") || contains(templateSource, "<|tool_call>call:")) { result.detectedToolParser = "gemma4"; result.detectedReasoningParser = "gemma4"; // gemma is always tied to its own parser for reasoning result.caps.supportsToolCalls = true; result.caps.supportsResponseFieldInToolDefinition = true; + + const bool mapsSingleQuotedArguments = contains(templateSource, "function['arguments'] is mapping"); + const bool mapsDoubleQuotedArguments = contains(templateSource, "function[\"arguments\"] is mapping"); + const bool acceptsSingleQuotedStringArguments = contains(templateSource, "function['arguments'] is string"); + const bool acceptsDoubleQuotedStringArguments = contains(templateSource, "function[\"arguments\"] is string"); + const bool mapsToolArguments = mapsSingleQuotedArguments || mapsDoubleQuotedArguments; + const bool acceptsStringToolArguments = acceptsSingleQuotedStringArguments || acceptsDoubleQuotedStringArguments; + result.caps.requiresObjectArguments = mapsToolArguments && !acceptsStringToolArguments; + + // Jinja2 treats mappings as sequences when a template iterates message.content; + // in that case part is a string key and part.get(...) fails, so response mapping + // conversion must stay disabled for content-parts templates. + const bool mapsResponse = contains(templateSource, "response is mapping"); + const bool iteratesPartsWithGet = contains(templateSource, "part.get('type')") || contains(templateSource, "part.get(\"type\")"); + result.caps.parseToolResponseJsonContent = mapsResponse && !iteratesPartsWithGet; return result; } @@ -71,7 +91,6 @@ ChatTemplateAnalysisResult ChatTemplateAnalyzer::analyze(const std::string& temp result.detectedToolParser = "minicpm5"; result.caps.supportsToolCalls = true; result.detectedReasoningParser = "minicpm5"; - return result; } diff --git a/src/llm/io_processing/chat_template/caps.hpp b/src/llm/io_processing/chat_template/caps.hpp index 287ade93cf..18860f7043 100644 --- a/src/llm/io_processing/chat_template/caps.hpp +++ b/src/llm/io_processing/chat_template/caps.hpp @@ -25,17 +25,25 @@ struct ChatTemplateCaps { // Some templates require tool_call arguments to be a dict/object rather than a stringified JSON. bool requiresObjectArguments = false; + // Some Gemma4 templates expect role:tool JSON object content as a mapping. This is + // intentionally separate from response-field support because Google-style templates + // that iterate content parts with part.get(...) must keep tool content as a string. + bool parseToolResponseJsonContent = false; + std::string missnamedReasoningField = ""; + // Some templates reject the optional OpenAI function.response field and require it + // to be stripped before rendering. This capability comes from upstream 2026.5. bool supportsResponseFieldInToolDefinition = false; bool needsWorkarounds() const { - return requiresObjectArguments || !missnamedReasoningField.empty() || supportsResponseFieldInToolDefinition; + return requiresObjectArguments || parseToolResponseJsonContent || !missnamedReasoningField.empty() || supportsResponseFieldInToolDefinition; } std::string toString() const { return std::string("supportsToolCalls=") + (supportsToolCalls ? "true" : "false") + ", requiresObjectArguments=" + (requiresObjectArguments ? "true" : "false") + + ", parseToolResponseJsonContent=" + (parseToolResponseJsonContent ? "true" : "false") + ", missnamedReasoningField=" + missnamedReasoningField + ", supportsResponseFieldInToolDefinition=" + (supportsResponseFieldInToolDefinition ? "true" : "false"); } diff --git a/src/llm/io_processing/gemma4/gemma4_reasoning_parser.cpp b/src/llm/io_processing/gemma4/gemma4_reasoning_parser.cpp index 5af329a69c..c7e848098c 100644 --- a/src/llm/io_processing/gemma4/gemma4_reasoning_parser.cpp +++ b/src/llm/io_processing/gemma4/gemma4_reasoning_parser.cpp @@ -22,24 +22,42 @@ #include "gemma4_reasoning_parser.hpp" namespace ovms { -void Gemma4ReasoningParser::skipToken(const std::vector& generatedTokens, size_t& pos, int64_t tokenId) { - if (pos < generatedTokens.size() && generatedTokens[pos] == tokenId) { - pos++; - } -} -std::optional Gemma4ReasoningParser::parseChunk(const std::string& chunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason finishReason) { +std::optional Gemma4ReasoningParser::parseChunk( + const std::string& chunk, + const std::vector& /*tokens*/, + ov::genai::GenerationFinishReason /*finishReason*/) { if (chunk.empty()) { - SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Received empty chunk for Gemma4ReasoningParser"); return std::nullopt; } - if (chunk.find(parsingConfig.startTags[0]) != std::string::npos || chunk.find(parsingConfig.endTag) != std::string::npos || - chunk.find(parsingConfig.preambleStartTags[0]) != std::string::npos) { + std::string text = chunk; + + // The generic OutputParser normally splits at , but keep this + // defensive stripping for direct parser use and boundary-sharing chunks. + const size_t endTagPos = text.rfind(parsingConfig.endTag); + if (endTagPos != std::string::npos) { + text = text.substr(0, endTagPos); + } + + // Gemma4's opener is a channel marker plus a role label. Strip it exactly + // once at phase entry. On post-tool continuation the chat template may have + // already placed the opener in the prompt, so generated text starts directly + // inside reasoning and there is nothing to strip. + if (!phaseEntryTagConsumed) { + const std::string& startTag = parsingConfig.startTags.front(); + const size_t startTagPos = text.find(startTag); + if (startTagPos != std::string::npos) { + text = text.substr(startTagPos + startTag.size()); + } + phaseEntryTagConsumed = true; + } + + if (text.empty()) { return std::nullopt; - } else { - return ReasoningDelta{chunk}; } - return std::nullopt; + + return ReasoningDelta{text}; } + } // namespace ovms diff --git a/src/llm/io_processing/gemma4/gemma4_reasoning_parser.hpp b/src/llm/io_processing/gemma4/gemma4_reasoning_parser.hpp index 0dfbd2fece..3a574c3a2d 100644 --- a/src/llm/io_processing/gemma4/gemma4_reasoning_parser.hpp +++ b/src/llm/io_processing/gemma4/gemma4_reasoning_parser.hpp @@ -16,40 +16,51 @@ #pragma once #include -#include +#include #include +#include +#include -#include "../qwen3/reasoning_parser.hpp" +#include "../base_output_parser.hpp" namespace ovms { -class Gemma4ReasoningParser : public Qwen3ReasoningParser { -protected: - const int64_t channelStartTokenId = 100; // <|channel> - const int64_t channelEndTokenId = 101; // - - const std::string reasoningStrIndicator = "thought\n"; - const std::string parsingStartTag = "<|channel>" + reasoningStrIndicator; - const std::string parsingEndTag = ""; - void skipToken(const std::vector& generatedTokens, size_t& pos, int64_t tokenId); +class Gemma4ReasoningParser : public BaseOutputParser { + bool phaseEntryTagConsumed{false}; public: Gemma4ReasoningParser() = delete; + + static OutputParsingConfig defaultParsingConfig() { + OutputParsingConfig cfg; + cfg.startTags = {"<|channel>thought\n"}; + // <|channel> is a single special token. The generic streamer can hold it + // until the following `thought\n` role label completes the semantic opener. + cfg.tokenIdStartTags = {"<|channel>"}; + cfg.endTag = ""; + cfg.needsSpecialTokens = true; + // Google's canonical Gemma4 tool sequence explicitly closes the thought + // channel with before <|tool_call>. Keep tool-start takeover as + // a tolerant recovery boundary only: if malformed/edge output or streaming + // state presents a complete native tool opener while reasoning still owns + // the stream, preserve the reasoning prefix and hand the opener intact to + // the tool parser instead of swallowing it as reasoning. + cfg.toolStartTerminatesReasoning = true; + return cfg; + } + explicit Gemma4ReasoningParser(ov::genai::Tokenizer& tokenizer, std::optional configOverride = std::nullopt) : - Qwen3ReasoningParser(tokenizer, [&]() -> std::optional { - if (configOverride.has_value()) - return configOverride; - OutputParsingConfig cfg; - cfg.startTags = {"<|channel>thought\n"}; - cfg.preambleStartTags = {"thought\n"}; - cfg.tokenIdStartTags = {"<|channel>"}; - cfg.endTag = ""; - cfg.needsSpecialTokens = true; - return cfg; - }()) { - resolveSpecialTokenIds(); + BaseOutputParser(tokenizer, + configOverride.has_value() ? std::move(*configOverride) : defaultParsingConfig()) {} + + void resetState() override { + phaseEntryTagConsumed = false; } - std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; + + std::optional parseChunk(const std::string& chunk, + const std::vector& tokens, + ov::genai::GenerationFinishReason finishReason) override; }; + } // namespace ovms diff --git a/src/llm/io_processing/gemma4/gemma4_tool_parser.cpp b/src/llm/io_processing/gemma4/gemma4_tool_parser.cpp index a434431920..d7d766e032 100644 --- a/src/llm/io_processing/gemma4/gemma4_tool_parser.cpp +++ b/src/llm/io_processing/gemma4/gemma4_tool_parser.cpp @@ -6,293 +6,817 @@ // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. //***************************************************************************** + #include "gemma4_tool_parser.hpp" -#include "../utils.hpp" -#include "../../../logging.hpp" -#include "../../../stringutils.hpp" -#include "rapidjson/error/en.h" + #include #include +#include +#include +#include #include +#include + +#include "../utils.hpp" +#include "../../../logging.hpp" +#include "../../../stringutils.hpp" +#include "src/port/rapidjson_document.hpp" +#include "src/port/rapidjson_stringbuffer.hpp" +#include "src/port/rapidjson_writer.hpp" namespace ovms { const std::string Gemma4ToolParser::TOOL_CALL_START_TAG = "<|tool_call>"; const std::string Gemma4ToolParser::TOOL_CALL_END_TAG = ""; const std::string Gemma4ToolParser::TOOL_CALL_NAME_PREFIX = "call:"; - -const std::string Gemma4ToolParser::TOOL_ARGS_START_INDICATOR = "{"; -const std::string Gemma4ToolParser::TOOL_ARGS_END_INDICATOR = "}"; const std::string Gemma4ToolParser::TOOL_ARGS_STRING_INDICATOR = "<|\"|>"; -const std::string Gemma4ToolParser::TOOL_ARGS_SEPARATOR_STR = ","; - const std::string Gemma4ToolParser::TURN_END_TAG = ""; const std::string Gemma4ToolParser::TOOL_RESPONSE_START_TAG = "<|tool_response>"; -const int64_t Gemma4ToolParser::botTokenId = 48; // <|tool_call> -const int64_t Gemma4ToolParser::eotTokenId = 49; // +namespace { -const int64_t Gemma4ToolParser::reasoningTokenId = 100; // <|channel> -const int64_t Gemma4ToolParser::reasoningEndTokenId = 101; // +using JsonWriter = rapidjson::Writer; -std::string Gemma4ToolParser::parseArrayParameter(const std::string& argumentStr) { - std::string body = argumentStr.substr(1, argumentStr.size() - 2); - trim(body); - if (body.empty()) { - return "[]"; +// Tool arguments are JSON text, not machine arithmetic. Preserve number tokens +// without routing large integers/decimals through uint64_t or double. +class NumberPreservingWriter : public JsonWriter { +public: + explicit NumberPreservingWriter(rapidjson::StringBuffer& buffer) : JsonWriter(buffer) {} + bool RawNumber(const char* value, rapidjson::SizeType length, bool) { + return RawValue(value, length, rapidjson::kNumberType); } +}; + +std::optional normalizeJsonLosslessly(const std::string& input) { + rapidjson::StringStream stream(input.c_str()); + rapidjson::Reader reader; + rapidjson::StringBuffer buffer; + NumberPreservingWriter writer(buffer); + if (!reader.Parse(stream, writer) || stream.Tell() != input.size()) + return std::nullopt; + return std::string(buffer.GetString(), buffer.GetSize()); +} - std::string parsedArray = "["; - bool firstElement = true; - for (const std::string& element : splitRespectingSpecialChars(body, TOOL_ARGS_SEPARATOR_STR, maskDelimitedStringValues(body, TOOL_ARGS_STRING_INDICATOR))) { - if (!firstElement) { - parsedArray += ","; - } - parsedArray += normalizeArgStr(element); - firstElement = false; +bool isValidJsonNumber(const std::string& token) { + size_t pos = 0; + if (pos < token.size() && token[pos] == '-') + ++pos; + if (pos == token.size()) + return false; + if (token[pos] == '0') { + ++pos; + } else { + if (!std::isdigit(static_cast(token[pos]))) + return false; + while (pos < token.size() && std::isdigit(static_cast(token[pos]))) + ++pos; } - parsedArray += "]"; - return parsedArray; + if (pos < token.size() && token[pos] == '.') { + ++pos; + const size_t digits = pos; + while (pos < token.size() && std::isdigit(static_cast(token[pos]))) + ++pos; + if (pos == digits) + return false; + } + if (pos < token.size() && (token[pos] == 'e' || token[pos] == 'E')) { + ++pos; + if (pos < token.size() && (token[pos] == '+' || token[pos] == '-')) + ++pos; + const size_t digits = pos; + while (pos < token.size() && std::isdigit(static_cast(token[pos]))) + ++pos; + if (pos == digits) + return false; + } + return pos == token.size(); } -std::string Gemma4ToolParser::parseObjectParameter(const std::string& argumentStr) { - std::string body = argumentStr.substr(1, argumentStr.size() - 2); - trim(body); - if (body.empty()) { - return "{}"; - } - - std::string parsedObject = "{"; - bool firstMember = true; - for (const std::string& member : splitRespectingSpecialChars(body, TOOL_ARGS_SEPARATOR_STR, maskDelimitedStringValues(body, TOOL_ARGS_STRING_INDICATOR))) { - const std::string maskedMember = maskDelimitedStringValues(member, TOOL_ARGS_STRING_INDICATOR); - size_t keyEndPos = findInStringRespectingSpecialChars(maskedMember, ":", 0); - if (keyEndPos == std::string::npos) { - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Object member does not contain a key separator, leaving argument unchanged. Member: {}", member); - return argumentStr; +void trimLocal(std::string& value) { + auto notSpace = [](unsigned char c) { return !std::isspace(c); }; + value.erase(value.begin(), std::find_if(value.begin(), value.end(), notSpace)); + value.erase(std::find_if(value.rbegin(), value.rend(), notSpace).base(), value.end()); +} + +bool saneToolName(const std::string& name) { + if (name.empty()) + return false; + return std::all_of(name.begin(), name.end(), [](unsigned char c) { + return std::isalnum(c) || c == '_' || c == '-' || c == '.'; + }); +} + +// Recover only the observed native leak shape where `call:` starts a logical line +// (optionally indented). This deliberately rejects prose such as +// `Documentation example: call:foo{...}` and quoted examples. The candidate is +// still validated again by the normal argument parser before it becomes executable. +std::optional findRecoverableBareCall( + const std::string& content, + size_t from, + const std::unordered_set& allowedToolNames, + bool enforceToolRegistry) { + size_t candidate = content.find(Gemma4ToolParser::TOOL_CALL_NAME_PREFIX, from); + while (candidate != std::string::npos) { + bool lineBoundary = candidate == from; + if (!lineBoundary) { + const size_t lineStartPos = content.rfind('\n', candidate - 1); + const size_t lineStart = lineStartPos == std::string::npos ? from : lineStartPos + 1; + lineBoundary = lineStart >= from; + for (size_t i = lineStart; lineBoundary && i < candidate; ++i) { + const char c = content[i]; + if (c != ' ' && c != '\t' && c != '\r') + lineBoundary = false; + } } - std::string key = member.substr(0, keyEndPos); - trim(key); - if (isWrappedByDelimiter(key, TOOL_ARGS_STRING_INDICATOR)) { - key = key.substr(TOOL_ARGS_STRING_INDICATOR.size(), key.size() - 2 * TOOL_ARGS_STRING_INDICATOR.size()); + if (!lineBoundary) { + candidate = content.find(Gemma4ToolParser::TOOL_CALL_NAME_PREFIX, candidate + Gemma4ToolParser::TOOL_CALL_NAME_PREFIX.size()); + continue; } - if (!firstMember) { - parsedObject += ","; + + const size_t nameStart = candidate + Gemma4ToolParser::TOOL_CALL_NAME_PREFIX.size(); + const size_t bracePos = content.find('{', nameStart); + const size_t parenPos = content.find('(', nameStart); + size_t argsPos = std::string::npos; + if (bracePos != std::string::npos) + argsPos = bracePos; + if (parenPos != std::string::npos && (argsPos == std::string::npos || parenPos < argsPos)) + argsPos = parenPos; + if (argsPos == std::string::npos) { + if (!enforceToolRegistry) + return candidate; + + std::string partialName = content.substr(nameStart); + trimLocal(partialName); + if (partialName.empty()) + return candidate; + + const bool couldBecomeAllowed = saneToolName(partialName) && std::any_of( + allowedToolNames.begin(), allowedToolNames.end(), [&](const std::string& allowedName) { + return allowedName.rfind(partialName, 0) == 0; + }); + if (couldBecomeAllowed) + return candidate; // hold only a viable streaming tool-name prefix + + candidate = content.find(Gemma4ToolParser::TOOL_CALL_NAME_PREFIX, candidate + Gemma4ToolParser::TOOL_CALL_NAME_PREFIX.size()); + continue; } - parsedObject += escapeAsJsonString(key) + ":" + normalizeArgStr(member.substr(keyEndPos + 1)); - firstMember = false; + + std::string name = content.substr(nameStart, argsPos - nameStart); + trimLocal(name); + const bool allowed = saneToolName(name) && (!enforceToolRegistry || allowedToolNames.count(name) != 0); + if (allowed) + return candidate; + + candidate = content.find(Gemma4ToolParser::TOOL_CALL_NAME_PREFIX, candidate + Gemma4ToolParser::TOOL_CALL_NAME_PREFIX.size()); } - parsedObject += "}"; - return parsedObject; + return std::nullopt; } -std::string Gemma4ToolParser::normalizeArgStr(const std::string& arg) { - std::string normalized = arg; - trim(normalized); - if (normalized.empty()) { - return "\"\""; +// Whether a trailing line fragment could still become a recoverable bare +// `call:` boundary with more streaming input. Only spaces plus a proper +// prefix of "call:" (no completed colon) are holdable here; a completed +// "call:" (with or without a tool name) is handled by findRecoverableBareCall +// as a complete split point, and anything else on the line already rules out a +// line-start bare call. +bool isHoldableBareCallPrefix(const std::string& lineText) { + if (lineText.empty()) + return false; + size_t i = 0; + while (i < lineText.size() && (lineText[i] == ' ' || lineText[i] == '\t' || lineText[i] == '\r')) + ++i; + const std::string rest = lineText.substr(i); + if (rest.empty()) + return true; // spaces only after a newline: may become " call:" next chunk + if (rest.size() >= Gemma4ToolParser::TOOL_CALL_NAME_PREFIX.size()) + return false; // completed "call:" or longer: split logic owns it, do not hold here + return Gemma4ToolParser::TOOL_CALL_NAME_PREFIX.compare(0, rest.size(), rest) == 0; +} + +// Earliest position from which trailing bytes must be held because they may +// still grow into a line-start bare `call:` boundary. Returns npos when the +// buffered suffix is safe to emit as ordinary content. +size_t bareCallHoldStart(const std::string& content, size_t from) { + if (content.size() <= from) + return std::string::npos; + const size_t nl = content.rfind('\n'); + size_t lineStartFull = (nl == std::string::npos) ? 0 : nl + 1; + if (lineStartFull < from) { + for (size_t i = lineStartFull; i < from; ++i) { + const char c = content[i]; + if (c != ' ' && c != '\t' && c != '\r') + return std::string::npos; // line already has prose: no bare call possible + } + if (isHoldableBareCallPrefix(content.substr(from))) + return from; + return std::string::npos; } + if (isHoldableBareCallPrefix(content.substr(lineStartFull))) + return lineStartFull; + return std::string::npos; +} - std::string lower = normalized; - std::transform(lower.begin(), lower.end(), lower.begin(), ::tolower); - if (lower == "true" || lower == "false" || lower == "null") { - return lower; +// Earliest position of a trailing partial "<|tool_call>" start tag that must +// be held until the next chunk proves or disproves the boundary. +size_t startTagHoldStart(const std::string& content, size_t from) { + const std::string& tag = Gemma4ToolParser::TOOL_CALL_START_TAG; + if (content.size() <= from || tag.size() <= 1) + return std::string::npos; + const size_t avail = content.size() - from; + size_t maxLen = std::min(avail, tag.size() - 1); + for (size_t len = maxLen; len > 0; --len) { + if (content.compare(content.size() - len, len, tag, 0, len) == 0) + return content.size() - len; } + return std::string::npos; +} - // Build valid JSON out of the Gemma4 specific syntax before handing it over to rapidjson. - if (isWrappedByDelimiter(normalized, TOOL_ARGS_STRING_INDICATOR)) { - normalized = escapeAsJsonString(normalized.substr(TOOL_ARGS_STRING_INDICATOR.size(), normalized.size() - 2 * TOOL_ARGS_STRING_INDICATOR.size())); - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Argument is a string, converted it to correct JSON format. Modified string: {}", normalized); - } else if (normalized.front() == '{' && normalized.back() == '}') { - normalized = parseObjectParameter(normalized); - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Argument is an object, converted it to correct JSON format. Modified string: {}", normalized); - } else if (normalized.front() == '[' && normalized.back() == ']') { - normalized = parseArrayParameter(normalized); - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Argument is an array, converted it to correct JSON format. Modified string: {}", normalized); +bool anchoredToolCallMayStartAt(const std::string& content, size_t pos) { + const std::string& tag = Gemma4ToolParser::TOOL_CALL_START_TAG; + const std::string& prefix = Gemma4ToolParser::TOOL_CALL_NAME_PREFIX; + if (pos + tag.size() > content.size() || content.compare(pos, tag.size(), tag) != 0) + return false; + const size_t afterTag = pos + tag.size(); + const size_t suffixSize = content.size() - afterTag; + if (suffixSize == 0) + return true; // full tag at tail; next chunk decides whether it is a call + if (content[afterTag] == ':') + return true; + if (suffixSize < prefix.size()) + return prefix.compare(0, suffixSize, content, afterTag, suffixSize) == 0; + return content.compare(afterTag, prefix.size(), prefix) == 0; +} + +std::optional findAnchoredToolCallStart(const std::string& content, size_t from) { + size_t pos = content.find(Gemma4ToolParser::TOOL_CALL_START_TAG, from); + while (pos != std::string::npos) { + if (anchoredToolCallMayStartAt(content, pos)) + return pos; + pos = content.find(Gemma4ToolParser::TOOL_CALL_START_TAG, pos + Gemma4ToolParser::TOOL_CALL_START_TAG.size()); } + return std::nullopt; +} + +class NativeValueParser { + const std::string& input; + size_t pos{0}; + JsonWriter& writer; - rapidjson::Document tempDoc; - tempDoc.Parse(normalized.c_str()); - if (!tempDoc.HasParseError()) { - return normalized; + bool startsWith(const std::string& marker) const { + return pos + marker.size() <= input.size() && input.compare(pos, marker.size(), marker) == 0; } - auto errorCode = tempDoc.GetParseError(); - auto errorMessage = rapidjson::GetParseError_En(errorCode); - size_t errorOffset = tempDoc.GetErrorOffset(); - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Failed to parse argument string as JSON, falling back to string value. Argument string: {}, Error: {} Offset: {}", normalized, errorMessage, errorOffset); + void skipWs() { + while (pos < input.size() && std::isspace(static_cast(input[pos]))) + ++pos; + } - return escapeAsJsonString(arg); -} + bool writeJsonToken(const std::string& token) { + auto normalized = normalizeJsonLosslessly(token); + if (!normalized) + return false; + if (!token.empty() && (std::isdigit(static_cast(token.front())) || token.front() == '-')) { + // Validate numeric syntax through RapidJSON while preserving the original + // lexical token. This keeps large/high-precision values lossless without + // allowing malformed forms such as `1.` or `1e` into OpenAI JSON. + if (*normalized != token) + return false; + return writer.RawValue(token.data(), static_cast(token.size()), rapidjson::kNumberType); + } + // This path accepts a quoted string or a bare scalar only. + const auto type = normalized->front() == '"' ? rapidjson::kStringType : + normalized->front() == 't' ? rapidjson::kTrueType : + normalized->front() == 'f' ? rapidjson::kFalseType : + normalized->front() == 'n' ? rapidjson::kNullType : rapidjson::kNumberType; + return writer.RawValue(normalized->data(), normalized->size(), type); + } -void Gemma4ToolParser::writeArgumentToWriter(const std::string& arg, rapidjson::Writer& writer) { - std::string normalized = normalizeArgStr(arg); + bool parseDelimitedString() { + if (!startsWith(Gemma4ToolParser::TOOL_ARGS_STRING_INDICATOR)) + return false; + pos += Gemma4ToolParser::TOOL_ARGS_STRING_INDICATOR.size(); + const size_t end = input.find(Gemma4ToolParser::TOOL_ARGS_STRING_INDICATOR, pos); + if (end == std::string::npos) + return false; + writer.String(input.data() + pos, static_cast(end - pos)); + pos = end + Gemma4ToolParser::TOOL_ARGS_STRING_INDICATOR.size(); + return true; + } - rapidjson::Document doc; - doc.Parse(normalized.c_str()); + bool parseJsonString() { + if (pos >= input.size() || input[pos] != '"') + return false; + const size_t start = pos++; + bool escaped = false; + while (pos < input.size()) { + const char c = input[pos++]; + if (escaped) { + escaped = false; + continue; + } + if (c == '\\') { + escaped = true; + continue; + } + if (c == '"') + return writeJsonToken(input.substr(start, pos - start)); + } + return false; + } - rapidjson::Value& argumentDoc = doc; - writeArgumentOfAnyType(argumentDoc, writer); -} + bool parseKey(std::string& key) { + skipWs(); + if (startsWith(Gemma4ToolParser::TOOL_ARGS_STRING_INDICATOR)) { + pos += Gemma4ToolParser::TOOL_ARGS_STRING_INDICATOR.size(); + const size_t end = input.find(Gemma4ToolParser::TOOL_ARGS_STRING_INDICATOR, pos); + if (end == std::string::npos) + return false; + key = input.substr(pos, end - pos); + pos = end + Gemma4ToolParser::TOOL_ARGS_STRING_INDICATOR.size(); + return true; + } + if (pos < input.size() && input[pos] == '"') { + const size_t start = pos++; + bool escaped = false; + while (pos < input.size()) { + const char c = input[pos++]; + if (escaped) { + escaped = false; + continue; + } + if (c == '\\') { + escaped = true; + continue; + } + if (c == '"') { + rapidjson::Document keyDoc; + const std::string token = input.substr(start, pos - start); + keyDoc.Parse(token.c_str()); + if (keyDoc.HasParseError() || !keyDoc.IsString()) + return false; + key.assign(keyDoc.GetString(), keyDoc.GetStringLength()); + return true; + } + } + return false; + } + const size_t start = pos; + while (pos < input.size() && input[pos] != ':') + ++pos; + if (pos == input.size()) + return false; + key = input.substr(start, pos - start); + trimLocal(key); + return !key.empty(); + } + + bool parseObject() { + if (pos >= input.size() || input[pos] != '{') + return false; + ++pos; + writer.StartObject(); + skipWs(); + if (pos < input.size() && input[pos] == '}') { + ++pos; + writer.EndObject(); + return true; + } + while (pos < input.size()) { + std::string key; + if (!parseKey(key)) + return false; + skipWs(); + if (pos >= input.size() || input[pos] != ':') + return false; + ++pos; + writer.Key(key.c_str(), static_cast(key.size())); + if (!parseValue()) + return false; + skipWs(); + if (pos < input.size() && input[pos] == ',') { + ++pos; + skipWs(); + continue; + } + if (pos < input.size() && input[pos] == '}') { + ++pos; + writer.EndObject(); + return true; + } + return false; + } + return false; + } + + bool parseArray() { + if (pos >= input.size() || input[pos] != '[') + return false; + ++pos; + writer.StartArray(); + skipWs(); + if (pos < input.size() && input[pos] == ']') { + ++pos; + writer.EndArray(); + return true; + } + while (pos < input.size()) { + if (!parseValue()) + return false; + skipWs(); + if (pos < input.size() && input[pos] == ',') { + ++pos; + skipWs(); + continue; + } + if (pos < input.size() && input[pos] == ']') { + ++pos; + writer.EndArray(); + return true; + } + return false; + } + return false; + } -std::pair Gemma4ToolParser::parseSingleArgument(const std::string& argumentStr) { - std::pair argument; + bool parseBareScalar() { + const size_t start = pos; + while (pos < input.size()) { + const char c = input[pos]; + if (c == ',' || c == '}' || c == ']' || c == ')') + break; + ++pos; + } + std::string token = input.substr(start, pos - start); + trimLocal(token); + if (token.empty()) + return false; + const bool numericCandidate = std::isdigit(static_cast(token.front())) || token.front() == '-'; + if (numericCandidate) { + if (!isValidJsonNumber(token)) + return false; + return writer.RawValue(token.data(), static_cast(token.size()), rapidjson::kNumberType); + } + if (writeJsonToken(token)) + return true; + writer.String(token.c_str(), static_cast(token.size())); + return true; + } - size_t colonPos = argumentStr.find(':'); - if (colonPos != std::string::npos) { - argument.first = argumentStr.substr(0, colonPos); - std::string value = argumentStr.substr(colonPos + 1); - argument.second = value; - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed argument - name: {}, value: {}", argument.first, argument.second); - } else { - argument.first = argumentStr; - argument.second = ""; - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Argument string: {} does not contain ':', setting name as entire string and value as empty", argumentStr); +public: + NativeValueParser(const std::string& input, JsonWriter& writer) : input(input), writer(writer) {} + + bool parseValue() { + skipWs(); + if (pos >= input.size()) + return false; + if (startsWith(Gemma4ToolParser::TOOL_ARGS_STRING_INDICATOR)) + return parseDelimitedString(); + if (input[pos] == '"') + return parseJsonString(); + if (input[pos] == '{') + return parseObject(); + if (input[pos] == '[') + return parseArray(); + return parseBareScalar(); + } + + bool parseArgumentsBody() { + writer.StartObject(); + skipWs(); + if (pos == input.size()) { + writer.EndObject(); + return true; + } + while (pos < input.size()) { + std::string key; + if (!parseKey(key)) + return false; + skipWs(); + if (pos >= input.size() || input[pos] != ':') + return false; + ++pos; + writer.Key(key.c_str(), static_cast(key.size())); + if (!parseValue()) + return false; + skipWs(); + if (pos == input.size()) { + writer.EndObject(); + return true; + } + if (input[pos] != ',') + return false; + ++pos; + skipWs(); + if (pos == input.size()) + return false; + } + return false; } - trim(argument.first); - return argument; + bool parseSingleValueFully() { + if (!parseValue()) + return false; + skipWs(); + return pos == input.size(); + } +}; + +std::optional normalizeSingleNativeValue(const std::string& arg) { + std::string value = arg; + trimLocal(value); + if (value.empty()) + return std::nullopt; + + rapidjson::StringBuffer buffer; + JsonWriter writer(buffer); + NativeValueParser parser(value, writer); + if (!parser.parseSingleValueFully()) + return std::nullopt; + return std::string(buffer.GetString(), buffer.GetSize()); +} + +} // namespace + +std::optional Gemma4ToolParser::parseNativeArgumentsBody(const std::string& argumentsBody) { + const std::string jsonCandidate = "{" + argumentsBody + "}"; + rapidjson::StringBuffer buffer; + JsonWriter writer(buffer); + NativeValueParser parser(argumentsBody, writer); + if (!parser.parseArgumentsBody()) + return std::nullopt; + return std::string(buffer.GetString(), buffer.GetSize()); } -std::vector> Gemma4ToolParser::parseArguments(const std::string& argumentsStr) { - std::vector args; - std::vector> parsedArgs; - - const std::string maskedArgumentsStr = maskDelimitedStringValues(argumentsStr, TOOL_ARGS_STRING_INDICATOR); - size_t argPos = 0; - while (argPos < argumentsStr.length()) { - size_t commaPos = findInStringRespectingSpecialChars(maskedArgumentsStr, TOOL_ARGS_SEPARATOR_STR, argPos); - if (commaPos == std::string::npos) { - auto remainingStr = argumentsStr.substr(argPos); - args.push_back(remainingStr); - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "No more commas found, adding remaining argument string: {}", remainingStr); +std::optional Gemma4ToolParser::findMatchingContainerEnd(const std::string& text, size_t openPos, char openChar, char closeChar, size_t& malformedEndTag) { + malformedEndTag = std::string::npos; + if (openPos >= text.size() || text[openPos] != openChar) + return std::nullopt; + + std::vector expectedClosers{closeChar}; + bool malformed = false; + size_t i = openPos + 1; + while (i < text.size()) { + if (text.compare(i, TOOL_ARGS_STRING_INDICATOR.size(), TOOL_ARGS_STRING_INDICATOR) == 0) { + const size_t valueStart = i + TOOL_ARGS_STRING_INDICATOR.size(); + const size_t valueEnd = text.find(TOOL_ARGS_STRING_INDICATOR, valueStart); + if (valueEnd == std::string::npos) + return std::nullopt; + i = valueEnd + TOOL_ARGS_STRING_INDICATOR.size(); + continue; + } + + if (text[i] == '"') { + ++i; + bool escaped = false; + while (i < text.size()) { + const char c = text[i++]; + if (escaped) { + escaped = false; + continue; + } + if (c == '\\') { + escaped = true; + continue; + } + if (c == '"') + break; + } + continue; + } + + if (text.compare(i, TOOL_CALL_END_TAG.size(), TOOL_CALL_END_TAG) == 0) { + malformedEndTag = i; + return std::nullopt; + } + + switch (text[i]) { + case '{': expectedClosers.push_back('}'); break; + case '[': expectedClosers.push_back(']'); break; + case '(': expectedClosers.push_back(')'); break; + case '}': + case ']': + case ')': + if (expectedClosers.empty() || expectedClosers.back() != text[i]) { + malformed = true; + break; + } + expectedClosers.pop_back(); + if (expectedClosers.empty() && !malformed) + return i; break; + default: break; } - std::string argStr = argumentsStr.substr(argPos, commaPos - argPos); - args.push_back(argStr); - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed argument string: {}", argStr); - argPos = commaPos + TOOL_ARGS_SEPARATOR_STR.length(); + ++i; } + return std::nullopt; +} - for (const std::string& arg : args) { - parsedArgs.push_back(parseSingleArgument(arg)); - } - return parsedArgs; +std::string Gemma4ToolParser::normalizeToolName(std::string rawName) { + trim(rawName); + if (rawName.rfind(TOOL_CALL_NAME_PREFIX, 0) == 0) + rawName.erase(0, TOOL_CALL_NAME_PREFIX.size()); + trim(rawName); + if (!rawName.empty() && rawName.front() == ':') + rawName.erase(rawName.begin()); + trim(rawName); + return rawName; +} + +std::string Gemma4ToolParser::normalizeArgStr(const std::string& arg) { + auto normalized = normalizeSingleNativeValue(arg); + return normalized.value_or(arg); +} + +std::string Gemma4ToolParser::parseArrayParameter(const std::string& argumentStr) { + return normalizeArgStr(argumentStr); +} + +std::string Gemma4ToolParser::parseObjectParameter(const std::string& argumentStr) { + return normalizeArgStr(argumentStr); } bool Gemma4ToolParser::parseInContentState() { - size_t toolCallStartTagPos = this->streamingContent.find(TOOL_CALL_START_TAG, this->streamingPosition); - if (toolCallStartTagPos != std::string::npos) { - if (toolCallStartTagPos > this->streamingPosition) { - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Content found before tool call start tag at position: {}", toolCallStartTagPos); + const auto toolCallStartTagPos = findAnchoredToolCallStart(streamingContent, streamingPosition); + if (toolCallStartTagPos.has_value()) { + if (toolCallStartTagPos.value() > streamingPosition) return true; - } - this->streamingPosition = toolCallStartTagPos + TOOL_CALL_START_TAG.length(); - this->currentState = State::ToolCallStarted; - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Detected start of tool call at position: {}", toolCallStartTagPos); + const size_t namePrefixPos = toolCallStartTagPos.value() + TOOL_CALL_START_TAG.length(); + const size_t suffixSize = streamingContent.size() - namePrefixPos; + if (suffixSize < TOOL_CALL_NAME_PREFIX.size() && + TOOL_CALL_NAME_PREFIX.compare(0, suffixSize, streamingContent, namePrefixPos, suffixSize) == 0) + return false; + const bool colonVariant = suffixSize > 0 && streamingContent[namePrefixPos] == ':'; + if (!colonVariant && streamingContent.compare(namePrefixPos, TOOL_CALL_NAME_PREFIX.size(), TOOL_CALL_NAME_PREFIX) != 0) + return true; + currentCallStartPos = toolCallStartTagPos.value(); + currentCallBare = false; + streamingPosition = namePrefixPos + (colonVariant ? 1 : TOOL_CALL_NAME_PREFIX.size()); + currentState = State::ToolCallStarted; + currentCallValid = true; return false; } + const auto bareCallPos = findRecoverableBareCall(streamingContent, streamingPosition, allowedToolNames, enforceToolRegistry); + if (bareCallPos.has_value()) { + if (bareCallPos.value() > streamingPosition) + return true; + currentCallStartPos = bareCallPos.value(); + currentCallBare = true; + streamingPosition = bareCallPos.value() + TOOL_CALL_NAME_PREFIX.size(); + currentState = State::ToolCallStarted; + currentCallValid = true; + return false; + } return true; } bool Gemma4ToolParser::parseInToolCallState() { - size_t argsPos = this->streamingContent.find(TOOL_ARGS_START_INDICATOR, this->streamingPosition); - if (argsPos == std::string::npos) { + const size_t endTagPos = streamingContent.find(TOOL_CALL_END_TAG, streamingPosition); + const size_t bracePos = streamingContent.find('{', streamingPosition); + const size_t parenPos = streamingContent.find('(', streamingPosition); + + size_t argsPos = std::string::npos; + if (bracePos != std::string::npos) + argsPos = bracePos; + if (parenPos != std::string::npos && (argsPos == std::string::npos || parenPos < argsPos)) + argsPos = parenPos; + + if (endTagPos != std::string::npos && (argsPos == std::string::npos || endTagPos < argsPos)) { + if (currentCallBare) { + // Bare `call:` ended by "" before any argument container: + // it was never an anchored call, so keep the bytes as prose. + streamingPosition = currentCallStartPos; + currentState = State::Content; + currentCallBare = false; + toolCall = {}; + currentCallValid = false; + return false; + } + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Gemma4 tool call ended before an argument container; dropping malformed call"); + streamingPosition = endTagPos + TOOL_CALL_END_TAG.size(); + currentState = State::AfterToolCall; + currentCallValid = false; + currentCallBare = false; + toolCall = {}; + return true; + } + if (argsPos == std::string::npos) + return false; + + std::string toolName = normalizeToolName(streamingContent.substr(streamingPosition, argsPos - streamingPosition)); + currentCallValid = saneToolName(toolName) && toolNameAllowed(toolName); + if (!currentCallValid) + SPDLOG_LOGGER_WARN(llm_calculator_logger, "Gemma4 parser refusing malformed or unavailable tool name: '{}'", toolName); + + if (!currentCallValid && currentCallBare) { + // A bare line-start `call:` without an anchored "<|tool_call>" marker + // is ordinary prose unless it names an available tool. Rewind so the + // bytes re-emit as content instead of being dropped as a refused call. + // findRecoverableBareCall will skip this now-delimited invalid name on + // the next pass, so the rewind terminates. + streamingPosition = currentCallStartPos; + currentState = State::Content; + currentCallBare = false; + toolCall = {}; return false; } + currentCallBare = false; - size_t toolNameStart = this->streamingContent.find(TOOL_CALL_NAME_PREFIX, this->streamingPosition); - if (toolNameStart != std::string::npos && toolNameStart < argsPos) { - toolNameStart += TOOL_CALL_NAME_PREFIX.length(); + currentArgsOpen = streamingContent[argsPos]; + currentArgsClose = currentArgsOpen == '(' ? ')' : '}'; + streamingPosition = argsPos + 1; + currentState = State::ToolCallParameters; + + if (currentCallValid) { + toolCall = ToolCall{generateRandomId(), toolName, ""}; } else { - toolNameStart = this->streamingPosition; + toolCall = {}; } - - std::string toolName = this->streamingContent.substr(toolNameStart, argsPos - toolNameStart); - trim(toolName); - this->toolCall = ToolCall{generateRandomId(), toolName, ""}; - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed tool name: {}", toolName); - this->streamingPosition = argsPos + TOOL_ARGS_START_INDICATOR.length(); - this->currentState = State::ToolCallParameters; - this->toolCallIndex++; return true; } bool Gemma4ToolParser::parseToolCallParametersState() { - if (this->streamingContent.back() == TOOL_ARGS_END_INDICATOR.back()) { - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Tool arguments end indicator found at the end of streaming content, attempting to parse arguments: {}", this->streamingContent.substr(this->streamingPosition)); - } - const std::string maskedStreamingContent = maskDelimitedStringValues(this->streamingContent, TOOL_ARGS_STRING_INDICATOR); - size_t pos = findInStringRespectingSpecialChars(maskedStreamingContent, TOOL_ARGS_END_INDICATOR, this->streamingPosition); - if (pos == std::string::npos) { - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Tool arguments end indicator not found in streaming content starting from position: {}", this->streamingPosition); + if (streamingPosition == 0) + return false; + const size_t openPos = streamingPosition - 1; + size_t endTagPos = std::string::npos; + auto closePos = findMatchingContainerEnd(streamingContent, openPos, currentArgsOpen, currentArgsClose, endTagPos); + if (!closePos.has_value()) { + if (endTagPos != std::string::npos) { + SPDLOG_LOGGER_WARN(llm_calculator_logger, "Gemma4 malformed tool arguments bounded by ; dropping current call"); + streamingPosition = endTagPos + TOOL_CALL_END_TAG.size(); + currentState = State::AfterToolCall; + currentCallValid = false; + currentCallBare = false; + toolCall = {}; + return true; + } return false; } - std::string argumentsStr = this->streamingContent.substr(this->streamingPosition, pos - this->streamingPosition); - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed arguments string: {}", argumentsStr); - std::vector> arguments = parseArguments(argumentsStr); - - rapidjson::Document argsDoc(rapidjson::kObjectType); - rapidjson::StringBuffer sb; - rapidjson::Writer argsWriter(sb); - argsWriter.StartObject(); - for (const std::pair& argument : arguments) { - argsWriter.Key(argument.first.c_str()); - writeArgumentToWriter(argument.second, argsWriter); + const std::string argumentsBody = streamingContent.substr(streamingPosition, closePos.value() - streamingPosition); + if (currentCallValid) { + auto parsedArguments = parseNativeArgumentsBody(argumentsBody); + if (parsedArguments.has_value()) { + toolCall.arguments = std::move(parsedArguments.value()); + } else { + SPDLOG_LOGGER_WARN(llm_calculator_logger, "Gemma4 native argument parse failed; refusing executable tool call '{}'.", toolCall.name); + currentCallValid = false; + toolCall = {}; + } } - argsWriter.EndObject(); - this->toolCall.arguments = sb.GetString(); - this->currentState = State::ToolCallEnded; - this->streamingPosition = pos + TOOL_ARGS_END_INDICATOR.length(); - + streamingPosition = closePos.value() + 1; + currentState = State::ToolCallEnded; return true; } bool Gemma4ToolParser::parseInToolCallEndedState() { - size_t nextToolCallPos = this->streamingContent.find(TOOL_CALL_NAME_PREFIX, this->streamingPosition); - size_t toolCallEndTagPos = this->streamingContent.find(TOOL_CALL_END_TAG, this->streamingPosition); - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Current state: ToolCallEnded. Streaming content from current position: {}", this->streamingContent.substr(this->streamingPosition)); - if (nextToolCallPos != std::string::npos && nextToolCallPos < toolCallEndTagPos) { - this->streamingPosition = nextToolCallPos; - this->currentState = State::ToolCallStarted; - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Detected next tool call at position: {}", nextToolCallPos); - } else if (toolCallEndTagPos != std::string::npos) { - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Detected end of tool call at position: {}", toolCallEndTagPos); - this->streamingPosition = toolCallEndTagPos + TOOL_CALL_END_TAG.length(); - this->currentState = State::AfterToolCall; - } else { - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Waiting for more data in ToolCallEnded state; no complete next tool call prefix or end tag found from position: {}", this->streamingPosition); - return false; + const size_t endTagPos = streamingContent.find(TOOL_CALL_END_TAG, streamingPosition); + const size_t nextCallPos = streamingContent.find(TOOL_CALL_NAME_PREFIX, streamingPosition); + + if (nextCallPos != std::string::npos && (endTagPos == std::string::npos || nextCallPos < endTagPos)) { + // A chained call after "<|tool_call>" framing stays anchored (drop on + // invalid); a bare line-start `call:` rewinds to content on invalid. + currentCallBare = true; + currentCallStartPos = nextCallPos; + const size_t tagPos = streamingContent.rfind(TOOL_CALL_START_TAG, nextCallPos); + if (tagPos != std::string::npos && tagPos >= streamingPosition && + tagPos + TOOL_CALL_START_TAG.size() <= nextCallPos) { + bool gapClean = true; + for (size_t i = tagPos + TOOL_CALL_START_TAG.size(); i < nextCallPos; ++i) { + const char c = streamingContent[i]; + if (c != ' ' && c != '\t' && c != '\r' && c != '\n') { + gapClean = false; + break; + } + } + if (gapClean) { + currentCallBare = false; + currentCallStartPos = tagPos; + } + } + streamingPosition = nextCallPos + TOOL_CALL_NAME_PREFIX.size(); + currentState = State::ToolCallStarted; + currentCallValid = true; + return true; } - return true; + if (endTagPos != std::string::npos) { + streamingPosition = endTagPos + TOOL_CALL_END_TAG.length(); + currentState = State::AfterToolCall; + currentCallBare = false; + return true; + } + return false; } bool Gemma4ToolParser::parseNewContent() { - switch (this->currentState) { - case State::Content: { - return parseInContentState(); - } - case State::ToolCallStarted: { - return parseInToolCallState(); - } - case State::ToolCallParameters: { - return parseToolCallParametersState(); - } - case State::ToolCallEnded: { - return parseInToolCallEndedState(); - } - case State::AfterToolCall: - break; + switch (currentState) { + case State::Content: return parseInContentState(); + case State::ToolCallStarted: return parseInToolCallState(); + case State::ToolCallParameters: return parseToolCallParametersState(); + case State::ToolCallEnded: return parseInToolCallEndedState(); + case State::AfterToolCall: break; } return false; } @@ -303,35 +827,70 @@ std::optional Gemma4ToolParser::wrapDeltaContent(const std::string& conte return ContentDelta{content}; } -ToolCallDelta Gemma4ToolParser::wrapDeltaArgs(const std::string& argsStr, int toolCallIndex) { - return ToolCallDelta{toolCallIndex, std::nullopt, std::nullopt, argsStr}; +ToolCallDelta Gemma4ToolParser::wrapDeltaArgs(const std::string& argsStr, int index) { + return ToolCallDelta{index, std::nullopt, std::nullopt, argsStr}; } std::optional Gemma4ToolParser::parseChunk(const std::string& chunk, const std::vector& /*tokens*/, ov::genai::GenerationFinishReason finishReason) { - if (!chunk.empty()) { - this->streamingContent += chunk; + // Emitted deltas own their strings. Only the unconsumed suffix belongs to + // this parser; it is not conversation memory. Preserve the argument opener + // while its container is incomplete (the scanner starts one byte before pos). + if (streamingPosition >= 4096) { + const size_t keep = currentState == State::ToolCallParameters ? 1 : 0; + streamingContent.erase(0, streamingPosition - keep); + streamingPosition = keep; } - - if (parseNewContent()) { - if (this->currentState == State::ToolCallParameters) { - return ToolCallDelta{toolCallIndex, generateRandomId(), this->toolCall.name, ""}; - } - if (this->currentState == State::ToolCallEnded) { - auto delta = wrapDeltaArgs(this->toolCall.arguments, toolCallIndex); - this->toolCall = ToolCall{}; - return delta; - } - if (this->currentState == State::Content) { - size_t contentEnd = this->streamingContent.find(TOOL_CALL_START_TAG, this->streamingPosition); - std::string content; - if (contentEnd != std::string::npos) { - content = this->streamingContent.substr(this->streamingPosition, contentEnd - this->streamingPosition); - } else { - content = this->streamingContent.substr(this->streamingPosition); + if (!chunk.empty()) + streamingContent += chunk; + + for (;;) { + const State stateBefore = currentState; + const size_t positionBefore = streamingPosition; + const bool ready = parseNewContent(); + + if (currentState == State::ToolCallEnded) { + if (currentCallValid && !toolCall.arguments.empty()) { + // An emitted header cannot be retracted from SSE or the unary + // accumulator. Publish the complete call only after validation. + auto delta = ToolCallDelta{++toolCallIndex, toolCall.id, toolCall.name, toolCall.arguments}; + toolCall = {}; + currentCallValid = false; + return delta; } - this->streamingPosition += content.size(); + // Nothing to emit and parseInToolCallEndedState found no following + // boundary (no next call, no end tag). Waiting here must not spin: + // the previous loop re-entered this block forever when the final + // STOP flush arrived in ToolCallEnded with an empty call. + toolCall = {}; + currentCallValid = false; + break; + } - // Structural/stop markers must never reach the client, on any chunk, not just the final flush. + if (ready && currentState == State::Content) { + const auto anchoredStart = findAnchoredToolCallStart(streamingContent, streamingPosition); + size_t contentEnd = anchoredStart.value_or(std::string::npos); + const auto bareCallPos = findRecoverableBareCall(streamingContent, streamingPosition, allowedToolNames, enforceToolRegistry); + if (bareCallPos.has_value() && (contentEnd == std::string::npos || bareCallPos.value() < contentEnd)) + contentEnd = bareCallPos.value(); + if (contentEnd == std::string::npos && finishReason == ov::genai::GenerationFinishReason::NONE) { + // No complete boundary yet: hold a trailing fragment that may still + // grow into "<|tool_call>" or a line-start bare "call:" split + // across streamer chunks (e.g. "call" + ":" under DELAY_N_TOKENS). + // Emitting it now as prose would make the boundary unrecoverable. + const size_t tagHold = startTagHoldStart(streamingContent, streamingPosition); + const size_t bareHold = bareCallHoldStart(streamingContent, streamingPosition); + size_t holdStart = std::string::npos; + if (tagHold != std::string::npos) + holdStart = tagHold; + if (bareHold != std::string::npos && (holdStart == std::string::npos || bareHold < holdStart)) + holdStart = bareHold; + if (holdStart != std::string::npos) + contentEnd = holdStart; + } + std::string content = contentEnd == std::string::npos + ? streamingContent.substr(streamingPosition) + : streamingContent.substr(streamingPosition, contentEnd - streamingPosition); + streamingPosition += content.size(); for (const std::string& tagToErase : {TURN_END_TAG, TOOL_RESPONSE_START_TAG}) { size_t tagPos = content.find(tagToErase); while (tagPos != std::string::npos) { @@ -339,31 +898,36 @@ std::optional Gemma4ToolParser::parseChunk(const std::string& chunk, cons tagPos = content.find(tagToErase, tagPos); } } - return wrapDeltaContent(content); } - if (this->currentState == State::AfterToolCall) { - this->currentState = State::Content; - } - } - if (finishReason != ov::genai::GenerationFinishReason::NONE) { - // Unary/STOP flush can arrive after a chunk that only advanced one state - // (e.g. parsed the tool name but not yet the immediately following "}"). - // Give the state machine one last chance to consume already-buffered data - // before deciding whether an arguments delta exists. - if (this->currentState == State::ToolCallParameters) { - parseToolCallParametersState(); + if (currentState == State::AfterToolCall) { + currentState = State::Content; + continue; } - if ((this->currentState == State::ToolCallParameters || this->currentState == State::ToolCallEnded) && !this->toolCall.arguments.empty()) { - return wrapDeltaArgs(this->toolCall.arguments, toolCallIndex); + if (finishReason != ov::genai::GenerationFinishReason::NONE && currentState == State::ToolCallParameters) { + if (parseToolCallParametersState()) + continue; } - if (this->currentState == State::Content && this->streamingPosition < this->streamingContent.size()) { - auto content = this->streamingContent.substr(this->streamingPosition); - this->streamingPosition += content.size(); + if (ready || currentState != stateBefore || streamingPosition != positionBefore) + continue; + break; + } + if (finishReason != ov::genai::GenerationFinishReason::NONE) { + if (currentState == State::ToolCallParameters) + parseToolCallParametersState(); + if (currentState == State::ToolCallEnded && currentCallValid && !toolCall.arguments.empty()) { + auto delta = ToolCallDelta{++toolCallIndex, toolCall.id, toolCall.name, toolCall.arguments}; + toolCall = {}; + currentCallValid = false; + return delta; + } + if (currentState == State::Content && streamingPosition < streamingContent.size()) { + auto content = streamingContent.substr(streamingPosition); + streamingPosition += content.size(); for (const std::string& tagToErase : {TURN_END_TAG, TOOL_RESPONSE_START_TAG}) { size_t tagPos = content.find(tagToErase); while (tagPos != std::string::npos) { @@ -371,7 +935,6 @@ std::optional Gemma4ToolParser::parseChunk(const std::string& chunk, cons tagPos = content.find(tagToErase, tagPos); } } - return wrapDeltaContent(content); } } @@ -379,38 +942,4 @@ std::optional Gemma4ToolParser::parseChunk(const std::string& chunk, cons return std::nullopt; } -bool Gemma4ToolParser::parseSingleToolCall(const std::string& toolStr, ToolCall& toolCall) { - size_t argsPos = toolStr.find(TOOL_ARGS_START_INDICATOR); - if (argsPos != std::string::npos) { - std::string toolNameWithPrefix = toolStr.substr(0, argsPos); - if (toolNameWithPrefix.find(TOOL_CALL_NAME_PREFIX) != 0) { - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Tool name does not start with expected prefix '{}'. Tool string: {}", TOOL_CALL_NAME_PREFIX, toolStr); - return false; - } - std::string toolName = toolNameWithPrefix.substr(TOOL_CALL_NAME_PREFIX.length()); - trim(toolName); - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed tool name: {}", toolName); - - int argsStrLen = toolStr.length() - argsPos - TOOL_ARGS_START_INDICATOR.length() - TOOL_ARGS_END_INDICATOR.length(); - std::string argsStr = toolStr.substr(argsPos + TOOL_ARGS_START_INDICATOR.length(), argsStrLen); - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Parsed args string: {}", argsStr); - std::vector> arguments = parseArguments(argsStr); - - toolCall.name = toolName; - rapidjson::Document argsDoc(rapidjson::kObjectType); - rapidjson::StringBuffer sb; - rapidjson::Writer argsWriter(sb); - argsWriter.StartObject(); - for (const std::pair& argument : arguments) { - argsWriter.Key(argument.first.c_str()); - writeArgumentToWriter(argument.second, argsWriter); - } - argsWriter.EndObject(); - toolCall.arguments = sb.GetString(); - toolCall.id = generateRandomId(); - return true; - } - return false; -} - } // namespace ovms diff --git a/src/llm/io_processing/gemma4/gemma4_tool_parser.hpp b/src/llm/io_processing/gemma4/gemma4_tool_parser.hpp index 460c5af9c9..31d0b82046 100644 --- a/src/llm/io_processing/gemma4/gemma4_tool_parser.hpp +++ b/src/llm/io_processing/gemma4/gemma4_tool_parser.hpp @@ -6,47 +6,40 @@ // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. //***************************************************************************** #pragma once + +#include +#include #include -#include +#include #include +#include #include "src/llm/io_processing/base_output_parser.hpp" #include "src/port/rapidjson_stringbuffer.hpp" #include "src/port/rapidjson_writer.hpp" namespace ovms { + class Gemma4ToolParser : public BaseOutputParser { -protected: +public: + // Public protocol constants are also used by the private recursive parser + // implementation and conformance tests. They are semantic markers, not state. static const std::string TOOL_CALL_START_TAG; static const std::string TOOL_CALL_END_TAG; static const std::string TOOL_CALL_NAME_PREFIX; - - static const std::string TOOL_ARGS_START_INDICATOR; - static const std::string TOOL_ARGS_END_INDICATOR; static const std::string TOOL_ARGS_STRING_INDICATOR; - static const std::string TOOL_ARGS_SEPARATOR_STR; static const std::string TURN_END_TAG; static const std::string TOOL_RESPONSE_START_TAG; - static const int64_t botTokenId; - static const int64_t eotTokenId; - static const int64_t reasoningTokenId; - static const int64_t reasoningEndTokenId; - +protected: enum class State { - Content, // Content -> ToolCallStarted (on TOOL_CALL_START_TAG) - ToolCallStarted, // ToolCallStarted -> ToolCallParameters (on TOOL_ARGS_START_INDICATOR, emits name) - ToolCallParameters, // ToolCallParameters -> ToolCallEnded (on TOOL_ARGS_END_INDICATOR, emits args) - ToolCallEnded, // ToolCallEnded -> ToolCallStarted (on TOOL_CALL_NAME_PREFIX) | AfterToolCall (on end tag) - AfterToolCall // AfterToolCall -> Content + Content, + ToolCallStarted, + ToolCallParameters, + ToolCallEnded, + AfterToolCall }; public: @@ -56,8 +49,15 @@ class Gemma4ToolParser : public BaseOutputParser { OutputParsingConfig cfg; cfg.startTags = {"<|tool_call>"}; cfg.tokenIdStartTags = {"<|tool_call>"}; + // Bare `call:` is a tolerance/recovery form, not the canonical Google + // Gemma4 protocol. Canonical tool calls use <|tool_call>call:name{...}. + // The syntax-only constructor retains broad text markers for compatibility; + // the registry-aware production constructor narrows both canonical and bare + // text starts to request tools while keeping the special-token start separate. + cfg.preambleStartTags = {"call:"}; cfg.endTag = ""; cfg.needsSpecialTokens = true; + cfg.ownsToolCallBoundaries = true; return cfg; } @@ -66,27 +66,81 @@ class Gemma4ToolParser : public BaseOutputParser { BaseOutputParser(tokenizer, configOverride.has_value() ? std::move(*configOverride) : defaultParsingConfig()) {} + // Registry-aware production form. Text-phase detection requires the complete + // native prefix for a request tool, including the immediate argument opener. + // This prevents ordinary prose containing a literal <|tool_call> marker from + // entering tool phase. tokenIdStartTags intentionally remains <|tool_call> so + // OVMSTextStreamer can still make the special token visible before text-phase + // confirmation. Bare `call:` recovery is narrowed the same way. + Gemma4ToolParser(ov::genai::Tokenizer& tokenizer, + const ToolsSchemas_t& toolsSchemas, + std::optional configOverride = std::nullopt) : + BaseOutputParser(tokenizer, + configOverride.has_value() ? std::move(*configOverride) : defaultParsingConfig()) { + for (const auto& [name, schema] : toolsSchemas) { + (void)schema; + allowedToolNames.insert(name); + } + enforceToolRegistry = !allowedToolNames.empty(); + if (enforceToolRegistry && !configOverride.has_value()) { + parsingConfig.startTags.clear(); + parsingConfig.preambleStartTags.clear(); + parsingConfig.startTags.reserve(allowedToolNames.size() * 4); + parsingConfig.preambleStartTags.reserve(allowedToolNames.size() * 2); + for (const auto& name : allowedToolNames) { + const std::string bracePreamble = TOOL_CALL_NAME_PREFIX + name + "{"; + const std::string parenPreamble = TOOL_CALL_NAME_PREFIX + name + "("; + parsingConfig.startTags.push_back(TOOL_CALL_START_TAG + bracePreamble); + parsingConfig.startTags.push_back(TOOL_CALL_START_TAG + parenPreamble); + parsingConfig.startTags.push_back(TOOL_CALL_START_TAG + ":" + name + "{"); + parsingConfig.startTags.push_back(TOOL_CALL_START_TAG + ":" + name + "("); + parsingConfig.preambleStartTags.push_back(bracePreamble); + parsingConfig.preambleStartTags.push_back(parenPreamble); + } + } + } + + std::optional pendingToolFrameDiagnostic() const override { + if (currentState != State::ToolCallStarted && currentState != State::ToolCallParameters) + return std::nullopt; + const size_t start = currentState == State::ToolCallParameters && streamingPosition > 0 + ? streamingPosition - 1 : streamingPosition; + return PendingToolFrameDiagnostic{ + currentState == State::ToolCallParameters ? "ToolCallParameters" : "ToolCallStarted", + streamingContent.size() - std::min(start, streamingContent.size()), toolCall.name}; + } + void resetState() override { streamingContent.clear(); streamingPosition = 0; currentState = State::Content; toolCall = {}; toolCallIndex = -1; + currentArgsOpen = '{'; + currentArgsClose = '}'; + currentCallValid = true; + currentCallBare = false; + currentCallStartPos = 0; } std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; + // Compatibility helpers retained for existing unit tests/callers. They now use + // the same recursive native-value parser as the streaming path. static std::string normalizeArgStr(const std::string& arg); static std::string parseArrayParameter(const std::string& argumentStr); static std::string parseObjectParameter(const std::string& argumentStr); private: - void writeArgumentToWriter(const std::string& arg, rapidjson::Writer& writer); + friend struct Gemma4ToolParserTestAccess; + static std::optional parseNativeArgumentsBody(const std::string& argumentsBody); + static std::optional findMatchingContainerEnd(const std::string& text, size_t openPos, char openChar, char closeChar, size_t& malformedEndTag); + static std::string normalizeToolName(std::string rawName); - std::pair parseSingleArgument(const std::string& argumentStr); - std::vector> parseArguments(const std::string& argumentsStr); + bool toolNameAllowed(const std::string& name) const { + return !enforceToolRegistry || allowedToolNames.count(name) != 0; + } - bool parseSingleToolCall(const std::string& toolStr, ToolCall& toolCall); bool parseNewContent(); bool parseInContentState(); bool parseInToolCallState(); @@ -101,5 +155,16 @@ class Gemma4ToolParser : public BaseOutputParser { State currentState{State::Content}; ToolCall toolCall; int toolCallIndex{-1}; + char currentArgsOpen{'{'}; + char currentArgsClose{'}'}; + bool currentCallValid{true}; + // Whether the in-flight call started from a bare line-start `call:` + // (no "<|tool_call>" anchor) and where that start sits in + // streamingContent. An unknown tool name on a bare call must be rewound + // and re-emitted as ordinary content; an anchored unknown call stays dropped. + bool currentCallBare{false}; + size_t currentCallStartPos{0}; + bool enforceToolRegistry{false}; + std::unordered_set allowedToolNames; }; } // namespace ovms diff --git a/src/llm/io_processing/generation_config_builder.hpp b/src/llm/io_processing/generation_config_builder.hpp index 1be3e33f9d..d78bb9c6c2 100644 --- a/src/llm/io_processing/generation_config_builder.hpp +++ b/src/llm/io_processing/generation_config_builder.hpp @@ -15,11 +15,18 @@ //***************************************************************************** #pragma once -#include +#include +#include #include +#include +#include #include +#include +#include + #include #include + #include "base_generation_config_builder.hpp" #include "phi4/generation_config_builder.hpp" #include "llama3/generation_config_builder.hpp" @@ -29,20 +36,189 @@ #include "../../logging.hpp" namespace ovms { + +class Gemma4GenerationConfigBuilder : public BaseGenerationConfigBuilder { + enum class ToolConstraintMode { + Disabled, + Auto, + Hard, + }; + + bool hardToolChoice{false}; + + static bool isValidToolName(const std::string& name) { + return !name.empty() && std::all_of(name.begin(), name.end(), [](unsigned char c) { + return std::isalnum(c) || c == '_' || c == '-' || c == '.'; + }); + } + + static bool isNamedToolChoice(const std::string& toolChoice) { + return !toolChoice.empty() && toolChoice != "auto" && toolChoice != "none" && toolChoice != "required"; + } + + static bool isHardToolChoiceImpl(const std::string& toolChoice) { + return toolChoice == "required" || isNamedToolChoice(toolChoice); + } + + static ToolConstraintMode getToolConstraintMode(const OpenAIRequest& request) { + if (request.toolNameSchemaMap.empty() || request.toolChoice == "none") { + return ToolConstraintMode::Disabled; + } + if (request.toolChoice.empty() || request.toolChoice == "auto") { + return ToolConstraintMode::Auto; + } + return ToolConstraintMode::Hard; + } + + static ov::genai::StructuredOutputConfig::Tag buildToolTag(const std::string& toolName, const ToolSchemaWrapper& toolSchemaWrapper) { + if (toolSchemaWrapper.stringRepr.empty()) { + throw std::invalid_argument("Gemma4 guided tool schema for '" + toolName + "' is empty"); + } + ov::genai::StructuredOutputConfig::Tag tag; + tag.begin = "<|tool_call>call:" + toolName; + tag.content = ov::genai::StructuredOutputConfig::JSONSchema(toolSchemaWrapper.stringRepr, 2); + tag.end = ""; + return tag; + } + + static std::vector buildToolTags(const OpenAIRequest& request) { + std::vector tags; + if (isNamedToolChoice(request.toolChoice)) { + const auto it = request.toolNameSchemaMap.find(request.toolChoice); + if (it == request.toolNameSchemaMap.end()) { + throw std::invalid_argument("Gemma4 named tool_choice references an unavailable tool: " + request.toolChoice); + } + if (!isValidToolName(it->first)) { + throw std::invalid_argument("Gemma4 tool name contains unsupported characters: " + it->first); + } + tags.push_back(buildToolTag(it->first, it->second)); + return tags; + } + tags.reserve(request.toolNameSchemaMap.size()); + for (const auto& [toolName, toolSchemaWrapper] : request.toolNameSchemaMap) { + if (!isValidToolName(toolName)) { + throw std::invalid_argument("Gemma4 tool name contains unsupported characters: " + toolName); + } + tags.push_back(buildToolTag(toolName, toolSchemaWrapper)); + } + return tags; + } + + static ov::genai::StructuredOutputConfig::StructuralTag buildTriggeredToolGrammar( + std::vector toolTags, + bool parallelToolCalls, + bool atLeastOne) { + using Structured = ov::genai::StructuredOutputConfig; + // The fde0762 sole-tag duplication was a superseded workaround. Pinned + // GenAI/xgrammar v0.1.31 repeats one alternative when stop_after_first=false. + auto triggeredTags = std::make_shared(); + triggeredTags->triggers = {"<|tool_call>"}; + triggeredTags->tags = std::move(toolTags); + triggeredTags->at_least_one = atLeastOne; + // xgrammar's structural-tag contract maps parallel_tool_calls=false to + // stop_after_first=true. With the default true, later triggers remain legal. + triggeredTags->stop_after_first = !parallelToolCalls; + return triggeredTags; + } + + static ov::genai::StructuredOutputConfig::StructuralTag buildAutoToolGrammar( + std::vector toolTags, + bool parallelToolCalls) { + // TriggeredTags supplies the free-text prefix. `auto` deliberately does + // not require the trigger, so ordinary prose remains legal. + return buildTriggeredToolGrammar(std::move(toolTags), parallelToolCalls, false); + } + + static ov::genai::StructuredOutputConfig::StructuralTag buildMandatoryToolGrammar( + std::vector toolTags, + bool parallelToolCalls) { + using Structured = ov::genai::StructuredOutputConfig; + // Keep alternatives unique; stop_after_first alone controls multiplicity. + + auto requiredTags = std::make_shared(); + requiredTags->tags = std::move(toolTags); + requiredTags->separator = ""; + requiredTags->at_least_one = true; + requiredTags->stop_after_first = !parallelToolCalls; + + // On a normal new model turn, canonical Google Gemma4 can either call a + // tool immediately or emit a complete thought channel and then call it. + // Selecting a named tool restricts the available tags, not the thought + // phase. xgrammar rejects empty ConstString, so optional thought is a + // Union of tools-only versus thought-then-tools. + auto thought = std::make_shared(); + thought->begin = "<|channel>thought\n"; + thought->content = Structured::AnyText(); + thought->end = ""; + + auto thoughtThenTools = std::make_shared(); + thoughtThenTools->elements = {thought, requiredTags}; + + auto alternatives = std::make_shared(); + alternatives->elements = {requiredTags, thoughtThenTools}; + return alternatives; + } + +public: + Gemma4GenerationConfigBuilder() = delete; + explicit Gemma4GenerationConfigBuilder(const ov::genai::GenerationConfig& baseConfig, bool enableToolGuidedGeneration, DecodingMethod decodingMethod) : + BaseGenerationConfigBuilder(baseConfig, enableToolGuidedGeneration, decodingMethod) {} + + bool shouldPreserveStructuredOutputOnValidationFailure() const override { + return hardToolChoice; + } + + void parseConfigFromRequest(const OpenAIRequest& request) override { + BaseGenerationConfigBuilder::parseConfigFromRequest(request); + hardToolChoice = isHardToolChoiceImpl(request.toolChoice); + + if (hardToolChoice && request.toolNameSchemaMap.empty()) { + throw std::invalid_argument("Gemma4 hard tool_choice requires at least one available tool schema"); + } + if (request.responseFormat.has_value() && request.toolChoice != "none" && !request.toolNameSchemaMap.empty()) { + throw std::invalid_argument("Gemma4 response_format cannot be combined with active tool generation constraints"); + } + + const ToolConstraintMode mode = getToolConstraintMode(request); + if (mode == ToolConstraintMode::Disabled) { + return; + } + + auto toolTags = buildToolTags(request); + if (toolTags.empty()) { + throw std::invalid_argument("Gemma4 active tool_choice did not produce an enforceable tool tag"); + } + + switch (mode) { + case ToolConstraintMode::Auto: + // OpenVINO GenAI TriggeredTags maps to xgrammar's lazy structural-tag + // dispatch: normal text is unconstrained until the tool marker appears, + // then the selected request tool name and JSON schema become authoritative. + setStructuralTagsConfig(buildAutoToolGrammar(std::move(toolTags), request.parallelToolCalls)); + return; + case ToolConstraintMode::Hard: + setStructuralTagsConfig(buildMandatoryToolGrammar(std::move(toolTags), request.parallelToolCalls)); + return; + case ToolConstraintMode::Disabled: + return; + } + } +}; + class GenerationConfigBuilder { std::unique_ptr builder_impl; public: GenerationConfigBuilder() = delete; - // Using tool parser name to select appropriate builder implementation to avoid introducing additional parameters. Might be insufficient in the future. explicit GenerationConfigBuilder(const ov::genai::GenerationConfig& baseConfig, std::string toolParserName, bool enableToolGuidedGeneration, DecodingMethod decodingMethod) { if (toolParserName == "llama3") { builder_impl = std::make_unique(baseConfig, enableToolGuidedGeneration, decodingMethod); } else if (toolParserName == "qwen3") { - // Qwen3 and Hermes3 share the same mechanism for generating tool calls, so we can use Hermes3GenerationConfigBuilder builder_impl = std::make_unique(baseConfig, enableToolGuidedGeneration, decodingMethod); } else if (toolParserName == "hermes3") { builder_impl = std::make_unique(baseConfig, enableToolGuidedGeneration, decodingMethod); + } else if (toolParserName == "gemma4") { + builder_impl = std::make_unique(baseConfig, enableToolGuidedGeneration, decodingMethod); } else if (toolParserName == "phi4") { builder_impl = std::make_unique(baseConfig, enableToolGuidedGeneration, decodingMethod); } else if (toolParserName == "devstral") { @@ -55,19 +231,16 @@ class GenerationConfigBuilder { } } - ov::genai::GenerationConfig& getConfig() { - return builder_impl->getConfig(); - } - - void adjustConfigForDecodingMethod() { - builder_impl->adjustConfigForDecodingMethod(); - } - - void validateStructuredOutputConfig(ov::genai::Tokenizer& tokenizer) { - builder_impl->validateStructuredOutputConfig(tokenizer); - } + ov::genai::GenerationConfig& getConfig() { return builder_impl->getConfig(); } + void adjustConfigForDecodingMethod() { builder_impl->adjustConfigForDecodingMethod(); } + void validateStructuredOutputConfig(ov::genai::Tokenizer& tokenizer) { builder_impl->validateStructuredOutputConfig(tokenizer); } void unsetStructuredOutputConfig() { + if (builder_impl->shouldPreserveStructuredOutputOnValidationFailure()) { + SPDLOG_LOGGER_WARN(llm_calculator_logger, + "Refusing to clear structured output after validation failure for Gemma4 required/named tool_choice; keeping generation fail-closed."); + return; + } builder_impl->unsetStructuredOutputConfig(); } @@ -75,8 +248,7 @@ class GenerationConfigBuilder { builder_impl->parseConfigFromRequest(request); } - void addStopString(const std::string& decodedStopString) { - builder_impl->addStopString(decodedStopString); - } + bool hasHardToolChoice() const { return builder_impl->shouldPreserveStructuredOutputOnValidationFailure(); } + void addStopString(const std::string& decodedStopString) { builder_impl->addStopString(decodedStopString); } }; } // namespace ovms diff --git a/src/llm/io_processing/input_processors/chat_template_adapter.cpp b/src/llm/io_processing/input_processors/chat_template_adapter.cpp index 6719e54708..92e66f2c69 100644 --- a/src/llm/io_processing/input_processors/chat_template_adapter.cpp +++ b/src/llm/io_processing/input_processors/chat_template_adapter.cpp @@ -48,11 +48,9 @@ void funcArgsToObjectHistory(ov::genai::ChatHistory& chatHistory) { continue; } std::string argsStr = args.get_string(); - // Parse and replace string arguments with the parsed JSON object try { function["arguments"] = ov::genai::JsonContainer::from_json_string(argsStr); } catch (...) { - // If parsing fails, leave as-is SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Failed to parse function arguments as JSON: {}", argsStr); continue; } @@ -60,6 +58,29 @@ void funcArgsToObjectHistory(ov::genai::ChatHistory& chatHistory) { } } +void toolResponseJsonContentToObjectHistory(ov::genai::ChatHistory& chatHistory) { + for (size_t msgIdx = 0; msgIdx < chatHistory.size(); ++msgIdx) { + auto message = chatHistory[msgIdx]; + if (!message.contains("role") || !message["role"].is_string() || message["role"].get_string() != "tool") { + continue; + } + if (!message.contains("content") || !message["content"].is_string()) { + continue; + } + + const std::string content = message["content"].get_string(); + try { + auto parsed = ov::genai::JsonContainer::from_json_string(content); + if (!parsed.is_object()) { + continue; + } + message["content"] = parsed; + } catch (...) { + SPDLOG_LOGGER_TRACE(llm_calculator_logger, "Tool response content is not a JSON object; keeping string content"); + } + } +} + void injectReasoningIntoMissnamedSection(ov::genai::ChatHistory& chatHistory, const std::string& templateReasoningFieldName) { for (size_t msgIdx = 0; msgIdx < chatHistory.size(); ++msgIdx) { auto message = chatHistory[msgIdx]; @@ -96,6 +117,9 @@ void applyToHistory(const ChatTemplateCaps& caps, ov::genai::ChatHistory& chatHi if (caps.requiresObjectArguments) { funcArgsToObjectHistory(chatHistory); } + if (caps.parseToolResponseJsonContent) { + toolResponseJsonContentToObjectHistory(chatHistory); + } if (!caps.missnamedReasoningField.empty()) { injectReasoningIntoMissnamedSection(chatHistory, caps.missnamedReasoningField); } diff --git a/src/llm/io_processing/input_processors/chat_template_adapter.hpp b/src/llm/io_processing/input_processors/chat_template_adapter.hpp index d3cf68ee0c..99e4efa0d7 100644 --- a/src/llm/io_processing/input_processors/chat_template_adapter.hpp +++ b/src/llm/io_processing/input_processors/chat_template_adapter.hpp @@ -29,6 +29,11 @@ namespace chat_template_adapter { // Models like Gemma require arguments as a dict/object, not a stringified JSON. void funcArgsToObjectHistory(ov::genai::ChatHistory& chatHistory); +// Converts JSON-object strings in role:tool content to objects. Deliberately +// limited to objects: arrays, scalars and non-JSON strings keep OpenAI content +// semantics. Whether this is safe for a concrete template is decided by caps. +void toolResponseJsonContentToObjectHistory(ov::genai::ChatHistory& chatHistory); + // Apply all relevant adaptations to the ChatHistory based on detected capabilities. void applyToHistory(const ChatTemplateCaps& caps, ov::genai::ChatHistory& chatHistory); diff --git a/src/llm/io_processing/input_processors/chat_template_processor.cpp b/src/llm/io_processing/input_processors/chat_template_processor.cpp index 83a04b3837..3549be6397 100644 --- a/src/llm/io_processing/input_processors/chat_template_processor.cpp +++ b/src/llm/io_processing/input_processors/chat_template_processor.cpp @@ -16,6 +16,7 @@ #include "chat_template_processor.hpp" +#include #include #include #include @@ -25,6 +26,79 @@ namespace ovms { +namespace { + +bool promptEndsInOpenGemma4Reasoning(const std::string& renderedPrompt) { + static const std::string marker = "<|channel>thought"; + const size_t end = renderedPrompt.find_last_not_of(" \t\r\n"); + if (end == std::string::npos || end + 1 < marker.size()) { + return false; + } + return renderedPrompt.compare(end + 1 - marker.size(), marker.size(), marker) == 0; +} + +} // namespace + +bool adaptGemma4HardToolGrammarForRenderedPrompt( + ov::genai::GenerationConfig& config, + const std::string& renderedPrompt) { + using Structured = ov::genai::StructuredOutputConfig; + + if (!config.structured_output_config.has_value() || !promptEndsInOpenGemma4Reasoning(renderedPrompt)) { + return false; + } + auto& structuralConfig = config.structured_output_config->structural_tags_config; + if (!structuralConfig.has_value()) { + return false; + } + auto* root = std::get_if(&structuralConfig.value()); + if (root == nullptr) { + return false; + } + auto* alternativesHolder = std::get_if>(root); + if (alternativesHolder == nullptr || !*alternativesHolder || (*alternativesHolder)->elements.size() != 2) { + return false; + } + + auto& alternatives = **alternativesHolder; + auto* requiredHolder = std::get_if>(&alternatives.elements[0]); + auto* thoughtSequenceHolder = std::get_if>(&alternatives.elements[1]); + if (requiredHolder == nullptr || !*requiredHolder || thoughtSequenceHolder == nullptr || !*thoughtSequenceHolder) { + return false; + } + + const auto& requiredTags = **requiredHolder; + const auto& thoughtSequence = **thoughtSequenceHolder; + if (!requiredTags.at_least_one || requiredTags.tags.empty() || thoughtSequence.elements.size() != 2) { + return false; + } + auto* thoughtHolder = std::get_if>(&thoughtSequence.elements[0]); + auto* repeatedRequiredHolder = std::get_if>(&thoughtSequence.elements[1]); + if (thoughtHolder == nullptr || !*thoughtHolder || repeatedRequiredHolder == nullptr || !*repeatedRequiredHolder || + *repeatedRequiredHolder != *requiredHolder) { + return false; + } + + const auto& thought = **thoughtHolder; + if (thought.begin != "<|channel>thought\n" || thought.end != "") { + return false; + } + for (const auto& tag : requiredTags.tags) { + if (tag.begin.rfind("<|tool_call>call:", 0) != 0 || tag.end != "") { + return false; + } + } + + auto triggered = std::make_shared(); + triggered->triggers = {"<|tool_call>"}; + triggered->tags = requiredTags.tags; + triggered->at_least_one = true; + triggered->stop_after_first = requiredTags.stop_after_first; + Structured::StructuralTag adapted = triggered; + structuralConfig = adapted; + return true; +} + #if (PYTHON_DISABLE == 0) ChatTemplateProcessor::ChatTemplateProcessor(ov::genai::Tokenizer& tokenizer, PyJinjaTemplateProcessor& templateProcessor) : @@ -124,6 +198,22 @@ absl::Status ChatTemplateProcessor::process(InputRequest& req) { return absl::Status(absl::StatusCode::kInvalidArgument, "Final prompt after applying chat template is empty"); } + + // Current Google Gemma4 continues the same model turn after a tool response + // and, with thinking enabled, can leave the rendered prompt ending in an open + // thought channel. The hard grammar was built before rendering and therefore + // described a new-turn suffix. Reconcile it here, where prompt state is known. + if (adaptGemma4HardToolGrammarForRenderedPrompt(req.generationConfig, req.promptText)) { + try { + req.generationConfig.structured_output_config.value().validate(tokenizer); + } catch (const std::exception& e) { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, + "Gemma4 prompt-aware hard tool grammar validation failed: {}", e.what()); + return absl::Status(absl::StatusCode::kInvalidArgument, + std::string("Gemma4 prompt-aware hard tool grammar validation failed: ") + e.what()); + } + } + return absl::OkStatus(); } diff --git a/src/llm/io_processing/input_processors/chat_template_processor.hpp b/src/llm/io_processing/input_processors/chat_template_processor.hpp index 108a593887..79befb7257 100644 --- a/src/llm/io_processing/input_processors/chat_template_processor.hpp +++ b/src/llm/io_processing/input_processors/chat_template_processor.hpp @@ -19,6 +19,7 @@ #include #include +#include #include #include "../base_input_processor.hpp" @@ -29,6 +30,14 @@ namespace ovms { +// Reconciles a hard Gemma4 tool grammar with the prompt state produced by the +// canonical Google template after a tool response. Returns true only when the +// config is recognized as Gemma4's hard tools-or-thought-then-tools grammar and +// the rendered prompt ends inside an already-open thought channel. +bool adaptGemma4HardToolGrammarForRenderedPrompt( + ov::genai::GenerationConfig& config, + const std::string& renderedPrompt); + // Applies the chat template to ChatHistory, producing req.promptText. // Active when: input is ChatHistory variant (CHAT_COMPLETIONS and RESPONSES). // diff --git a/src/llm/io_processing/output_parser.cpp b/src/llm/io_processing/output_parser.cpp index 213ea51b10..4a8c0a299b 100644 --- a/src/llm/io_processing/output_parser.cpp +++ b/src/llm/io_processing/output_parser.cpp @@ -47,24 +47,12 @@ OutputParser::TagLookupStatus OutputParser::StreamOutputCache::lookupTag(const s return TagLookupStatus::NOT_FOUND; } if (tag.size() > buffer.size()) { - /* - If the tag is longer than the buffer, we check if the buffer and tag overlap (either partially or fully for exact match) - They do overlap, we assume that tag may appear in the future, so we return FOUND_INCOMPLETE - otherwise we return NOT_FOUND - */ if (stringsOverlap(buffer, tag)) { return TagLookupStatus::FOUND_INCOMPLETE; } else { return TagLookupStatus::NOT_FOUND; } } else if (tag.size() < buffer.size()) { - /* - If the tag is shorter than the buffer, we check: - a) if the tag is a substring of the buffer (tag is fully matched) - b) if the buffer and tag overlap (part of the tag is matched) - in the first case we return FOUND_COMPLETE, in the second FOUND_INCOMPLETE - otherwise we return NOT_FOUND - */ if (buffer.find(tag) != std::string::npos) { return TagLookupStatus::FOUND_COMPLETE; } else if (stringsOverlap(buffer, tag)) { @@ -73,13 +61,6 @@ OutputParser::TagLookupStatus OutputParser::StreamOutputCache::lookupTag(const s return TagLookupStatus::NOT_FOUND; } } else { - /* - If the tag and buffer are of the same length, we check: - a) if they are equal (tag is fully matched) - b) if they overlap (part of the tag is matched) - in the first case we return FOUND_COMPLETE, in the second FOUND_INCOMPLETE - otherwise we return NOT_FOUND - */ if (buffer == tag) { return TagLookupStatus::FOUND_COMPLETE; } else if (stringsOverlap(buffer, tag)) { @@ -91,7 +72,6 @@ OutputParser::TagLookupStatus OutputParser::StreamOutputCache::lookupTag(const s } OutputParser::TagLookupStatus OutputParser::StreamOutputCache::lookupTags(const std::vector& tags) const { - // We look for multiple tags and return the status in the following priority: FOUND COMPLETE > FOUND_INCOMPLETE > NOT_FOUND TagLookupStatus finalTagLookupStatus = TagLookupStatus::NOT_FOUND; for (const auto& tag : tags) { auto tagLookupStatus = lookupTag(tag); @@ -120,10 +100,9 @@ const std::string& OutputParser::StreamOutputCache::getBuffer() const { std::optional OutputParser::parseContentChunk(ProcessingPhase newPhase) { auto result = contentParser->parseChunk(streamOutputCache.getBuffer(), {}, ov::genai::GenerationFinishReason::NONE); if (!result.has_value()) - return std::nullopt; // hold — keep buffer + return std::nullopt; streamOutputCache.clear(); processingPhase = newPhase; - // Suppress preamble-only ContentDelta (empty text = structural tag consumed, nothing to emit). if (const auto* cd = std::get_if(&*result)) { if (cd->text.empty()) return std::nullopt; @@ -135,7 +114,18 @@ std::optional OutputParser::parseToolCallChunk(const std::vector if (!toolParser) { throw std::runtime_error("Tool parser is not available, cannot parse tool call chunk"); } - // Bytes after the end tag belong to the next phase — preserve them before clearing. + if (toolParser->getParsingConfig().ownsToolCallBoundaries) { + const bool bareRecovery = streamOutputCache.getBuffer().rfind("call:", 0) == 0 && + finishReason == ov::genai::GenerationFinishReason::NONE; + auto result = toolParser->parseChunk(streamOutputCache.getBuffer(), tokens, finishReason); + streamOutputCache.clear(); + processingPhase = TOOL_CALLS_PROCESSING_TOOL; + if (bareRecovery && result.has_value() && std::holds_alternative(*result)) { + pendingDelta = std::move(result); + return std::nullopt; + } + return result; + } std::string remainder; const std::string& endTag = toolParser->getParsingConfig().endTag; if (!endTag.empty()) { @@ -162,7 +152,6 @@ std::optional OutputParser::parseReasoningChunk(const std::vectorgetParsingConfig().endTag; if (!endTag.empty()) { @@ -204,7 +193,7 @@ OutputParser::OutputParser(ov::genai::Tokenizer& tokenizer, const std::string to } else if (toolParserName == "lfm2") { toolParser = std::make_unique(tokenizer); } else if (toolParserName == "gemma4") { - toolParser = std::make_unique(tokenizer); + toolParser = std::make_unique(tokenizer, toolNameSchemaMap); } else if (toolParserName == "onyx") { toolParser = std::make_unique(tokenizer, toolNameSchemaMap); } else if (toolParserName == "minicpm5") { @@ -231,11 +220,6 @@ OutputParser::OutputParser(ov::genai::Tokenizer& tokenizer, const std::string to "\". Supported reasoning parsers are: " + getSupportedReasoningParserNamesAsString()); } - // Model/output formats whose structural tokens must stay visible in the content/unknown phase - // (e.g. GptOss uses <|channel|>... throughout the stream; devstral's [TOOL_CALLS] tag and - // minicpm5's /<|im_end|> must be visible before parser-owned phases begin). For all other - // parser combinations the content phase decodes with skip_special_tokens=true (the default, - // lower noise). Each parser that requires this sets defaultDecodingWithSpecialTokens in its config. if (toolParserName == "onyx" || reasoningParserName == "onyx") contentParser = std::make_unique(tokenizer); else if (toolParserName == "gptoss" || reasoningParserName == "gptoss") @@ -246,8 +230,6 @@ OutputParser::OutputParser(ov::genai::Tokenizer& tokenizer, const std::string to "<|end|>", "<|return|>"}); else if (toolParserName == "gemma4") - // "<|channel>thought\n"/"" guard against a reasoning re-entry mid-CONTENT - // (e.g. an empty "ghost" thought channel) leaking into visible content. contentParser = std::make_unique(tokenizer, std::vector{"", "<|tool_response>", "<|channel>thought\n", ""}); else if (toolParserName == "lfm2") contentParser = std::make_unique(tokenizer, std::vector{"<|im_end|>"}); @@ -289,6 +271,7 @@ std::string OutputParser::getToolParserStartTag() const { void OutputParser::resetStreamingState() { processingPhase = UNKNOWN; streamOutputCache.clear(); + pendingDelta.reset(); if (toolParser) toolParser->resetState(); if (reasoningParser) @@ -301,8 +284,6 @@ void OutputParser::resetStreamingState() { } bool OutputParser::needSpecialTokensForCurrentDecode(bool userWantsSpecialTokens) const { - // Content / unknown phase: use the computed baseline for this parser combination; - // also honour user preference here (scoped to content — does not override parser phases). if (processingPhase == CONTENT || processingPhase == UNKNOWN) { return defaultDecodingWithSpecialTokens || userWantsSpecialTokens; } @@ -316,8 +297,6 @@ bool OutputParser::needSpecialTokensForCurrentDecode(bool userWantsSpecialTokens } std::string OutputParser::getPhaseStartTagForToken(int64_t tokenId, bool toolsAvailable) const { - // The guard conditions mirror isPhaseStartToken: don't re-fire for a phase we are - // already in (the parser's own text-based detection handles re-entry there). if (toolParser && toolsAvailable) { const auto& tokenMap = toolParser->getResolvedStartTokenToTag(); auto it = tokenMap.find(tokenId); @@ -343,9 +322,6 @@ void OutputParser::setImplicitReasoningStart(bool value) { return; } reasoningParser->setImplicitStart(value); - // Bias the streaming state machine: the model output is expected to begin already - // inside the reasoning segment, so skip the UNKNOWN phase and go straight to REASONING. - // When value is false, restore the default initial phase. if (processingPhase == UNKNOWN || processingPhase == REASONING) { processingPhase = value ? REASONING : UNKNOWN; } @@ -358,26 +334,32 @@ void OutputParser::detectAndSetImplicitReasoningStart(const std::string& rendere std::string trimmed = renderedPrompt; rtrim(trimmed); const auto& startTags = reasoningParser->getParsingConfig().startTags; - bool detected = std::any_of(startTags.begin(), startTags.end(), - [&](const std::string& tag) { return !tag.empty() && endsWith(trimmed, tag); }); + // Tags may end with whitespace (e.g. Gemma4 "<|channel>thought\n"). + // Since the prompt is rtrimmed, compare against an rtrimmed tag copy so + // a prompt ending inside the thought channel is still detected. + bool detected = std::any_of(startTags.begin(), startTags.end(), [&](const std::string& tag) { + if (tag.empty()) { + return false; + } + std::string trimmedTag = tag; + rtrim(trimmedTag); + return !trimmedTag.empty() && endsWith(trimmed, trimmedTag); + }); setImplicitReasoningStart(detected); return; } std::optional OutputParser::parseChunk(const std::string& chunkResponse, const std::vector& tokens, const bool toolsAvailable, ov::genai::GenerationFinishReason finishReason) { - /* - Using appropriate parser based on the current processing phase - Call to this method should return either result from parserContentChunk, parseToolCallChunk, parseReasoningChunk when we can determine the phase - or std::nullopt when we are waiting for more chunks to determine if we should switch phase or not. - Note that mentioned methods do not take chunk as argument, they read it from streamOutputCache and are responsible for clearing the cache, - so only use those methods or return nullopt. - */ - bool reasoningParserExistsAndSupportsStreaming = reasoningParser && !reasoningParser->getParsingConfig().startTags.empty() && !reasoningParser->getParsingConfig().endTag.empty(); bool toolParserExistsAndSupportsStreaming = toolParser && !toolParser->getParsingConfig().startTags.empty(); bool applyToolParser = toolParserExistsAndSupportsStreaming && toolsAvailable; streamOutputCache.add(chunkResponse); + if (pendingDelta.has_value()) { + auto result = std::move(pendingDelta); + pendingDelta.reset(); + return result; + } if (llm_calculator_logger->should_log(spdlog::level::trace)) { std::string tokenIds; @@ -387,100 +369,130 @@ std::optional OutputParser::parseChunk(const std::string& chunkResponse, tokenIds += ", "; tokenIds += std::to_string(tokens[i]); } - std::string processingPhaseStr; switch (processingPhase) { - case UNKNOWN: - processingPhaseStr = "UNKNOWN"; - break; - case CONTENT: - processingPhaseStr = "CONTENT"; - break; - case REASONING: - processingPhaseStr = "REASONING"; - break; - case TOOL_CALLS_PROCESSING_TOOL: - processingPhaseStr = "TOOL_CALLS_PROCESSING_TOOL"; - break; - case TOOL_CALLS_WAITING_FOR_TOOL: - processingPhaseStr = "TOOL_CALLS_WAITING_FOR_TOOL"; - break; - default: - processingPhaseStr = "UNKNOWN"; - break; + case UNKNOWN: processingPhaseStr = "UNKNOWN"; break; + case CONTENT: processingPhaseStr = "CONTENT"; break; + case REASONING: processingPhaseStr = "REASONING"; break; + case TOOL_CALLS_PROCESSING_TOOL: processingPhaseStr = "TOOL_CALLS_PROCESSING_TOOL"; break; + case TOOL_CALLS_WAITING_FOR_TOOL: processingPhaseStr = "TOOL_CALLS_WAITING_FOR_TOOL"; break; + default: processingPhaseStr = "UNKNOWN"; break; } - SPDLOG_LOGGER_TRACE(llm_calculator_logger, "OutputParser::parseChunk[PROCESSING_PHASE={}] called with {} tokens, text=\"{}\", finish_reason={}, token IDs=[{}]", processingPhaseStr, tokens.size(), chunkResponse, static_cast(finishReason), tokenIds); } if (processingPhase == UNKNOWN) { - // If we are in the UNKNOWN phase, we need to determine if we should switch to CONTENT, REASONING, or TOOL_CALLS phase. TagLookupStatus anyStartTagStatus = TagLookupStatus::NOT_FOUND; if (reasoningParserExistsAndSupportsStreaming) { - // Check if reasoning start tag has been received TagLookupStatus reasoningStartTagStatus = streamOutputCache.lookupTags(reasoningParser->getParsingConfig().startTags); if (reasoningStartTagStatus == TagLookupStatus::NOT_FOUND) { - // If reasoning start tag is not found, check if any of the special start tags are found reasoningStartTagStatus = streamOutputCache.lookupTags(reasoningParser->getParsingConfig().preambleStartTags); } if (reasoningStartTagStatus == TagLookupStatus::FOUND_COMPLETE) { return parseReasoningChunk(tokens, finishReason); - } // else startTagStatus is FOUND_INCOMPLETE or NOT_FOUND, we continue processing, so potential tool parser start tag is not missed + } anyStartTagStatus = reasoningStartTagStatus; } if (applyToolParser) { - // Check if tool call start tag has been received TagLookupStatus toolCallStartTagStatus = streamOutputCache.lookupTags(toolParser->getParsingConfig().startTags); if (toolCallStartTagStatus == TagLookupStatus::NOT_FOUND) { - // If tool call start tag is not found, check if any of the special start tags are found toolCallStartTagStatus = streamOutputCache.lookupTags(toolParser->getParsingConfig().preambleStartTags); } if (toolCallStartTagStatus == TagLookupStatus::FOUND_COMPLETE) { return parseToolCallChunk(tokens, finishReason); - } // else startTagStatus is FOUND_INCOMPLETE or NOT_FOUND, we continue processing + } if (toolCallStartTagStatus == TagLookupStatus::FOUND_INCOMPLETE) { - anyStartTagStatus = toolCallStartTagStatus; // We have at least one incomplete start tag + anyStartTagStatus = toolCallStartTagStatus; } } if ((!reasoningParserExistsAndSupportsStreaming && !applyToolParser) || finishReason != ov::genai::GenerationFinishReason::NONE || anyStartTagStatus == TagLookupStatus::NOT_FOUND) { - // If no special parsers are available, generation has finished or we have no start tags we just return content chunks and switch to CONTENT phase. return parseContentChunk(); } - // If we are here, it means we have incomplete start tag for either reasoning or tool parser, so we wait for more chunks return std::nullopt; } else if (processingPhase == REASONING) { - // If we are in the REASONING phase, we check if parsing end tag is found and if so, switch to UNKNOWN phase. - TagLookupStatus endTagStatus = streamOutputCache.lookupTag(reasoningParser->getParsingConfig().endTag); + const auto& reasoningConfig = reasoningParser->getParsingConfig(); + TagLookupStatus endTagStatus = streamOutputCache.lookupTag(reasoningConfig.endTag); + + if (reasoningConfig.toolStartTerminatesReasoning && applyToolParser) { + const auto& toolStartTags = toolParser->getParsingConfig().startTags; + TagLookupStatus toolStartTagStatus = streamOutputCache.lookupTags(toolStartTags); + + if (toolStartTagStatus == TagLookupStatus::FOUND_COMPLETE) { + const std::string& buf = streamOutputCache.getBuffer(); + size_t toolStartPos = std::string::npos; + for (const auto& tag : toolStartTags) { + const size_t pos = buf.find(tag); + if (pos != std::string::npos && (toolStartPos == std::string::npos || pos < toolStartPos)) { + toolStartPos = pos; + } + } + const size_t reasoningEndPos = reasoningConfig.endTag.empty() ? std::string::npos : buf.find(reasoningConfig.endTag); + + // Canonical Google Gemma4 closes the thought channel with + // before <|tool_call>. A parser may opt into toolStartTerminatesReasoning + // only as a tolerance/recovery boundary for malformed or edge output. + // Take over only when that recovery opener precedes an explicit closer. + if (toolStartPos != std::string::npos && + (reasoningEndPos == std::string::npos || toolStartPos < reasoningEndPos)) { + const std::string reasoningPrefix = buf.substr(0, toolStartPos); + const std::string toolRemainder = buf.substr(toolStartPos); + streamOutputCache.clear(); + processingPhase = TOOL_CALLS_PROCESSING_TOOL; + streamOutputCache.add(toolRemainder); + + if (!reasoningPrefix.empty()) { + auto reasoningDelta = reasoningParser->parseChunk(reasoningPrefix, tokens, finishReason); + if (reasoningDelta.has_value()) { + return reasoningDelta; + } + } + return parseToolCallChunk(tokens, finishReason); + } + } + + // Hold back a partial tool opener at the tail instead of leaking its + // bytes into reasoning. This mirrors the boundary holdback used by + // the dedicated Gemma4 parsers in other runtimes. + if (toolStartTagStatus == TagLookupStatus::FOUND_INCOMPLETE && + endTagStatus != TagLookupStatus::FOUND_COMPLETE && + finishReason == ov::genai::GenerationFinishReason::NONE) { + return std::nullopt; + } + } + if (endTagStatus == TagLookupStatus::FOUND_COMPLETE) { - // Switch back to UNKNOWN phase (we can have either CONTENT or TOOL_CALLS next) return parseReasoningChunk(tokens, finishReason, UNKNOWN); } else if (endTagStatus == TagLookupStatus::FOUND_INCOMPLETE && finishReason == ov::genai::GenerationFinishReason::NONE) { - return std::nullopt; // Wait for more chunks to determine if end tag is complete + return std::nullopt; } return parseReasoningChunk(tokens, finishReason); } else if (processingPhase == CONTENT) { - // If we are in the CONTENT phase, we check if tool parser start tag is found and if so, switch to TOOL_CALLS phase. - // TOOL_CALLS is the only phase that can be processed after CONTENT. if (applyToolParser) { + // Gemma4 owns tool-call/content boundaries after ordinary prose has + // started. The generic content parser cannot recover split line-start + // `call:` or strip Gemma4 turn/tool-response markers without losing + // byte ownership, so route through the owning parser once requested. + if (toolParser->getParsingConfig().ownsToolCallBoundaries) + return parseToolCallChunk(tokens, finishReason); TagLookupStatus toolStartTagStatus = streamOutputCache.lookupTags(toolParser->getParsingConfig().startTags); if (toolStartTagStatus == TagLookupStatus::FOUND_COMPLETE) { return parseToolCallChunk(tokens, finishReason); } else if (toolStartTagStatus == TagLookupStatus::FOUND_INCOMPLETE && finishReason == ov::genai::GenerationFinishReason::NONE) { - return std::nullopt; // Wait for more chunks to determine if end tag is complete + return std::nullopt; } return parseContentChunk(); } return parseContentChunk(); } else if (processingPhase == TOOL_CALLS_PROCESSING_TOOL) { - // Active tool call: accumulate until the end tag, then transition to WAITING_FOR_TOOL - // to determine whether another tool call or a content turn follows. + if (toolParser->getParsingConfig().ownsToolCallBoundaries) + return parseToolCallChunk(tokens, finishReason); TagLookupStatus toolEndTagStatus = streamOutputCache.lookupTag(toolParser->getParsingConfig().endTag); if (toolEndTagStatus == TagLookupStatus::FOUND_INCOMPLETE && finishReason == ov::genai::GenerationFinishReason::NONE) { - return std::nullopt; // Wait for more chunks to determine if end tag is complete + return std::nullopt; } if (toolEndTagStatus == TagLookupStatus::FOUND_COMPLETE) { return parseToolCallChunk(tokens, finishReason, TOOL_CALLS_WAITING_FOR_TOOL); diff --git a/src/llm/io_processing/output_parser.hpp b/src/llm/io_processing/output_parser.hpp index 47d4338f21..503ed80fdb 100644 --- a/src/llm/io_processing/output_parser.hpp +++ b/src/llm/io_processing/output_parser.hpp @@ -48,8 +48,11 @@ namespace ovms { // - Implement parseChunk() to process the text it receives during its active phase. // The parser may maintain arbitrary internal state and buffers to satisfy its own // format requirements; OutputParser does not inspect or constrain that state. -// - Return a JSON delta (OpenAI streaming format) or nullopt to signal "nothing to +// - Return a typed Delta event (ContentDelta/ReasoningDelta/ToolCallDelta/ +// FinishDelta/AudioDelta, see delta.hpp) or nullopt to signal "nothing to // emit yet"; the orchestrator propagates that decision upstream unchanged. +// OpenAI wire format is produced downstream by the Chat/Responses emitters, +// never here — parsers speak Delta, not model or wire protocol. // // Design invariant: OutputParser must contain NO logic specific to any individual model // format. All format-specific behaviour must be encapsulated in the parser subclasses @@ -91,6 +94,7 @@ class OutputParser { // Streaming related members ProcessingPhase processingPhase = UNKNOWN; StreamOutputCache streamOutputCache; + std::optional pendingDelta; bool implicitReasoningStart = false; // Baseline decode mode for content/unknown phases — true when the model/output format @@ -115,6 +119,9 @@ class OutputParser { public: OutputParser() = delete; explicit OutputParser(ov::genai::Tokenizer& tokenizer, const std::string toolParserName, const std::string reasoningParserName, const ToolsSchemas_t& toolNameSchemaMap); + std::optional pendingToolFrameDiagnostic() const { + return toolParser ? toolParser->pendingToolFrameDiagnostic() : std::nullopt; + } bool isToolParserAvailable() const; bool isReasoningParserAvailable() const; @@ -129,7 +136,7 @@ class OutputParser { // Parse one decoded chunk in streaming mode. // // Contract: - // - Returns a JSON delta conforming to the OpenAI streaming API, or nullopt when no + // - Returns a typed Delta event (see delta.hpp), or nullopt when no // output can yet be produced (partial tag match, preamble stripping, etc.). // - Processes AT MOST ONE phase per call. If a chunk spans a phase boundary (e.g. a // token whose text contains both an end tag and the start of the next phase), the bytes diff --git a/src/llm/io_processing/output_parsing_config.hpp b/src/llm/io_processing/output_parsing_config.hpp index bc1dd99ffa..26dd20cf6b 100644 --- a/src/llm/io_processing/output_parsing_config.hpp +++ b/src/llm/io_processing/output_parsing_config.hpp @@ -53,12 +53,12 @@ namespace ovms { // Whether the content/unknown phase also needs special tokens is determined at the // OutputParser level via defaultDecodingWithSpecialTokens, not in the per-parser config. // -// Content/unknown phase decode mode: -// defaultDecodingWithSpecialTokens — when true, decoding uses skip_special_tokens=false -// even in the content/unknown phase. Set by parsers -// whose model format emits structural special tokens -// before their own active phase begins (e.g. GptOss, -// devstral, minicpm5). +// Cross-parser transition: +// toolStartTerminatesReasoning — while this reasoning parser is active, a tool parser +// start tag is also an implicit reasoning end. Some model +// protocols (notably Gemma4) allow a tool call to begin +// directly from the thought channel without first emitting +// the ordinary reasoning end tag. struct OutputParsingConfig { std::vector startTags; std::vector tokenIdStartTags; @@ -67,8 +67,13 @@ struct OutputParsingConfig { std::vector stringsToErase; bool needsSpecialTokens = false; + // The tool parser owns quoted end markers and subsequent call framing. + // The generic router must not split or replay its input at a raw endTag. + bool ownsToolCallBoundaries = false; // See comment block above. bool defaultDecodingWithSpecialTokens = false; + // See cross-parser transition comment above. + bool toolStartTerminatesReasoning = false; }; } // namespace ovms diff --git a/src/llm/ovms_text_streamer.cpp b/src/llm/ovms_text_streamer.cpp index 5764c6e290..f65d486c1b 100644 --- a/src/llm/ovms_text_streamer.cpp +++ b/src/llm/ovms_text_streamer.cpp @@ -73,6 +73,32 @@ void OVMSTextStreamer::apply_decode_params(bool decode_special_tokens) { } std::optional OVMSTextStreamer::handle_decoding_params_change(int64_t token) { + // Reconcile the decode mode for the phase that was established by the + // previously flushed chunk before inspecting the current token. This matters + // at a reasoning/tool handoff: the previous phase may require visible special + // tokens while UNKNOWN/content returns to the user's skip_special_tokens=true + // baseline. The current token itself may already be the next phase opener. + if (m_output_parser) { + bool decode_with_special_tokens = m_output_parser->needSpecialTokensForCurrentDecode(m_user_wants_special); + if (decode_with_special_tokens != m_decode_special_tokens) { + if (!m_tokens_cache.empty()) { + const std::string text = m_tokenizer.decode(m_tokens_cache, m_additional_detokenization_params); + if (text.size() > m_printed_len) { + const auto s = flush_chunk(text, text.size(), ov::genai::GenerationFinishReason::NONE); + if (s != ov::genai::StreamingStatus::RUNNING) + return s; + } + } + m_tokens_cache.clear(); + m_decoded_lengths.clear(); + m_printed_len = 0; + // Flushing can itself complete a parser phase. Re-read the desired + // mode so the setting reflects the phase that will consume `token`. + decode_with_special_tokens = m_output_parser->needSpecialTokensForCurrentDecode(m_user_wants_special); + apply_decode_params(decode_with_special_tokens); + } + } + if (m_output_parser && !m_decode_special_tokens) { const std::string startTag = m_output_parser->getPhaseStartTagForToken(token, m_tools_available); if (!startTag.empty()) { @@ -102,28 +128,11 @@ std::optional OVMSTextStreamer::handle_decoding_para } } - if (m_output_parser) { - const bool decode_with_special_tokens = m_output_parser->needSpecialTokensForCurrentDecode(m_user_wants_special); - if (decode_with_special_tokens != m_decode_special_tokens) { - if (!m_tokens_cache.empty()) { - const std::string text = m_tokenizer.decode(m_tokens_cache, m_additional_detokenization_params); - if (text.size() > m_printed_len) { - const auto s = flush_chunk(text, text.size(), ov::genai::GenerationFinishReason::NONE); - if (s != ov::genai::StreamingStatus::RUNNING) - return s; - } - } - m_tokens_cache.clear(); - m_decoded_lengths.clear(); - m_printed_len = 0; - apply_decode_params(decode_with_special_tokens); - } - } - return std::nullopt; } ov::genai::StreamingStatus OVMSTextStreamer::write(int64_t token) { + ++m_generated_tokens; if (llm_calculator_logger->should_log(spdlog::level::trace)) m_all_tokens.push_back(token); @@ -191,6 +200,10 @@ ov::genai::StreamingStatus OVMSTextStreamer::write(int64_t token, bool immediate } void OVMSTextStreamer::end() { + end(ov::genai::GenerationFinishReason::STOP); +} + +void OVMSTextStreamer::end(ov::genai::GenerationFinishReason finish_reason) { if (llm_calculator_logger->should_log(spdlog::level::trace) && !m_all_tokens.empty()) { const ov::AnyMap no_skip_params{{ov::genai::skip_special_tokens.name(), false}}; const std::string full_decode = m_tokenizer.decode(m_all_tokens, no_skip_params); @@ -233,21 +246,21 @@ void OVMSTextStreamer::end() { for (const int64_t token : unprinted) { const auto status = write(token, /*immediate_flush=*/true); if (status != ov::genai::StreamingStatus::RUNNING) { - break; // cancelled mid-drain; still deliver the STOP signal below + break; // cancelled mid-drain; still deliver the terminal reason below } } - // Always deliver the STOP signal so parsers that rely on finishReason==STOP - // for cleanup receive it (e.g. hasPendingState flush in Lfm2ToolParser, - // argument string finalisation in Hermes3ToolParser). + // Deliver the actual terminal reason. The legacy no-argument end() retains STOP. const std::string final_text = m_tokens_cache.empty() ? std::string{} : m_tokenizer.decode(m_tokens_cache, m_additional_detokenization_params); - flush_chunk(final_text, m_printed_len, ov::genai::GenerationFinishReason::STOP); + flush_chunk(final_text, m_printed_len, finish_reason); m_tokens_cache.clear(); m_decoded_lengths.clear(); m_printed_len = 0; + m_generated_tokens = 0; + m_all_tokens.clear(); } // ----------------------------------------------------------------------------- @@ -295,6 +308,16 @@ ov::genai::StreamingStatus OVMSTextStreamer::flush_chunk( } const bool isLast = (finish_reason != ov::genai::GenerationFinishReason::NONE); + if (isLast && m_output_parser) { + if (const auto pending = m_output_parser->pendingToolFrameDiagnostic()) { + SPDLOG_LOGGER_WARN(llm_calculator_logger, + "Incomplete tool frame: parser_phase={} finish_reason={} pending_tool_frame=true buffered_bytes={} generated_tokens={} tool_name={}", + pending->phase, + finish_reason == ov::genai::GenerationFinishReason::LENGTH ? "LENGTH" : + finish_reason == ov::genai::GenerationFinishReason::TOOL_CALL ? "TOOL_CALL" : "STOP", + pending->bufferedBytes, m_generated_tokens, pending->toolName); + } + } if (delta.has_value()) { return m_callback(std::move(*delta), isLast); } diff --git a/src/llm/ovms_text_streamer.hpp b/src/llm/ovms_text_streamer.hpp index a41be270bb..ab966c5e41 100644 --- a/src/llm/ovms_text_streamer.hpp +++ b/src/llm/ovms_text_streamer.hpp @@ -35,7 +35,8 @@ namespace ovms { // Guarantees provided by OVMSTextStreamer: // - Ordered delivery: tokens are passed to OutputParser in the exact generation order, // one logical chunk at a time. -// - Final flush: end() ALWAYS calls parseChunk("", [], finishReason=STOP) after all tokens +// - Final flush: end(reason) calls parseChunk with the terminal reason after all tokens; +// the no-argument end() preserves STOP. This is done after all tokens // have been processed. This is the "at least one subsequent call after every phase // transition" guarantee that OutputParser depends on to drain buffered remainders. // - Phase-aware decode mode: after every write(), the streamer queries @@ -83,6 +84,7 @@ class OVMSTextStreamer : public ov::genai::TextStreamer { ov::genai::StreamingStatus write(int64_t token) override; ov::genai::StreamingStatus write(const std::vector& tokens) override; void end() override; + void end(ov::genai::GenerationFinishReason finish_reason); private: std::shared_ptr m_output_parser; @@ -111,6 +113,7 @@ class OVMSTextStreamer : public ov::genai::TextStreamer { // All token IDs received by write() in order, used for end() trace logging. std::vector m_all_tokens; + size_t m_generated_tokens = 0; }; } // namespace ovms diff --git a/src/llm/servable.cpp b/src/llm/servable.cpp index 8e0f35a094..2e1bd37f99 100644 --- a/src/llm/servable.cpp +++ b/src/llm/servable.cpp @@ -13,12 +13,28 @@ // See the License for the specific language governing permissions and // limitations under the License. //***************************************************************************** +#include +#include +#include +#include +#include +#include +#include +#include #include +#include +#include #include #include +#include +#include #include #include +#ifdef _WIN32 +#include +#endif + #pragma warning(push) #pragma warning(disable : 4005 4309 6001 6385 6386 6326 6011 4005 4456 6246 6313) #pragma GCC diagnostic push @@ -26,6 +42,8 @@ #include "mediapipe/framework/calculator_graph.h" #include #include +#include +#include #pragma GCC diagnostic pop #pragma warning(pop) @@ -44,6 +62,440 @@ #include "../tokenize/tokenize_parser.hpp" namespace ovms { +namespace { +void finishTextStreamer(const std::shared_ptr& streamer, + ov::genai::GenerationFinishReason reason) { + if (auto ovmsStreamer = std::dynamic_pointer_cast(streamer)) + ovmsStreamer->end(reason); + else + streamer->end(); +} +} // namespace +namespace { + +constexpr const char* SESSION_HEADER = "x-ovms-session-id"; + +std::string asciiLower(std::string value) { + std::transform(value.begin(), value.end(), value.begin(), [](unsigned char c) { + return static_cast(std::tolower(c)); + }); + return value; +} + +std::optional getSessionIdHeader(const std::unordered_map& headers) { + for (const auto& [name, value] : headers) { + if (asciiLower(name) == SESSION_HEADER) + return value; + } + return std::nullopt; +} + +std::string serializeJson(const rapidjson::Document& doc) { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc.Accept(writer); + return std::string(buffer.GetString(), buffer.GetSize()); +} + +std::string utcNowIso8601() { + const auto now = std::chrono::system_clock::now(); + const std::time_t tt = std::chrono::system_clock::to_time_t(now); + std::tm tm{}; +#ifdef _WIN32 + gmtime_s(&tm, &tt); +#else + gmtime_r(&tt, &tm); +#endif + std::ostringstream os; + os << std::put_time(&tm, "%Y-%m-%dT%H:%M:%SZ"); + return os.str(); +} + +uint64_t parseEnvUnsigned(const char* name, uint64_t fallback, uint64_t minimum) { + const char* raw = std::getenv(name); + if (raw == nullptr || *raw == '\0') + return fallback; + try { + const uint64_t value = std::stoull(raw); + if (value >= minimum) + return value; + } catch (...) { + } + SPDLOG_LOGGER_WARN(llm_calculator_logger, "Ignoring invalid {}='{}'; using {}", name, raw, fallback); + return fallback; +} + +std::shared_ptr processSessionStateStore() { + static std::shared_ptr store = SessionStateStore::fromEnvironment(); + return store; +} + +} // namespace + +class SessionStateStore::Impl { +public: + struct Manifest { + uint32_t seed = 0; + uint64_t nextTurn = 1; + std::string lastAccess; + std::string model; + }; + + struct CacheEntry { + Manifest manifest; + std::list::iterator lruIt; + }; + + std::filesystem::path root; + size_t cacheEntries; + uint64_t maxBytes; + size_t maxRequestBytes; + uint64_t bytesUsed = 0; + std::mutex mutex; + std::list lru; + std::unordered_map cache; + + Impl(std::string rootDirectory, size_t cacheEntries, uint64_t maxBytes, size_t maxRequestBytes) : + root(std::move(rootDirectory)), + cacheEntries(std::max(1, cacheEntries)), + maxBytes(std::max(1024 * 1024, maxBytes)), + maxRequestBytes(std::max(1024, maxRequestBytes)) { + if (root.empty()) + return; + std::error_code ec; + std::filesystem::create_directories(root, ec); + if (ec) { + SPDLOG_LOGGER_ERROR(llm_calculator_logger, "Cannot create OVMS session store '{}': {}", root.string(), ec.message()); + root.clear(); + return; + } + for (std::filesystem::recursive_directory_iterator it(root, ec), end; !ec && it != end; it.increment(ec)) { + if (it->is_regular_file(ec)) + bytesUsed += static_cast(it->file_size(ec)); + } + if (ec) + SPDLOG_LOGGER_WARN(llm_calculator_logger, "Session store size scan incomplete for '{}': {}", root.string(), ec.message()); + } + + bool enabled() const { + return !root.empty(); + } + + static bool validSessionId(const std::string& id) { + if (id.empty() || id.size() > 128) + return false; + return std::all_of(id.begin(), id.end(), [](unsigned char c) { + return std::isalnum(c) || c == '.' || c == '_' || c == '-'; + }); + } + + std::filesystem::path sessionDir(const std::string& id) const { + return root / id; + } + + std::filesystem::path manifestPath(const std::string& id) const { + return sessionDir(id) / "manifest.json"; + } + + static std::string turnName(uint64_t index) { + std::ostringstream os; + os << std::setw(12) << std::setfill('0') << index; + return os.str(); + } + + static std::string readText(const std::filesystem::path& path) { + std::ifstream in(path, std::ios::binary); + return std::string(std::istreambuf_iterator(in), std::istreambuf_iterator()); + } + + uint64_t currentFileSize(const std::filesystem::path& path) const { + std::error_code ec; + if (!std::filesystem::exists(path, ec)) + return 0; + const auto size = std::filesystem::file_size(path, ec); + return ec ? 0 : static_cast(size); + } + + absl::Status checkQuota(const std::vector>& writes) const { + int64_t delta = 0; + for (const auto& [path, content] : writes) { + const uint64_t oldSize = currentFileSize(path); + const uint64_t newSize = static_cast(content.size()); + if (newSize >= oldSize) + delta += static_cast(newSize - oldSize); + else + delta -= static_cast(oldSize - newSize); + } + if (delta > 0 && (bytesUsed > maxBytes || static_cast(delta) > maxBytes - bytesUsed)) + return absl::ResourceExhaustedError("OVMS session store byte quota exceeded"); + return absl::OkStatus(); + } + + absl::Status atomicWrite(const std::filesystem::path& path, const std::string& content) { + std::error_code ec; + std::filesystem::create_directories(path.parent_path(), ec); + if (ec) + return absl::InternalError("cannot create session journal directory: " + ec.message()); + + const uint64_t oldSize = currentFileSize(path); + std::filesystem::path tmp = path; + tmp += ".tmp-" + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()); + { + std::ofstream out(tmp, std::ios::binary | std::ios::trunc); + if (!out.good()) + return absl::InternalError("cannot open temporary session journal file"); + out.write(content.data(), static_cast(content.size())); + out.flush(); + if (!out.good()) { + out.close(); + std::filesystem::remove(tmp, ec); + return absl::InternalError("cannot write temporary session journal file"); + } + } + +#ifdef _WIN32 + if (!MoveFileExW(tmp.c_str(), path.c_str(), MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH)) { + const DWORD err = GetLastError(); + std::filesystem::remove(tmp, ec); + return absl::InternalError("cannot atomically replace session journal file, Win32 error " + std::to_string(err)); + } +#else + std::filesystem::rename(tmp, path, ec); + if (ec) { + std::filesystem::remove(tmp, ec); + return absl::InternalError("cannot atomically replace session journal file: " + ec.message()); + } +#endif + + const uint64_t newSize = static_cast(content.size()); + if (newSize >= oldSize) + bytesUsed += newSize - oldSize; + else + bytesUsed -= std::min(bytesUsed, oldSize - newSize); + return absl::OkStatus(); + } + + static std::string manifestJson(const std::string& id, const Manifest& manifest) { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + writer.StartObject(); + writer.Key("schema_version"); writer.Uint(1); + writer.Key("session_id"); writer.String(id.c_str(), static_cast(id.size())); + writer.Key("seed"); writer.Uint(manifest.seed); + writer.Key("next_turn"); writer.Uint64(manifest.nextTurn); + writer.Key("last_access"); writer.String(manifest.lastAccess.c_str(), static_cast(manifest.lastAccess.size())); + if (!manifest.model.empty()) { + writer.Key("model"); writer.String(manifest.model.c_str(), static_cast(manifest.model.size())); + } + writer.EndObject(); + return std::string(buffer.GetString(), buffer.GetSize()); + } + + absl::StatusOr loadManifestFromDisk(const std::string& id) { + const auto path = manifestPath(id); + std::error_code ec; + if (!std::filesystem::exists(path, ec)) + return Manifest{}; + const std::string text = readText(path); + rapidjson::Document doc; + doc.Parse(text.c_str(), text.size()); + if (doc.HasParseError() || !doc.IsObject() || !doc.HasMember("seed") || !doc["seed"].IsUint() || + !doc.HasMember("next_turn") || !doc["next_turn"].IsUint64()) + return absl::DataLossError("invalid OVMS session manifest for '" + id + "'"); + Manifest manifest; + manifest.seed = doc["seed"].GetUint(); + manifest.nextTurn = doc["next_turn"].GetUint64(); + if (doc.HasMember("last_access") && doc["last_access"].IsString()) + manifest.lastAccess.assign(doc["last_access"].GetString(), doc["last_access"].GetStringLength()); + if (doc.HasMember("model") && doc["model"].IsString()) + manifest.model.assign(doc["model"].GetString(), doc["model"].GetStringLength()); + return manifest; + } + + void putCache(const std::string& id, const Manifest& manifest) { + auto it = cache.find(id); + if (it != cache.end()) { + lru.erase(it->second.lruIt); + cache.erase(it); + } + lru.push_front(id); + cache.emplace(id, CacheEntry{manifest, lru.begin()}); + while (cache.size() > cacheEntries) { + const std::string evict = lru.back(); + lru.pop_back(); + cache.erase(evict); + } + } + + absl::StatusOr loadManifest(const std::string& id) { + auto it = cache.find(id); + if (it != cache.end()) { + Manifest manifest = it->second.manifest; + lru.erase(it->second.lruIt); + lru.push_front(id); + it->second.lruIt = lru.begin(); + return manifest; + } + auto loaded = loadManifestFromDisk(id); + if (!loaded.ok()) + return loaded.status(); + putCache(id, *loaded); + return *loaded; + } + + static uint32_t generateSeed() { + static thread_local std::mt19937 rng{std::random_device{}()}; + uint32_t seed = 0; + while (seed == 0) + seed = rng(); + return seed; + } + + static std::string generationConfigJson(const ov::genai::GenerationConfig& config, const std::string& toolChoice) { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + writer.StartObject(); + writer.Key("rng_seed"); writer.Uint64(config.rng_seed); + writer.Key("temperature"); writer.Double(config.temperature); + writer.Key("top_p"); writer.Double(config.top_p); + writer.Key("top_k"); writer.Uint64(config.top_k); + writer.Key("min_p"); writer.Double(config.min_p); + writer.Key("do_sample"); writer.Bool(config.do_sample); + writer.Key("num_beams"); writer.Uint64(config.num_beams); + writer.Key("max_new_tokens"); writer.Uint64(config.max_new_tokens); + writer.Key("max_length"); writer.Uint64(config.max_length); + writer.Key("ignore_eos"); writer.Bool(config.ignore_eos); + writer.Key("repetition_penalty"); writer.Double(config.repetition_penalty); + writer.Key("tool_choice"); writer.String(toolChoice.c_str(), static_cast(toolChoice.size())); + writer.Key("structured_output_active"); writer.Bool(config.structured_output_config.has_value()); + writer.EndObject(); + return std::string(buffer.GetString(), buffer.GetSize()); + } +}; + +SessionStateStore::SessionStateStore(std::string rootDirectory, size_t cacheEntries, uint64_t maxBytes, size_t maxRequestBytes) : + impl(std::make_unique(std::move(rootDirectory), cacheEntries, maxBytes, maxRequestBytes)) {} + +SessionStateStore::~SessionStateStore() = default; +SessionStateStore::SessionStateStore(SessionStateStore&&) noexcept = default; +SessionStateStore& SessionStateStore::operator=(SessionStateStore&&) noexcept = default; + +std::shared_ptr SessionStateStore::fromEnvironment() { + const char* root = std::getenv("OVMS_SESSION_STORE_DIR"); + const std::string rootDirectory = (root == nullptr) ? std::string{} : std::string(root); + const size_t cacheEntries = static_cast(parseEnvUnsigned("OVMS_SESSION_CACHE_ENTRIES", DEFAULT_CACHE_ENTRIES, 1)); + const uint64_t maxBytes = parseEnvUnsigned("OVMS_SESSION_MAX_BYTES", DEFAULT_MAX_BYTES, 1024 * 1024); + const size_t maxRequestBytes = static_cast(parseEnvUnsigned("OVMS_SESSION_MAX_REQUEST_BYTES", DEFAULT_MAX_REQUEST_BYTES, 1024)); + if (rootDirectory.empty()) { + SPDLOG_LOGGER_INFO(llm_calculator_logger, "session-state: store disabled; OVMS_SESSION_STORE_DIR is unset"); + } else { + SPDLOG_LOGGER_INFO(llm_calculator_logger, + "session-state: constructing store from OVMS_SESSION_STORE_DIR={}", + rootDirectory); + } + return std::make_shared(rootDirectory, cacheEntries, maxBytes, maxRequestBytes); +} + +bool SessionStateStore::enabled() const { + return impl && impl->enabled(); +} + +absl::StatusOr SessionStateStore::beginTurn( + const std::string& sessionId, + const std::string& rawBody, + rapidjson::Document& effectiveDocument) { + if (!enabled()) + return SessionTurnContext{}; + if (!Impl::validSessionId(sessionId)) + return absl::InvalidArgumentError("invalid OVMS session id"); + if (rawBody.size() > impl->maxRequestBytes) + return absl::ResourceExhaustedError("OVMS session request exceeds configured journal request limit"); + if (!effectiveDocument.IsObject()) + return absl::InvalidArgumentError("session persistence requires a JSON object request"); + + std::lock_guard lock(impl->mutex); + auto manifestResult = impl->loadManifest(sessionId); + if (!manifestResult.ok()) + return manifestResult.status(); + auto manifest = *manifestResult; + + std::optional requestedSeed; + if (effectiveDocument.HasMember("seed")) { + if (!effectiveDocument["seed"].IsUint()) + return absl::InvalidArgumentError("session seed must be an unsigned 32-bit integer"); + requestedSeed = effectiveDocument["seed"].GetUint(); + } + + const bool existing = std::filesystem::exists(impl->manifestPath(sessionId)); + if (!existing) { + manifest.seed = requestedSeed.value_or(Impl::generateSeed()); + manifest.nextTurn = 1; + } else if (requestedSeed.has_value() && requestedSeed.value() != manifest.seed) { + return absl::InvalidArgumentError( + "session seed conflict: persisted=" + std::to_string(manifest.seed) + + ", requested=" + std::to_string(requestedSeed.value())); + } + + auto& allocator = effectiveDocument.GetAllocator(); + if (effectiveDocument.HasMember("seed")) + effectiveDocument["seed"].SetUint(manifest.seed); + else { + rapidjson::Value seedName("seed", allocator); + effectiveDocument.AddMember(seedName, manifest.seed, allocator); + } + + if (effectiveDocument.HasMember("model") && effectiveDocument["model"].IsString()) + manifest.model.assign(effectiveDocument["model"].GetString(), effectiveDocument["model"].GetStringLength()); + manifest.lastAccess = utcNowIso8601(); + + const uint64_t turnIndex = manifest.nextTurn++; + const auto turnDir = impl->sessionDir(sessionId) / "turns" / Impl::turnName(turnIndex); + const auto rawPath = turnDir / "raw-request.json"; + const auto effectivePath = turnDir / "effective-request.json"; + const auto manifestPath = impl->manifestPath(sessionId); + const std::string effectiveBody = serializeJson(effectiveDocument); + const std::string manifestBody = Impl::manifestJson(sessionId, manifest); + + const std::vector> writes{ + {rawPath, rawBody}, + {effectivePath, effectiveBody}, + {manifestPath, manifestBody}, + }; + auto quotaStatus = impl->checkQuota(writes); + if (!quotaStatus.ok()) + return quotaStatus; + for (const auto& [path, content] : writes) { + auto status = impl->atomicWrite(path, content); + if (!status.ok()) + return status; + } + impl->putCache(sessionId, manifest); + + SessionTurnContext turn; + turn.active = true; + turn.sessionId = sessionId; + turn.turnIndex = turnIndex; + turn.seed = manifest.seed; + return turn; +} + +absl::Status SessionStateStore::recordGenerationConfig( + const SessionTurnContext& turn, + const ov::genai::GenerationConfig& config, + const std::string& toolChoice) { + if (!turn.active || !enabled()) + return absl::OkStatus(); + if (!Impl::validSessionId(turn.sessionId)) + return absl::InvalidArgumentError("invalid OVMS session id"); + + std::lock_guard lock(impl->mutex); + const auto path = impl->sessionDir(turn.sessionId) / "turns" / Impl::turnName(turn.turnIndex) / "generation-config.json"; + const std::string content = Impl::generationConfigJson(config, toolChoice); + auto quotaStatus = impl->checkQuota({{path, content}}); + if (!quotaStatus.ok()) + return quotaStatus; + return impl->atomicWrite(path, content); +} double calculatePrefillSpeed(size_t inputTokenCount, double ttftMs) { return ttftMs > 0.0 ? (1000.0 * inputTokenCount) / ttftMs : 0.0; @@ -105,6 +557,44 @@ absl::Status GenAiServable::loadRequest(std::shared_ptrpayload = payload; + // All legacy parsers override parseRequest; session setup must precede dispatch. + if (executionContext->endpoint == Endpoint::TOKENIZE) + return absl::OkStatus(); + auto sessionStore = processSessionStateStore(); + executionContext->sessionTurn = {}; + auto sessionId = getSessionIdHeader(executionContext->payload.headers); + const char* storeDir = std::getenv("OVMS_SESSION_STORE_DIR"); + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, + "session-state: uri={} header_count={} session_present={} store_enabled={} store_path={}", + payload.uri, + payload.headers.size(), + sessionId.has_value(), + sessionStore->enabled(), + storeDir ? storeDir : ""); + if (sessionId.has_value()) { + if (sessionStore->enabled()) { + auto turn = sessionStore->beginTurn(sessionId.value(), executionContext->payload.body, *executionContext->payload.parsedJson); + if (!turn.ok()) { + SPDLOG_LOGGER_ERROR(llm_calculator_logger, + "session-state: beginTurn failed status={}", + turn.status().ToString()); + return turn.status(); + } + executionContext->sessionTurn = std::move(*turn); + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, + "session-state: beginTurn turn={} seed={} store_path={}", + executionContext->sessionTurn.turnIndex, + executionContext->sessionTurn.seed, + storeDir ? storeDir : ""); + } else { + static std::once_flag warningOnce; + std::call_once(warningOnce, []() { + SPDLOG_LOGGER_WARN(llm_calculator_logger, + "X-OVMS-Session-ID received but OVMS_SESSION_STORE_DIR is not configured; session persistence is disabled"); + }); + } + } + return absl::OkStatus(); } @@ -191,6 +681,7 @@ absl::Status GenAiServable::parseRequest(std::shared_ptrinputRequest = std::move(*inputRequestResult); + return absl::OkStatus(); } @@ -220,6 +711,15 @@ absl::Status GenAiServable::prepareInputs(std::shared_ptrsessionTurn.active) { + auto journalStatus = processSessionStateStore()->recordGenerationConfig( + executionContext->sessionTurn, + executionContext->inputRequest.generationConfig, + executionContext->apiHandler->getToolChoice()); + if (!journalStatus.ok()) + return journalStatus; + } + auto status = validateInputCompatibility(executionContext); if (!status.ok()) { return status; @@ -293,7 +793,7 @@ absl::Status GenAiServable::prepareCompleteResponse(std::shared_ptrtextStreamer->write(output.generated_ids); - executionContext->textStreamer->end(); + finishTextStreamer(executionContext->textStreamer, output.finish_reason); localDeltas = executionContext->deltaChannel.drain(); } else { // Multiple sequences: each beam requires its own independent stateful streamer @@ -313,7 +813,7 @@ absl::Status GenAiServable::prepareCompleteResponse(std::shared_ptrwrite(output.generated_ids); - tempStreamer->end(); + tempStreamer->end(output.finish_reason); } allDeltas.push_back(std::move(localDeltas)); @@ -358,9 +858,9 @@ absl::Status GenAiServable::preparePartialResponse(std::shared_ptrtextStreamer->end(); + finishTextStreamer(executionContext->textStreamer, finishReason); } // Drain all deltas accumulated during this write()/end() cycle. @@ -486,7 +986,6 @@ absl::Status prepareLegacyPartialResponse(std::shared_ptrgenerate() returns and results are assigned) // to guarantee results is populated before we read finish_reasons and perf_metrics. - // Also ensures success flag is accurate. legacyCtx->finished.wait(); if (!legacyCtx->success) { return absl::InvalidArgumentError("Request processing failed, check its correctness."); @@ -539,6 +1038,9 @@ void logRequestDetails(const ovms::HttpPayload& payload) { parsedJson->Accept(writer); SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Request body: {}", buffer.GetString()); SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Request uri: {}", payload.uri); + for (const auto& [name, value] : payload.headers) { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Request header: {}={}", name, value); + } } } // namespace ovms diff --git a/src/llm/servable.hpp b/src/llm/servable.hpp index ca8bdb6205..77e52a3206 100644 --- a/src/llm/servable.hpp +++ b/src/llm/servable.hpp @@ -16,6 +16,7 @@ #pragma once #include +#include #include #include #include @@ -31,6 +32,8 @@ #include "io_processing/delta.hpp" #include "openvino/genai/text_streamer.hpp" #include "mediapipe/framework/calculator_graph.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" #pragma GCC diagnostic pop #pragma warning(pop) @@ -80,6 +83,52 @@ enum class ChatTemplateMode { JINJA, // Use Python Jinja2 module for chat template processing }; +// Lightweight handle for one durable request journal entry. It deliberately does +// not retain request bodies in RAM; those live in the disk-backed session store. +struct SessionTurnContext { + bool active = false; + std::string sessionId; + uint64_t turnIndex = 0; + uint32_t seed = 0; +}; + +// Process-wide session metadata store used by OpenAI HTTP turns. The public API is +// intentionally generic: model-specific builders only see an ordinary explicit +// request seed after beginTurn() has resolved the session contract. +class SessionStateStore { +public: + static constexpr size_t DEFAULT_CACHE_ENTRIES = 64; + static constexpr uint64_t DEFAULT_MAX_BYTES = 1024ULL * 1024ULL * 1024ULL; + static constexpr size_t DEFAULT_MAX_REQUEST_BYTES = 8ULL * 1024ULL * 1024ULL; + + SessionStateStore(std::string rootDirectory, + size_t cacheEntries = DEFAULT_CACHE_ENTRIES, + uint64_t maxBytes = DEFAULT_MAX_BYTES, + size_t maxRequestBytes = DEFAULT_MAX_REQUEST_BYTES); + ~SessionStateStore(); + SessionStateStore(SessionStateStore&&) noexcept; + SessionStateStore& operator=(SessionStateStore&&) noexcept; + SessionStateStore(const SessionStateStore&) = delete; + SessionStateStore& operator=(const SessionStateStore&) = delete; + + static std::shared_ptr fromEnvironment(); + bool enabled() const; + + absl::StatusOr beginTurn( + const std::string& sessionId, + const std::string& rawBody, + rapidjson::Document& effectiveDocument); + + absl::Status recordGenerationConfig( + const SessionTurnContext& turn, + const ov::genai::GenerationConfig& config, + const std::string& toolChoice); + +private: + class Impl; + std::unique_ptr impl; +}; + // Thread-safe channel for parsed streaming deltas. // The producer (OVMSTextStreamer callback, possibly on a background executor thread) // calls push(); the consumer (preparePartialResponse, always on the calculator thread) @@ -141,6 +190,7 @@ struct DeltaChannel { struct GenAiServableExecutionContext { // Common API related members HttpPayload payload; + SessionTurnContext sessionTurn; Endpoint endpoint; std::shared_ptr apiHandler; // Populated in parseRequest(); carries all GenAI inputs including the generation config. diff --git a/src/test/llm/gemma4_fast/BUILD b/src/test/llm/gemma4_fast/BUILD new file mode 100644 index 0000000000..576f1e8fc0 --- /dev/null +++ b/src/test/llm/gemma4_fast/BUILD @@ -0,0 +1,23 @@ +load("//:common_settings.bzl", "COMMON_LOCAL_DEFINES", "COMMON_STATIC_LIBS_LINKOPTS", "COPTS_TESTS") + +cc_test( + name = "gemma4_parser_contract_test", + env_inherit = ["GEMMA4_TOKENIZER_PATH", "PATH"], + srcs = [ + "gemma4_parser_contract_test.cpp", + "gemma4_reasoning_semantic_refit_test.cpp", + "gemma4_recovery_contract_test.cpp", + ], + deps = [ + "@com_google_googletest//:gtest_main", + "//src:test_platform_utils", + "//src/llm:output_parsers", + "//src/llm:text_streamer", + "//third_party:genai", + "//third_party:openvino", + ], + copts = COPTS_TESTS, + local_defines = COMMON_LOCAL_DEFINES, + linkopts = COMMON_STATIC_LIBS_LINKOPTS, + linkstatic = 1, +) diff --git a/src/test/llm/gemma4_fast/gemma4_parser_contract_test.cpp b/src/test/llm/gemma4_fast/gemma4_parser_contract_test.cpp new file mode 100644 index 0000000000..47a2edad2f --- /dev/null +++ b/src/test/llm/gemma4_fast/gemma4_parser_contract_test.cpp @@ -0,0 +1,316 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "../../../llm/io_processing/output_parser.hpp" +#include "../../../llm/io_processing/gemma4/gemma4_reasoning_parser.hpp" +#include "../../../llm/io_processing/gemma4/gemma4_tool_parser.hpp" +#include "../../../llm/ovms_text_streamer.hpp" +#include "../../platform_utils.hpp" + +using namespace ovms; + +namespace ovms { +struct Gemma4ToolParserTestAccess { + static size_t bufferedBytes(const Gemma4ToolParser& parser) { + return parser.streamingContent.size(); + } +}; +} + +namespace { +template +struct Overloaded : Ts... { + using Ts::operator()...; +}; +template +Overloaded(Ts...) -> Overloaded; + +#ifdef _WIN32 +const std::string tokenizerPath = getWindowsRepoRootPath() + "\\src\\test\\llm_testing\\OpenVINO\\gemma-4-E4B-it-int4-ov"; +#else +const std::string tokenizerPath = "/ovms/src/test/llm_testing/OpenVINO/gemma-4-E4B-it-int4-ov"; +#endif + +const std::string questionSchema = R"({"type":"object","properties":{"questions":{"type":"array","items":{"type":"object","properties":{"question":{"type":"string"},"header":{"type":"string"},"options":{"type":"array","items":{"type":"object","properties":{"label":{"type":"string"},"description":{"type":"string"}}}},"multiple":{"type":"boolean"},"custom":{"type":"boolean"}}}}}})"; + +ParsedOutput parseWithStreamer( + const ov::genai::Tokenizer& tokenizer, + OutputParser& outputParser, + const std::vector& generatedTokens) { + outputParser.resetStreamingState(); + + ParsedOutput result; + std::vector toolCalls; + auto callback = [&](Delta delta, bool /*isLast*/) { + std::visit(Overloaded{ + [&](const ContentDelta& d) { result.content.append(d.text); }, + [&](const ReasoningDelta& d) { result.reasoning.append(d.text); }, + [&](const ToolCallDelta& d) { + if (d.index < 0) + return; + const auto idx = static_cast(d.index); + if (idx >= toolCalls.size()) + toolCalls.resize(idx + 1); + auto& tc = toolCalls[idx]; + if (d.id) + tc.id = *d.id; + if (d.name) + tc.name = *d.name; + tc.arguments.append(d.arguments); + }, + [](const FinishDelta&) {}, + [](const AudioDelta&) {}, + }, + delta); + return ov::genai::StreamingStatus::RUNNING; + }; + + auto parserPtr = std::shared_ptr(&outputParser, [](OutputParser*) {}); + const ov::AnyMap decodeParams{{ov::genai::skip_special_tokens.name(), false}}; + OVMSTextStreamer streamer(tokenizer, parserPtr, true, std::move(callback), decodeParams); + for (int64_t token : generatedTokens) + streamer.write(token); + streamer.end(); + + // Match the API accumulator: never hide header-only or sparse calls. + result.toolCalls = std::move(toolCalls); + return result; +} + +class Gemma4ParserFastContractTest : public ::testing::Test { +protected: + static std::unique_ptr tokenizer; + + static void SetUpTestSuite() { + const char* configured = std::getenv("GEMMA4_TOKENIZER_PATH"); + tokenizer = std::make_unique(configured ? configured : tokenizerPath); + } + + static void TearDownTestSuite() { + tokenizer.reset(); + } + + static ToolsSchemas_t questionTools() { + ToolsSchemas_t tools; + tools.emplace("question", ToolSchemaWrapper{nullptr, questionSchema}); + return tools; + } + + ParsedOutput parse(const std::string& input, const ToolsSchemas_t& tools = questionTools()) { + OutputParser parser(*tokenizer, "gemma4", "gemma4", tools); + auto tensor = tokenizer->encode(input).input_ids; + std::vector tokens(tensor.data(), tensor.data() + tensor.get_size()); + return parseWithStreamer(*tokenizer, parser, tokens); + } +}; + +std::unique_ptr Gemma4ParserFastContractTest::tokenizer; + +const std::string nativeQuestionArgs = + R"(questions:[{question:<|"|>Pick one?<|"|>,header:<|"|>Test<|"|>,options:[{label:<|"|>A<|"|>,description:<|"|>Alpha<|"|>},{label:<|"|>B<|"|>,description:<|"|>Beta<|"|>}],multiple:false,custom:true}])"; + +const std::string expectedQuestionJson = + R"({"questions":[{"question":"Pick one?","header":"Test","options":[{"label":"A","description":"Alpha"},{"label":"B","description":"Beta"}],"multiple":false,"custom":true}]})"; +} // namespace + +TEST_F(Gemma4ParserFastContractTest, ParsesOpenCodeQuestionArrayOfObjectsRecursively) { + auto parsed = parse("<|tool_call>call:question{" + nativeQuestionArgs + "}"); + ASSERT_EQ(parsed.toolCalls.size(), 1u); + EXPECT_EQ(parsed.toolCalls[0].name, "question"); + EXPECT_EQ(parsed.toolCalls[0].arguments, expectedQuestionJson); +} + +TEST_F(Gemma4ParserFastContractTest, PreservesNestedScalarTypes) { + auto parsed = parse(R"(<|tool_call>call:question{questions:[],meta:{count:2,score:22.8,missing:null,flags:[true,false,null,3]}})"); + ASSERT_EQ(parsed.toolCalls.size(), 1u); + EXPECT_EQ(parsed.toolCalls[0].arguments, + R"({"questions":[],"meta":{"count":2,"score":22.8,"missing":null,"flags":[true,false,null,3]}})"); +} + +TEST_F(Gemma4ParserFastContractTest, NativeDelimitedKeysDoNotLeakProtocolMarkers) { + auto parsed = parse(R"(<|tool_call>call:question{<|"|>questions<|"|>:[],<|"|>label<|"|>:<|"|>safe<|"|>})"); + ASSERT_EQ(parsed.toolCalls.size(), 1u); + EXPECT_EQ(parsed.toolCalls[0].arguments, R"({"questions":[],"label":"safe"})"); + EXPECT_EQ(parsed.toolCalls[0].arguments.find("<|\"|>"), std::string::npos); +} + +TEST_F(Gemma4ParserFastContractTest, StringPayloadCannotBreakStructuralScanning) { + auto parsed = parse(R"(<|tool_call>call:question{questions:[],template:<|"|>Hello {name}, items: [a, b, c], json={"x":1}<|"|>})"); + ASSERT_EQ(parsed.toolCalls.size(), 1u); + EXPECT_EQ(parsed.toolCalls[0].arguments, + R"({"questions":[],"template":"Hello {name}, items: [a, b, c], json={\"x\":1}"})"); +} + +TEST_F(Gemma4ParserFastContractTest, AcceptsParenthesizedArgumentsWhenAnchored) { + auto parsed = parse("<|tool_call>call:question(" + nativeQuestionArgs + ")"); + ASSERT_EQ(parsed.toolCalls.size(), 1u); + EXPECT_EQ(parsed.toolCalls[0].arguments, expectedQuestionJson); +} + +TEST_F(Gemma4ParserFastContractTest, AcceptsColonNameVariantWhenAnchored) { + auto parsed = parse("<|tool_call>:question{" + nativeQuestionArgs + "}"); + ASSERT_EQ(parsed.toolCalls.size(), 1u); + EXPECT_EQ(parsed.toolCalls[0].name, "question"); +} + +TEST_F(Gemma4ParserFastContractTest, AcceptsDirectCallAfterReasoning) { + auto parsed = parse("<|channel>thought\nNeed user inputcall:question{" + nativeQuestionArgs + "}"); + EXPECT_EQ(parsed.reasoning, "Need user input"); + ASSERT_EQ(parsed.toolCalls.size(), 1u); + EXPECT_EQ(parsed.toolCalls[0].name, "question"); +} + +TEST_F(Gemma4ParserFastContractTest, UnknownToolIsNotExecutable) { + auto parsed = parse(R"(<|tool_call>call:not_in_request{x:1})"); + EXPECT_TRUE(parsed.toolCalls.empty()); +} + +TEST_F(Gemma4ParserFastContractTest, MalformedCallIsBoundedAndLaterValidCallSurvives) { + auto parsed = parse(R"(<|tool_call>call:question{questions:[{question:<|"|>broken<|"|>}<|tool_call>call:question{questions:[]})"); + ASSERT_EQ(parsed.toolCalls.size(), 1u); + EXPECT_EQ(parsed.toolCalls[0].name, "question"); + EXPECT_EQ(parsed.toolCalls[0].arguments, R"({"questions":[]})"); +} + +TEST_F(Gemma4ParserFastContractTest, TruncatedArgumentsEmitNoToolCall) { + auto parsed = parse(R"(<|tool_call>call:question{questions:[)"); + EXPECT_TRUE(parsed.toolCalls.empty()); +} + +TEST_F(Gemma4ParserFastContractTest, RepeatedSameFunctionCallsKeepCallLocalIdentity) { + auto parsed = parse( + R"(<|tool_call>call:question{questions:[]})" + R"(<|tool_call>call:question{questions:[]})" + R"(<|tool_call>call:question{questions:[]})"); + ASSERT_EQ(parsed.toolCalls.size(), 3u); + for (const auto& call : parsed.toolCalls) { + EXPECT_EQ(call.name, "question"); + EXPECT_EQ(call.arguments, R"({"questions":[]})"); + EXPECT_FALSE(call.id.empty()); + } + EXPECT_NE(parsed.toolCalls[0].id, parsed.toolCalls[1].id); + EXPECT_NE(parsed.toolCalls[0].id, parsed.toolCalls[2].id); + EXPECT_NE(parsed.toolCalls[1].id, parsed.toolCalls[2].id); +} + +TEST_F(Gemma4ParserFastContractTest, JsonStringEndMarkerIsPayload) { + auto parsed = parse(R"(<|tool_call>call:question{"text":"keep literal","questions":[]})"); + ASSERT_EQ(parsed.toolCalls.size(), 1u); + EXPECT_EQ(parsed.toolCalls[0].arguments, R"({"text":"keep literal","questions":[]})"); + EXPECT_TRUE(parsed.content.empty()); +} + +TEST_F(Gemma4ParserFastContractTest, NativeStringEndMarkerIsPayload) { + auto parsed = parse(R"(<|tool_call>call:question{text:<|"|>keep literal<|"|>,questions:[]})"); + ASSERT_EQ(parsed.toolCalls.size(), 1u); + EXPECT_EQ(parsed.toolCalls[0].arguments, R"({"text":"keep literal","questions":[]})"); + EXPECT_TRUE(parsed.content.empty()); +} + +TEST_F(Gemma4ParserFastContractTest, PreservesJsonEscapesAndWindowsPaths) { + auto parsed = parse(R"(<|tool_call>call:question{"questions":[],"path":"C:\\git\\x","text":"quote: \"x\"; slash: \\; braces: {}[]"})"); + ASSERT_EQ(parsed.toolCalls.size(), 1u); + EXPECT_EQ(parsed.toolCalls[0].arguments, R"({"questions":[],"path":"C:\\git\\x","text":"quote: \"x\"; slash: \\; braces: {}[]"})"); +} + +TEST_F(Gemma4ParserFastContractTest, ReasoningBoundaryChunkKeepsText) { + Gemma4ReasoningParser parser(*tokenizer); + auto delta = parser.parseChunk("<|channel>thought\nNeed user input", {}, ov::genai::GenerationFinishReason::STOP); + ASSERT_TRUE(delta.has_value()); + ASSERT_TRUE(std::holds_alternative(*delta)); + EXPECT_EQ(std::get(*delta).text, "Need user input"); +} + +TEST_F(Gemma4ParserFastContractTest, CompleteCallSurvivesEveryByteSplit) { + const std::string input = R"(<|tool_call>call:question{"text":"keep literal","questions":[]})"; + for (size_t split = 0; split <= input.size(); ++split) { + SCOPED_TRACE(split); + OutputParser parser(*tokenizer, "gemma4", "gemma4", questionTools()); + std::vector calls; + for (const auto& chunk : {input.substr(0, split), input.substr(split)}) { + auto delta = parser.parseChunk(chunk, {}, true, ov::genai::GenerationFinishReason::NONE); + if (delta && std::holds_alternative(*delta)) + calls.push_back(std::get(*delta)); + } + auto final = parser.parseChunk("", {}, true, ov::genai::GenerationFinishReason::STOP); + if (final && std::holds_alternative(*final)) + calls.push_back(std::get(*final)); + ASSERT_EQ(calls.size(), 1u); + EXPECT_EQ(calls[0].name.value_or(""), "question"); + EXPECT_EQ(calls[0].arguments, R"({"text":"keep literal","questions":[]})"); + } +} + +TEST_F(Gemma4ParserFastContractTest, NonGemmaReasoningDoesNotTreatLiteralToolMarkerAsBoundary) { + OutputParser parser(*tokenizer, "gemma4", "qwen3", questionTools()); + parser.parseChunk("", {}, true, ov::genai::GenerationFinishReason::NONE); + auto delta = parser.parseChunk("literal <|tool_call> example", {}, true, ov::genai::GenerationFinishReason::NONE); + ASSERT_TRUE(delta.has_value()); + ASSERT_TRUE(std::holds_alternative(*delta)); + EXPECT_EQ(std::get(*delta).text, "literal <|tool_call> example"); +} + +TEST_F(Gemma4ParserFastContractTest, PreservesNumbersBeyondMachinePrecision) { + const std::string numbers = "[18446744073709551617,-9223372036854775809,0.123456789012345678901,1e400,-0]"; + for (const std::string key : {"values", "\"values\""}) { + auto parsed = parse("<|tool_call>call:question{" + key + ":" + numbers + "}"); + ASSERT_EQ(parsed.toolCalls.size(), 1u); + EXPECT_EQ(parsed.toolCalls[0].arguments, "{\"values\":" + numbers + "}"); + } + EXPECT_EQ(Gemma4ToolParser::normalizeArgStr(numbers), numbers); +} + +TEST_F(Gemma4ParserFastContractTest, MalformedNumericLexemesNeverBecomeExecutableArguments) { + for (const std::string value : {"1.", "1e", "-", "01"}) { + SCOPED_TRACE(value); + auto parsed = parse("<|tool_call>call:question{value:" + value + "}"); + EXPECT_TRUE(parsed.toolCalls.empty()); + } +} + +TEST_F(Gemma4ParserFastContractTest, ManyCallsReleaseConsumedBufferAndKeepOwnedDeltas) { + Gemma4ToolParser parser(*tokenizer); + std::vector calls; + for (int i = 0; i < 300; ++i) { + const std::string input = "<|tool_call>call:question{value:" + std::to_string(i) + "}"; + // Drive every internal phase, retaining deltas as a caller would. + for (int phase = 0; phase < 6; ++phase) { + auto delta = parser.parseChunk(phase == 0 ? input : "", {}, ov::genai::GenerationFinishReason::NONE); + if (delta && std::holds_alternative(*delta)) + calls.push_back(std::get(*delta)); + } + EXPECT_LT(Gemma4ToolParserTestAccess::bufferedBytes(parser), 8192u); + } + ASSERT_EQ(calls.size(), 300u); + for (int i = 0; i < 300; ++i) { + EXPECT_EQ(calls[i].index, i); + EXPECT_EQ(calls[i].arguments, "{\"value\":" + std::to_string(i) + "}"); + } + parser.resetState(); + EXPECT_EQ(Gemma4ToolParserTestAccess::bufferedBytes(parser), 0u); + EXPECT_EQ(calls.front().arguments, "{\"value\":0}"); +} diff --git a/src/test/llm/gemma4_fast/gemma4_reasoning_semantic_refit_test.cpp b/src/test/llm/gemma4_fast/gemma4_reasoning_semantic_refit_test.cpp new file mode 100644 index 0000000000..8a8dd7bc46 --- /dev/null +++ b/src/test/llm/gemma4_fast/gemma4_reasoning_semantic_refit_test.cpp @@ -0,0 +1,186 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +//***************************************************************************** + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "../../../llm/io_processing/output_parser.hpp" +#include "../../platform_utils.hpp" + +using namespace ovms; + +namespace { +#ifdef _WIN32 +const std::string tokenizerPath = getWindowsRepoRootPath() + "\\src\\test\\llm_testing\\OpenVINO\\gemma-4-E4B-it-int4-ov"; +#else +const std::string tokenizerPath = "/ovms/src/test/llm_testing/OpenVINO/gemma-4-E4B-it-int4-ov"; +#endif + +const std::string questionSchema = R"({"type":"object","properties":{"questions":{"type":"array"}}})"; +const std::string questionCall = "<|tool_call>call:question{questions:[]}"; +const std::string toolStart = "<|tool_call>"; + +ToolsSchemas_t questionTools() { + ToolsSchemas_t tools; + tools.emplace("question", ToolSchemaWrapper{nullptr, questionSchema}); + return tools; +} + +std::optional driveUntilToolCall(OutputParser& parser, const std::string& firstChunk) { + for (int step = 0; step < 8; ++step) { + const auto delta = parser.parseChunk( + step == 0 ? firstChunk : std::string{}, + {}, + true, + step == 7 ? ov::genai::GenerationFinishReason::STOP : ov::genai::GenerationFinishReason::NONE); + if (delta && std::holds_alternative(*delta)) { + return std::get(*delta); + } + } + return std::nullopt; +} + +void expectQuestionCall(const std::optional& call) { + ASSERT_TRUE(call.has_value()); + EXPECT_EQ(call->name.value_or(""), "question"); + EXPECT_EQ(call->arguments, R"({"questions":[]})"); +} + +class Gemma4ReasoningSemanticRefitTest : public ::testing::Test { +protected: + static std::unique_ptr tokenizer; + + static void SetUpTestSuite() { + const char* configured = std::getenv("GEMMA4_TOKENIZER_PATH"); + tokenizer = std::make_unique(configured ? configured : tokenizerPath); + } + + static void TearDownTestSuite() { + tokenizer.reset(); + } +}; + +std::unique_ptr Gemma4ReasoningSemanticRefitTest::tokenizer; +} // namespace + +TEST_F(Gemma4ReasoningSemanticRefitTest, CanonicalReasoningCloseTransitionsToTool) { + OutputParser parser(*tokenizer, "gemma4", "gemma4", questionTools()); + + auto reasoning = parser.parseChunk( + "<|channel>thought\nNeed another tool" + questionCall, + {}, + true, + ov::genai::GenerationFinishReason::NONE); + + ASSERT_TRUE(reasoning.has_value()); + ASSERT_TRUE(std::holds_alternative(*reasoning)); + EXPECT_EQ(std::get(*reasoning).text, "Need another tool"); + expectQuestionCall(driveUntilToolCall(parser, "")); +} + +TEST_F(Gemma4ReasoningSemanticRefitTest, RecoveryToolStartEndsOpenReasoningWithoutCanonicalCloser) { + OutputParser parser(*tokenizer, "gemma4", "gemma4", questionTools()); + + auto first = parser.parseChunk( + "<|channel>thought\nNeed another tool", + {}, + true, + ov::genai::GenerationFinishReason::NONE); + ASSERT_TRUE(first.has_value()); + ASSERT_TRUE(std::holds_alternative(*first)); + EXPECT_EQ(std::get(*first).text, "Need another tool"); + + expectQuestionCall(driveUntilToolCall(parser, questionCall)); +} + +TEST_F(Gemma4ReasoningSemanticRefitTest, RecoveryImplicitPromptReasoningCanTransitionDirectlyToTool) { + OutputParser parser(*tokenizer, "gemma4", "gemma4", questionTools()); + parser.detectAndSetImplicitReasoningStart("prompt<|channel>thought\n"); + + expectQuestionCall(driveUntilToolCall(parser, questionCall)); +} + +TEST_F(Gemma4ReasoningSemanticRefitTest, RecoverySameChunkReasoningPrefixIsPreservedBeforeToolHandoff) { + OutputParser parser(*tokenizer, "gemma4", "gemma4", questionTools()); + parser.detectAndSetImplicitReasoningStart("prompt<|channel>thought\n"); + + auto reasoning = parser.parseChunk( + "Need another tool" + questionCall, + {}, + true, + ov::genai::GenerationFinishReason::NONE); + + ASSERT_TRUE(reasoning.has_value()); + ASSERT_TRUE(std::holds_alternative(*reasoning)); + EXPECT_EQ(std::get(*reasoning).text, "Need another tool"); + expectQuestionCall(driveUntilToolCall(parser, "")); +} + +TEST_F(Gemma4ReasoningSemanticRefitTest, RecoveryPartialToolMarkerIsHeldBackInsteadOfLeakingIntoReasoning) { + OutputParser parser(*tokenizer, "gemma4", "gemma4", questionTools()); + parser.detectAndSetImplicitReasoningStart("prompt<|channel>thought\n"); + + auto partial = parser.parseChunk( + "Need another tool<|tool_", + {}, + true, + ov::genai::GenerationFinishReason::NONE); + EXPECT_FALSE(partial.has_value()); + + auto reasoning = parser.parseChunk( + "call>call:question{questions:[]}", + {}, + true, + ov::genai::GenerationFinishReason::NONE); + ASSERT_TRUE(reasoning.has_value()); + ASSERT_TRUE(std::holds_alternative(*reasoning)); + EXPECT_EQ(std::get(*reasoning).text, "Need another tool"); + + expectQuestionCall(driveUntilToolCall(parser, "")); +} + +TEST_F(Gemma4ReasoningSemanticRefitTest, RecoveryToolOpenerSurvivesEveryByteSplitWhileReasoningOwnsStream) { + const std::string suffix = "call:question{questions:[]}"; + + for (size_t split = 0; split <= toolStart.size(); ++split) { + SCOPED_TRACE(split); + OutputParser parser(*tokenizer, "gemma4", "gemma4", questionTools()); + parser.detectAndSetImplicitReasoningStart("prompt<|channel>thought\n"); + + std::string reasoningText; + std::optional toolCall; + + auto consume = [&](const std::string& chunk, ov::genai::GenerationFinishReason finishReason) { + auto delta = parser.parseChunk(chunk, {}, true, finishReason); + if (!delta.has_value()) + return; + if (std::holds_alternative(*delta)) + reasoningText += std::get(*delta).text; + else if (std::holds_alternative(*delta)) + toolCall = std::get(*delta); + }; + + consume("Need another tool" + toolStart.substr(0, split), ov::genai::GenerationFinishReason::NONE); + consume(toolStart.substr(split) + suffix, ov::genai::GenerationFinishReason::NONE); + for (int step = 0; step < 8 && !toolCall.has_value(); ++step) { + consume("", step == 7 ? ov::genai::GenerationFinishReason::STOP : ov::genai::GenerationFinishReason::NONE); + } + + EXPECT_EQ(reasoningText, "Need another tool"); + expectQuestionCall(toolCall); + } +} diff --git a/src/test/llm/gemma4_fast/gemma4_recovery_contract_test.cpp b/src/test/llm/gemma4_fast/gemma4_recovery_contract_test.cpp new file mode 100644 index 0000000000..21d2307fa5 --- /dev/null +++ b/src/test/llm/gemma4_fast/gemma4_recovery_contract_test.cpp @@ -0,0 +1,229 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../logging.hpp" + +#include "../../../llm/io_processing/output_parser.hpp" +#include "../../../llm/io_processing/gemma4/gemma4_tool_parser.hpp" +#include "../../../llm/ovms_text_streamer.hpp" +#include "../../platform_utils.hpp" + +using namespace ovms; + +namespace { +#ifdef _WIN32 +const std::string tokenizerPath = getWindowsRepoRootPath() + "\\src\\test\\llm_testing\\OpenVINO\\gemma-4-E4B-it-int4-ov"; +#else +const std::string tokenizerPath = "/ovms/src/test/llm_testing/OpenVINO/gemma-4-E4B-it-int4-ov"; +#endif + +const std::string questionSchema = R"({"type":"object","properties":{"questions":{"type":"array"}}})"; + +class TerminalLogCapture { + std::vector savedSinks = llm_calculator_logger->sinks(); + spdlog::level::level_enum savedLevel = llm_calculator_logger->level(); +public: + std::ostringstream output; + TerminalLogCapture() { + llm_calculator_logger->sinks() = {std::make_shared(output)}; + llm_calculator_logger->set_level(spdlog::level::warn); + } + ~TerminalLogCapture() { + llm_calculator_logger->sinks() = savedSinks; + llm_calculator_logger->set_level(savedLevel); + } +}; + +// Let the baseline compile and exercise its STOP-only end(). The repaired +// overload must consume the real terminal reason, rather than infer a budget. +template +auto finishWithReason(T& streamer, ov::genai::GenerationFinishReason reason, int) + -> decltype(streamer.end(reason), void()) { streamer.end(reason); } +template +void finishWithReason(T& streamer, ov::genai::GenerationFinishReason, long) { streamer.end(); } + +class Gemma4BareRecoveryContractTest : public ::testing::Test { +protected: + static std::unique_ptr tokenizer; + + static void SetUpTestSuite() { + const char* configured = std::getenv("GEMMA4_TOKENIZER_PATH"); + tokenizer = std::make_unique(configured ? configured : tokenizerPath); + } + + static void TearDownTestSuite() { + tokenizer.reset(); + } + + static ToolsSchemas_t questionTools() { + ToolsSchemas_t tools; + tools.emplace("question", ToolSchemaWrapper{nullptr, questionSchema}); + return tools; + } + + ParsedOutput parseWithSpecialTokensSkipped(const std::string& input) { + auto parser = std::make_shared(*tokenizer, "gemma4", "gemma4", questionTools()); + ParsedOutput result; + std::vector toolCalls; + + auto callback = [&](Delta delta, bool /*isLast*/) { + if (const auto* content = std::get_if(&delta)) { + result.content.append(content->text); + } else if (const auto* reasoning = std::get_if(&delta)) { + result.reasoning.append(reasoning->text); + } else if (const auto* call = std::get_if(&delta)) { + if (call->index >= 0) { + const auto index = static_cast(call->index); + if (index >= toolCalls.size()) + toolCalls.resize(index + 1); + auto& accumulated = toolCalls[index]; + if (call->id) + accumulated.id = *call->id; + if (call->name) + accumulated.name = *call->name; + accumulated.arguments.append(call->arguments); + } + } + return ov::genai::StreamingStatus::RUNNING; + }; + + const ov::AnyMap decodeParams{{ov::genai::skip_special_tokens.name(), true}}; + OVMSTextStreamer streamer(*tokenizer, parser, true, std::move(callback), decodeParams); + auto tensor = tokenizer->encode(input, ov::genai::add_special_tokens(false)).input_ids; + for (size_t i = 0; i < tensor.get_size(); ++i) + streamer.write(tensor.data()[i]); + streamer.end(); + + result.toolCalls = std::move(toolCalls); + return result; + } +}; + +std::unique_ptr Gemma4BareRecoveryContractTest::tokenizer; +} // namespace + +TEST_F(Gemma4BareRecoveryContractTest, IncompleteCanonicalFrameTerminatesWithLengthDiagnosticAndNoDeltas) { + ToolsSchemas_t tools; + tools.emplace("echo", ToolSchemaWrapper{nullptr, + R"({"type":"object","properties":{"text":{"type":"string"}},"required":["text"]})"}); + auto parser = std::make_shared(*tokenizer, "gemma4", "gemma4", tools); + TerminalLogCapture capture; + std::vector deltas; + OVMSTextStreamer streamer(*tokenizer, parser, true, + [&deltas](Delta delta, bool) { + deltas.push_back(std::move(delta)); + return ov::genai::StreamingStatus::RUNNING; + }, {{ov::genai::skip_special_tokens.name(), true}}); + const std::string frame = "<|tool_call>call:echo{\n\n\n\n"; + const auto encoded = tokenizer->encode(frame, ov::genai::add_special_tokens(false)).input_ids; + streamer.write(std::vector(encoded.data(), encoded.data() + encoded.get_size())); + finishWithReason(streamer, ov::genai::GenerationFinishReason::LENGTH, 0); + for (const auto& delta : deltas) { + EXPECT_FALSE(std::holds_alternative(delta)); + EXPECT_FALSE(std::holds_alternative(delta)); + } + const auto diagnostic = capture.output.str(); + EXPECT_NE(diagnostic.find("pending_tool_frame=true"), std::string::npos) << diagnostic; + EXPECT_NE(diagnostic.find("finish_reason=LENGTH"), std::string::npos) << diagnostic; + EXPECT_NE(diagnostic.find("parser_phase="), std::string::npos) << diagnostic; + EXPECT_NE(diagnostic.find("buffered_bytes="), std::string::npos) << diagnostic; + EXPECT_NE(diagnostic.find("generated_tokens=" + std::to_string(encoded.get_size())), std::string::npos) << diagnostic; + EXPECT_NE(diagnostic.find("tool_name=echo"), std::string::npos) << diagnostic; +} + +TEST_F(Gemma4BareRecoveryContractTest, AllowedToolNameFollowedByProseStaysContent) { + OutputParser parser(*tokenizer, "gemma4", "gemma4", questionTools()); + + auto delta = parser.parseChunk( + "call:question prose", {}, true, ov::genai::GenerationFinishReason::STOP); + + ASSERT_TRUE(delta.has_value()); + ASSERT_TRUE(std::holds_alternative(*delta)); + EXPECT_EQ(std::get(*delta).text, "call:question prose"); +} + +TEST_F(Gemma4BareRecoveryContractTest, LiteralCanonicalToolMarkerInProseStaysContent) { + OutputParser parser(*tokenizer, "gemma4", "gemma4", questionTools()); + const std::string input = "Documentation marker <|tool_call> is literal, not a call."; + + auto delta = parser.parseChunk( + input, {}, true, ov::genai::GenerationFinishReason::STOP); + + ASSERT_TRUE(delta.has_value()); + ASSERT_TRUE(std::holds_alternative(*delta)); + EXPECT_EQ(std::get(*delta).text, input); +} + +TEST_F(Gemma4BareRecoveryContractTest, RegistryAwareParserReleasesImpossibleBarePrefixAsContent) { + Gemma4ToolParser parser(*tokenizer, questionTools()); + + auto delta = parser.parseChunk( + "call:question prose", {}, ov::genai::GenerationFinishReason::STOP); + + ASSERT_TRUE(delta.has_value()); + ASSERT_TRUE(std::holds_alternative(*delta)); + EXPECT_EQ(std::get(*delta).text, "call:question prose"); +} + +TEST_F(Gemma4BareRecoveryContractTest, BareCallRecoverySurvivesToolNameChunkSplit) { + OutputParser parser(*tokenizer, "gemma4", "gemma4", questionTools()); + + auto first = parser.parseChunk( + "call:quest", {}, true, ov::genai::GenerationFinishReason::NONE); + EXPECT_FALSE(first.has_value()); + + auto second = parser.parseChunk( + "ion{questions:[]}", {}, true, ov::genai::GenerationFinishReason::NONE); + EXPECT_FALSE(second.has_value()); + + std::optional toolDelta; + for (int drain = 0; drain < 4 && !toolDelta.has_value(); ++drain) { + auto delta = parser.parseChunk("", {}, true, ov::genai::GenerationFinishReason::NONE); + if (delta.has_value() && std::holds_alternative(*delta)) { + toolDelta = std::move(delta); + } + } + + ASSERT_TRUE(toolDelta.has_value()); + const auto& call = std::get(*toolDelta); + EXPECT_EQ(call.index, 0); + EXPECT_EQ(call.name.value_or(""), "question"); + EXPECT_EQ(call.arguments, R"({"questions":[]})"); +} + +TEST_F(Gemma4BareRecoveryContractTest, SkipSpecialTokensStillPreservesCanonicalReasoningToToolHandoff) { + const auto parsed = parseWithSpecialTokensSkipped( + "<|channel>thought\nNeed user input" + "<|tool_call>call:question{questions:[]}"); + + EXPECT_EQ(parsed.reasoning, "Need user input"); + EXPECT_TRUE(parsed.content.empty()); + ASSERT_EQ(parsed.toolCalls.size(), 1u); + EXPECT_EQ(parsed.toolCalls[0].name, "question"); + EXPECT_EQ(parsed.toolCalls[0].arguments, R"({"questions":[]})"); +} diff --git a/src/test/llm/gemma4_overlay/BUILD b/src/test/llm/gemma4_overlay/BUILD new file mode 100644 index 0000000000..30f7a3f990 --- /dev/null +++ b/src/test/llm/gemma4_overlay/BUILD @@ -0,0 +1,29 @@ +load("//:common_settings.bzl", "COMMON_LOCAL_DEFINES", "COMMON_STATIC_LIBS_LINKOPTS", "COPTS_TESTS") + +cc_test( + name = "gemma4_chat_template_overlay_contract_test", + srcs = ["gemma4_chat_template_overlay_contract_test.cpp"], + deps = [ + "@com_google_googletest//:gtest_main", + "//src/llm:chat_template_analyzer", + "//src/llm:io_processing_input_processors", + "//third_party:genai", + ], + copts = COPTS_TESTS, + local_defines = COMMON_LOCAL_DEFINES, + linkopts = COMMON_STATIC_LIBS_LINKOPTS, + linkstatic = 1, +) + +cc_test( + name = "gemma4_google_jinja_contract_test", + srcs = ["gemma4_google_jinja_contract_test.cpp"], + deps = [ + "@com_google_googletest//:gtest_main", + "//src/llm:chat_template_analyzer", + ], + copts = COPTS_TESTS, + local_defines = COMMON_LOCAL_DEFINES, + linkopts = COMMON_STATIC_LIBS_LINKOPTS, + linkstatic = 1, +) diff --git a/src/test/llm/gemma4_overlay/gemma4_chat_template_overlay_contract_test.cpp b/src/test/llm/gemma4_overlay/gemma4_chat_template_overlay_contract_test.cpp new file mode 100644 index 0000000000..00194b6566 --- /dev/null +++ b/src/test/llm/gemma4_overlay/gemma4_chat_template_overlay_contract_test.cpp @@ -0,0 +1,128 @@ +// Copyright 2026 Intel Corporation +// Licensed under the Apache License, Version 2.0. + +#include + +#include +#include + +#include "src/llm/io_processing/chat_template/analyzer.hpp" +#include "src/llm/io_processing/input_processors/chat_template_adapter.hpp" + +using namespace ovms; + +namespace { + +ov::genai::ChatHistory buildHistory(const std::string& messagesJson) { + ov::genai::ChatHistory history; + auto container = ov::genai::JsonContainer::from_json_string(messagesJson); + for (size_t i = 0; i < container.size(); ++i) { + history.push_back(container[i]); + } + return history; +} + +} // namespace + +TEST(Gemma4ChatTemplateOverlayContractTest, ComposesUpstreamResponseFieldAndMappingCapabilities) { + const std::string templateSource = R"( + {{ '<|tool_call>call:' }} + {% if response is mapping %}{{ response }}{% endif %} + )"; + + const auto result = ChatTemplateAnalyzer::analyze(templateSource); + ASSERT_TRUE(result.detectedToolParser.has_value()); + EXPECT_EQ(result.detectedToolParser.value(), "gemma4"); + EXPECT_TRUE(result.caps.supportsToolCalls); + EXPECT_TRUE(result.caps.supportsResponseFieldInToolDefinition); + EXPECT_TRUE(result.caps.parseToolResponseJsonContent); +} + +TEST(Gemma4ChatTemplateOverlayContractTest, CurrentGoogleTemplateRequiresObjectToolArguments) { + const std::string templateSource = R"( + {{ '<|tool_call>call:' }} + {% if function['arguments'] is mapping %} + {{ function['arguments'] }} + {% elif function['arguments'] is none %} + {% else %} + {{ raise_exception('tool_calls[].function.arguments must be a JSON object (mapping), not a string') }} + {% endif %} + )"; + + const auto result = ChatTemplateAnalyzer::analyze(templateSource); + ASSERT_TRUE(result.detectedToolParser.has_value()); + EXPECT_EQ(result.detectedToolParser.value(), "gemma4"); + EXPECT_TRUE(result.caps.requiresObjectArguments); +} + +TEST(Gemma4ChatTemplateOverlayContractTest, CompatibleGemmaTemplateThatAcceptsStringArgumentsDoesNotForceConversion) { + const std::string templateSource = R"( + {{ '<|tool_call>call:' }} + {% if function['arguments'] is mapping %} + {{ function['arguments'] }} + {% elif function['arguments'] is string %} + {{ function['arguments'] }} + {% endif %} + )"; + + const auto result = ChatTemplateAnalyzer::analyze(templateSource); + ASSERT_TRUE(result.detectedToolParser.has_value()); + EXPECT_EQ(result.detectedToolParser.value(), "gemma4"); + EXPECT_FALSE(result.caps.requiresObjectArguments); +} + +TEST(Gemma4ChatTemplateOverlayContractTest, ObjectArgumentAdaptationPreservesNestedOpenAIArguments) { + auto history = buildHistory(R"([ + {"role":"assistant","content":"","tool_calls":[ + {"id":"call_repo","type":"function","function":{ + "name":"publish_review_evidence", + "arguments":"{\"head_sha\":\"798e99e04d53fba2b1c87bd6b88260f0d6c3ca83\",\"nested\":{\"dirty\":true},\"items\":[1,2]}" + }} + ]} + ])"); + + chat_template_adapter::funcArgsToObjectHistory(history); + + ASSERT_TRUE(history[0]["tool_calls"][0]["function"]["arguments"].is_object()); + const auto args = history[0]["tool_calls"][0]["function"]["arguments"]; + EXPECT_EQ(args["head_sha"].get_string(), "798e99e04d53fba2b1c87bd6b88260f0d6c3ca83"); + EXPECT_EQ(args["nested"].to_json_string(), R"({"dirty":true})"); + EXPECT_EQ(args["items"].to_json_string(), R"([1,2])"); +} + +TEST(Gemma4ChatTemplateOverlayContractTest, GooglePartsIterationKeepsToolContentString) { + const std::string templateSource = R"( + {{ '<|tool_call>call:' }} + {% if response is mapping %}{{ response }}{% endif %} + {% for part in message.content %}{{ part.get('type') }}{% endfor %} + )"; + + const auto result = ChatTemplateAnalyzer::analyze(templateSource); + ASSERT_TRUE(result.detectedToolParser.has_value()); + EXPECT_EQ(result.detectedToolParser.value(), "gemma4"); + EXPECT_TRUE(result.caps.supportsResponseFieldInToolDefinition); + EXPECT_FALSE(result.caps.parseToolResponseJsonContent); +} + +TEST(Gemma4ChatTemplateOverlayContractTest, MappingConversionPreservesOpaqueNestedToolResult) { + static const std::string expectedSha = "798e99e04d53fba2b1c87bd6b88260f0d6c3ca83"; + auto history = buildHistory(R"([ + {"role":"tool","tool_call_id":"call_repo", + "content":"{\"head_sha\":\"798e99e04d53fba2b1c87bd6b88260f0d6c3ca83\",\"nested\":{\"dirty\":true}}"} + ])"); + + chat_template_adapter::toolResponseJsonContentToObjectHistory(history); + + ASSERT_TRUE(history[0]["content"].is_object()); + EXPECT_EQ(history[0]["content"]["head_sha"].get_string(), expectedSha); + EXPECT_EQ(history[0]["content"]["nested"].to_json_string(), R"({"dirty":true})"); +} + +TEST(Gemma4ChatTemplateOverlayContractTest, MappingConversionLeavesArraysScalarsAndNonJsonUntouched) { + for (const std::string content : {"[1,2,3]", "42", "true", "not json"}) { + auto history = buildHistory(std::string("[{\"role\":\"tool\",\"content\":\"") + content + "\"}]"); + chat_template_adapter::toolResponseJsonContentToObjectHistory(history); + ASSERT_TRUE(history[0]["content"].is_string()); + EXPECT_EQ(history[0]["content"].get_string(), content); + } +} diff --git a/src/test/llm/gemma4_overlay/gemma4_google_jinja_contract_test.cpp b/src/test/llm/gemma4_overlay/gemma4_google_jinja_contract_test.cpp new file mode 100644 index 0000000000..a5e219ace0 --- /dev/null +++ b/src/test/llm/gemma4_overlay/gemma4_google_jinja_contract_test.cpp @@ -0,0 +1,83 @@ +// Copyright 2026 Intel Corporation +// Licensed under the Apache License, Version 2.0. + +#include + +#include + +#include "src/llm/io_processing/chat_template/analyzer.hpp" + +using namespace ovms; + +namespace { + +bool contains(const std::string& haystack, const std::string& needle) { + return haystack.find(needle) != std::string::npos; +} + +// Semantic oracle, not a vendored copy of the full Google template. +// Provenance: google/gemma-4-31B-it chat_template.jinja @ 68abe48 +// Visible HF title at verification time: +// fix: chat template — null handling, reasoning preservation, +// turn-tag balance, input validation (#118) +const std::string kGoogleGemma4CanonicalSemanticSnippet = R"JINJA( +{{ '<|tool_call>call:' }} +{% set ns = namespace(prev_message_type=None, prev_non_tool_role=None) %} +{% set enable_thinking = enable_thinking | default(false) %} +{% set preserve_thinking = preserve_thinking | default(false) %} +{% if 'response' in tool_data['function'] %},response:{type:<|"|>OBJECT<|"|>}{% endif %} +{% if argument is none %}{{ 'null' }}{% endif %} +{% set thinking_gate = preserve_thinking and message.get('tool_calls') %} +{% if function['arguments'] is mapping %}{{ function['arguments'] }} +{% elif function['arguments'] is none %} +{% else %}{{ raise_exception('tool_calls[].function.arguments must be a JSON object (mapping), not a string') }}{% endif %} +{% if ns.prev_message_type == 'tool_response' and enable_thinking %}{{ '<|channel>thought\n' }}{% endif %} +)JINJA"; + +// Local OVMS compatibility oracle. This intentionally models the checked-in +// fixture after #4365, not the canonical Google template byte-for-byte. +const std::string kOvmsGemma4CompatibilitySnippet = R"JINJA( +{# Modifications to original chat template: ignore response field from tool definition #} +{{ '<|tool_call>call:' }} +{% if function['arguments'] is mapping %}{{ function['arguments'] }} +{% elif function['arguments'] is string %}{{ function['arguments'] }}{% endif %} +{% for k in range(loop.index0 + 1, loop_messages | length) %} + {% if loop_messages[k]['role'] == 'tool' %}<|tool_response>response:{{ name }}{}{% endif %} +{% endfor %} +)JINJA"; + +} // namespace + +TEST(Gemma4GoogleJinjaContractTest, PinnedGoogleSemanticOracleCarriesCanonicalWireFeatures) { + const auto result = ChatTemplateAnalyzer::analyze(kGoogleGemma4CanonicalSemanticSnippet); + + ASSERT_TRUE(result.detectedToolParser.has_value()); + EXPECT_EQ(result.detectedToolParser.value(), "gemma4"); + EXPECT_TRUE(result.caps.supportsToolCalls); + EXPECT_TRUE(result.caps.requiresObjectArguments); + + EXPECT_TRUE(contains(kGoogleGemma4CanonicalSemanticSnippet, "prev_non_tool_role")); + EXPECT_TRUE(contains(kGoogleGemma4CanonicalSemanticSnippet, "preserve_thinking")); + EXPECT_TRUE(contains(kGoogleGemma4CanonicalSemanticSnippet, "argument is none")); + EXPECT_TRUE(contains(kGoogleGemma4CanonicalSemanticSnippet, "'null'")); + EXPECT_TRUE(contains(kGoogleGemma4CanonicalSemanticSnippet, "'response' in tool_data['function']")); + EXPECT_TRUE(contains(kGoogleGemma4CanonicalSemanticSnippet, "raise_exception")); + EXPECT_TRUE(contains(kGoogleGemma4CanonicalSemanticSnippet, "<|channel>thought\\n")); +} + +TEST(Gemma4GoogleJinjaContractTest, LocalCompatibilityOracleIsNotMistakenForCanonicalGoogleTemplate) { + const auto result = ChatTemplateAnalyzer::analyze(kOvmsGemma4CompatibilitySnippet); + + ASSERT_TRUE(result.detectedToolParser.has_value()); + EXPECT_EQ(result.detectedToolParser.value(), "gemma4"); + EXPECT_TRUE(result.caps.supportsToolCalls); + + // This is deliberate local compatibility: OpenAI string arguments can be + // accepted here because OVMS also has adapter tests for string->object + // conversion. The Google canonical oracle above is stricter. + EXPECT_FALSE(result.caps.requiresObjectArguments); + EXPECT_TRUE(contains(kOvmsGemma4CompatibilitySnippet, "function['arguments'] is string")); + EXPECT_FALSE(contains(kOvmsGemma4CompatibilitySnippet, "argument is none")); + EXPECT_FALSE(contains(kOvmsGemma4CompatibilitySnippet, "prev_non_tool_role")); + EXPECT_FALSE(contains(kOvmsGemma4CompatibilitySnippet, "preserve_thinking")); +} diff --git a/src/test/llm/generation_config/BUILD b/src/test/llm/generation_config/BUILD new file mode 100644 index 0000000000..e46ce9c3e9 --- /dev/null +++ b/src/test/llm/generation_config/BUILD @@ -0,0 +1,54 @@ +load("//:common_settings.bzl", "COMMON_LOCAL_DEFINES", "COMMON_STATIC_LIBS_LINKOPTS", "COPTS_TESTS") + +cc_test( + name = "gemma4_generation_contract_test", + env_inherit = ["PATH"], + srcs = ["gemma4_generation_contract_test.cpp"], + deps = [ + "@com_google_googletest//:gtest_main", + "//src:test_platform_utils", + "//src/llm:generation_config_builders", + "//third_party:genai", + "//third_party:openvino", + ], + copts = COPTS_TESTS, + local_defines = COMMON_LOCAL_DEFINES, + linkopts = COMMON_STATIC_LIBS_LINKOPTS, + linkstatic = 1, +) + +cc_test( + name = "gemma4_prompt_state_generation_contract_test", + env_inherit = ["PATH"], + srcs = ["gemma4_prompt_state_generation_contract_test.cpp"], + deps = [ + "@com_google_googletest//:gtest_main", + "//src/llm:generation_config_builders", + "//src/llm:io_processing_input_processors", + "//third_party:genai", + "//third_party:openvino", + ], + copts = COPTS_TESTS, + local_defines = COMMON_LOCAL_DEFINES, + linkopts = COMMON_STATIC_LIBS_LINKOPTS, + linkstatic = 1, +) + +cc_test( + name = "openai_parallel_tool_calls_contract_test", + env_inherit = ["PATH"], + srcs = ["openai_parallel_tool_calls_contract_test.cpp"], + deps = [ + "@com_google_googletest//:gtest_main", + "@com_github_tencent_rapidjson//:rapidjson", + "//src:test_platform_utils", + "//src/llm:openai_completions_api_handler", + "//src/llm:openai_responses_handler", + "//third_party:genai", + "//third_party:openvino", + ], + copts = COPTS_TESTS, + local_defines = COMMON_LOCAL_DEFINES, + linkopts = COMMON_STATIC_LIBS_LINKOPTS, + linkstatic = 1, +) diff --git a/src/test/llm/generation_config/gemma4_generation_contract_test.cpp b/src/test/llm/generation_config/gemma4_generation_contract_test.cpp new file mode 100644 index 0000000000..9905e0b12c --- /dev/null +++ b/src/test/llm/generation_config/gemma4_generation_contract_test.cpp @@ -0,0 +1,465 @@ +// Copyright 2026 Intel Corporation +// Licensed under the Apache License, Version 2.0. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "src/llm/io_processing/generation_config_builder.hpp" +#include "src/test/platform_utils.hpp" + +using namespace ovms; +using Structured = ov::genai::StructuredOutputConfig; + +TEST(Gemma4WhitespaceContractTest, RealBuilderSerializesBoundInsideEveryToolSchema) { + for (const std::string choice : {"auto", "required", "echo"}) { + for (bool parallel : {false, true}) { + SCOPED_TRACE(choice + (parallel ? ":parallel" : ":single")); + OpenAIRequest request; + request.toolChoice = choice; + request.parallelToolCalls = parallel; + request.toolNameSchemaMap.emplace("echo", ToolSchemaWrapper{nullptr, + R"({"type":"object","properties":{"text":{"type":"string"}},"required":["text"]})"}); + Gemma4GenerationConfigBuilder builder({}, true, STANDARD); + builder.parseConfigFromRequest(request); + const auto config = builder.getConfig(); + const auto& root = std::get( + config.structured_output_config.value().structural_tags_config.value()); + const auto json = std::visit([](const auto& tag) { + return Structured::structural_tag_to_json(tag); + }, root); + EXPECT_NE(json.find("\"max_whitespace_cnt\": 2"), std::string::npos) << json; + if (const char* directory = std::getenv("GEMMA4_WHITESPACE_GRAMMAR_DIR")) { + std::filesystem::create_directories(directory); + const auto name = choice + (parallel ? "-parallel.json" : "-single.json"); + std::ofstream(std::filesystem::path(directory) / name) << json; + } + } + } +} + +namespace { +const std::string emptySchema = R"({"type":"object","properties":{},"additionalProperties":false})"; +const std::string responseSchema = R"({"type":"structural_tag","format":{"type":"json_schema","json_schema":{"type":"object","properties":{"answer":{"type":"string"}}}}})"; +const std::string openCodeQuestionSchema = R"({"type":"object","properties":{"questions":{"type":"array","items":{"type":"object","properties":{"question":{"type":"string"},"header":{"type":"string"},"options":{"type":"array","items":{"type":"object","properties":{"label":{"type":"string"},"description":{"type":"string"}},"required":["label","description"]}},"multiple":{"type":"boolean"},"custom":{"type":"boolean"}},"required":["question","header","options"]}}},"required":["questions"]})"; + +OpenAIRequest requestWithTools(const std::string& choice) { + OpenAIRequest request; + request.toolChoice = choice; + request.toolNameSchemaMap.emplace("first", ToolSchemaWrapper{nullptr, emptySchema}); + request.toolNameSchemaMap.emplace("second", ToolSchemaWrapper{nullptr, emptySchema}); + return request; +} + +const Structured::StructuralTag& rootGrammar(const ov::genai::GenerationConfig& config) { + return std::get( + config.structured_output_config.value().structural_tags_config.value()); +} + +template +const T& grammar(const ov::genai::GenerationConfig& config) { + const auto& root = rootGrammar(config); + if (const auto* sequence = std::get_if>(&root)) + return *std::get>((*sequence)->elements.back()); + if (const auto* alternatives = std::get_if>(&root)) { + for (const auto& element : (*alternatives)->elements) { + if (const auto* tags = std::get_if>(&element)) + return **tags; + if (const auto* sequence = std::get_if>(&element)) + return *std::get>((*sequence)->elements.back()); + } + } + return *std::get>(root); +} + +const Structured::TriggeredTags& autoGrammar(const ov::genai::GenerationConfig& config) { + return *std::get>(rootGrammar(config)); +} + +std::string grammarString(const ov::genai::GenerationConfig& config) { + return std::visit([](const auto& value) { + return Structured::structural_tag_to_string(value); + }, rootGrammar(config)); +} + +std::string availableGemmaTokenizerPath() { + if (const char* configured = std::getenv("GEMMA4_TOKENIZER_PATH"); configured && std::filesystem::exists(configured)) + return configured; + const auto fixture = getGenericFullPathForSrcTest("/ovms/src/test/llm_testing/OpenVINO/gemma-4-E4B-it-int4-ov"); + return std::filesystem::exists(fixture) ? fixture : std::string{}; +} + +bool pinnedXGrammarAccepts(const std::string& format, const std::string& text) { + const char* python = std::getenv("GEMMA4_XGRAMMAR_PYTHON"); + const char* pythonPath = std::getenv("GEMMA4_XGRAMMAR_PYTHONPATH"); + if (!python || !pythonPath) + throw std::runtime_error("GEMMA4_XGRAMMAR_PYTHON/PYTHONPATH are required"); + const auto root = std::filesystem::temp_directory_path() / "ovms-gemma4-xgrammar-semantic"; + std::filesystem::create_directories(root); + const auto script = root / "accept.py"; + const auto grammar = root / "grammar.json"; + const auto input = root / "input.txt"; + const auto result = root / "result.txt"; + std::ofstream(script) << R"PY(import importlib.metadata, json, sys +import xgrammar as xg +assert importlib.metadata.version("xgrammar") == "0.1.31" +fmt=json.load(open(sys.argv[1], encoding="utf-8")) +text=open(sys.argv[2], encoding="utf-8").read() +ti=xg.TokenizerInfo([chr(i) for i in range(128)], xg.VocabType.RAW, vocab_size=128) +grammar=xg.Grammar.from_structural_tag({"type":"structural_tag","format":fmt}) +matcher=xg.GrammarMatcher(xg.GrammarCompiler(ti).compile_grammar(grammar), terminate_without_stop_token=True) +open(sys.argv[3], "w", encoding="ascii").write("1" if matcher.accept_string(text) else "0") +)PY"; + std::ofstream(grammar) << format; + std::ofstream(input) << text; + const std::string command = "cmd.exe /d /s /c \"set PYTHONPATH=" + std::string(pythonPath) + "&& \"" + python + + "\" \"" + script.string() + "\" \"" + grammar.string() + "\" \"" + input.string() + "\" \"" + result.string() + "\"\""; + if (std::system(command.c_str()) != 0) + throw std::runtime_error("pinned xgrammar semantic probe failed"); + std::ifstream output(result); + char accepted = '0'; + output >> accepted; + return accepted == '1'; +} +} // namespace + +TEST(Gemma4GenerationContractTest, AbsentToolsAndNonePreserveResponseFormat) { + for (bool guided : {false, true}) { + for (bool response : {false, true}) { + for (bool tools : {false, true}) { + auto request = tools ? requestWithTools("none") : OpenAIRequest{}; + if (response) + request.responseFormat = responseSchema; + GenerationConfigBuilder builder({}, "gemma4", guided, STANDARD); + builder.parseConfigFromRequest(request); + EXPECT_EQ(builder.getConfig().structured_output_config.has_value(), response); + } + } + } +} + +TEST(Gemma4GenerationContractTest, ValidationFallbackPolicyIsGemmaSpecific) { + auto request = requestWithTools("required"); + + GenerationConfigBuilder gemma({}, "gemma4", false, STANDARD); + gemma.parseConfigFromRequest(request); + ASSERT_TRUE(gemma.getConfig().structured_output_config.has_value()); + gemma.unsetStructuredOutputConfig(); + EXPECT_TRUE(gemma.getConfig().structured_output_config.has_value()); + + GenerationConfigBuilder hermes({}, "hermes3", false, STANDARD); + hermes.parseConfigFromRequest(request); + ASSERT_TRUE(hermes.getConfig().structured_output_config.has_value()); + hermes.unsetStructuredOutputConfig(); + EXPECT_FALSE(hermes.getConfig().structured_output_config.has_value()); +} + +TEST(Gemma4GenerationContractTest, HardChoicesAllowReasoningBeforeMandatoryToolSelection) { + const auto tokenizerPath = availableGemmaTokenizerPath(); + if (tokenizerPath.empty()) + GTEST_SKIP() << "Gemma4 tokenizer fixture is not available"; + for (const std::string choice : {"required", "second"}) { + SCOPED_TRACE(choice); + auto request = requestWithTools(choice); + GenerationConfigBuilder builder({}, "gemma4", false, STANDARD); + builder.parseConfigFromRequest(request); + const auto& root = rootGrammar(builder.getConfig()); + ASSERT_TRUE(std::holds_alternative>(root)); + const auto& alternatives = *std::get>(root); + ASSERT_EQ(alternatives.elements.size(), 2u); + const auto& toolsOnly = *std::get>(alternatives.elements[0]); + EXPECT_TRUE(toolsOnly.at_least_one); + EXPECT_EQ(toolsOnly.tags.size(), choice == "second" ? 1u : 2u); + if (choice == "second") + EXPECT_EQ(toolsOnly.tags[0].begin, "<|tool_call>call:second"); + const auto& thoughtThenTools = *std::get>(alternatives.elements[1]); + ASSERT_EQ(thoughtThenTools.elements.size(), 2u); + const auto& thought = *std::get>(thoughtThenTools.elements[0]); + EXPECT_EQ(thought.begin, "<|channel>thought\n"); + EXPECT_EQ(thought.end, ""); + const auto& toolsAfterThought = *std::get>(thoughtThenTools.elements[1]); + EXPECT_TRUE(toolsAfterThought.at_least_one); + EXPECT_EQ(toolsAfterThought.tags.size(), choice == "second" ? 1u : 2u); + if (choice == "second") + EXPECT_EQ(toolsAfterThought.tags[0].begin, "<|tool_call>call:second"); + + ov::genai::Tokenizer tokenizer(tokenizerPath); + EXPECT_NO_THROW(builder.validateStructuredOutputConfig(tokenizer)); + } +} + +TEST(Gemma4GenerationContractTest, HardChoiceCannotBeClearedButAutoMayFallbackAfterValidationFailure) { + for (const std::string choice : {"required", "second"}) { + auto request = requestWithTools(choice); + GenerationConfigBuilder builder({}, "gemma4", false, STANDARD); + builder.parseConfigFromRequest(request); + ASSERT_TRUE(builder.getConfig().structured_output_config.has_value()); + EXPECT_NO_THROW(builder.unsetStructuredOutputConfig()) << choice; + EXPECT_TRUE(builder.getConfig().structured_output_config.has_value()) << choice; + } + + auto autoRequest = requestWithTools("auto"); + GenerationConfigBuilder autoBuilder({}, "gemma4", false, STANDARD); + autoBuilder.parseConfigFromRequest(autoRequest); + ASSERT_TRUE(autoBuilder.getConfig().structured_output_config.has_value()); + EXPECT_NO_THROW(autoBuilder.unsetStructuredOutputConfig()); + EXPECT_FALSE(autoBuilder.getConfig().structured_output_config.has_value()); +} + +TEST(Gemma4GenerationContractTest, AutoUsesTriggeredToolGrammarAndHardChoicesStayImmediateAndRepeatable) { + for (bool guided : {false, true}) { + for (const std::string choice : {"auto", "required", "second"}) { + SCOPED_TRACE(choice); + auto request = requestWithTools(choice); + GenerationConfigBuilder builder({}, "gemma4", guided, STANDARD); + builder.parseConfigFromRequest(request); + ASSERT_TRUE(builder.getConfig().structured_output_config.has_value()); + if (choice == "auto") { + const auto& triggered = autoGrammar(builder.getConfig()); + ASSERT_EQ(triggered.triggers.size(), 1u); + EXPECT_EQ(triggered.triggers[0], "<|tool_call>"); + EXPECT_FALSE(triggered.at_least_one) + << "Gemma4 auto must permit ordinary prose without a tool call"; + EXPECT_FALSE(triggered.stop_after_first); + ASSERT_EQ(triggered.tags.size(), 2u); + + std::set begins; + for (const auto& tag : triggered.tags) { + begins.insert(tag.begin); + EXPECT_EQ(tag.end, ""); + ASSERT_TRUE(std::holds_alternative(tag.content)); + EXPECT_EQ(std::get(tag.content).value, emptySchema); + } + EXPECT_EQ(begins, (std::set{"<|tool_call>call:first", "<|tool_call>call:second"})); + } else { + const auto& tags = grammar(builder.getConfig()); + EXPECT_TRUE(tags.at_least_one); + EXPECT_FALSE(tags.stop_after_first); + EXPECT_TRUE(tags.separator.empty()); + ASSERT_EQ(tags.tags.size(), choice == "second" ? 1u : 2u); + if (choice == "second") + EXPECT_EQ(tags.tags[0].begin, "<|tool_call>call:second"); + } + } + } +} + +TEST(Gemma4GenerationContractTest, ParallelToolCallsControlsGrammarRepeatability) { + for (const std::string choice : {"auto", "required", "second"}) { + for (bool parallel : {false, true}) { + SCOPED_TRACE(choice + std::string(parallel ? ":parallel" : ":single")); + auto request = requestWithTools(choice); + request.parallelToolCalls = parallel; + GenerationConfigBuilder builder({}, "gemma4", false, STANDARD); + builder.parseConfigFromRequest(request); + + if (choice == "auto") { + const auto& triggered = autoGrammar(builder.getConfig()); + EXPECT_EQ(triggered.stop_after_first, !parallel); + } else { + const auto& root = rootGrammar(builder.getConfig()); + ASSERT_TRUE(std::holds_alternative>(root)); + const auto& alternatives = *std::get>(root); + ASSERT_EQ(alternatives.elements.size(), 2u); + const auto& toolsOnly = *std::get>(alternatives.elements[0]); + EXPECT_EQ(toolsOnly.stop_after_first, !parallel); + const auto& thoughtThenTools = *std::get>(alternatives.elements[1]); + const auto& toolsAfterThought = *std::get>(thoughtThenTools.elements[1]); + EXPECT_EQ(toolsAfterThought.stop_after_first, !parallel); + } + } + } +} + +TEST(Gemma4GenerationContractTest, AutoTriggeredGrammarValidatesWithGemmaTokenizer) { + const auto tokenizerPath = availableGemmaTokenizerPath(); + if (tokenizerPath.empty()) + GTEST_SKIP() << "Gemma4 tokenizer fixture is not available"; + auto request = requestWithTools("auto"); + GenerationConfigBuilder builder({}, "gemma4", false, STANDARD); + builder.parseConfigFromRequest(request); + ASSERT_TRUE(builder.getConfig().structured_output_config.has_value()); + + ov::genai::Tokenizer tokenizer(tokenizerPath); + EXPECT_NO_THROW(builder.validateStructuredOutputConfig(tokenizer)); +} + +TEST(Gemma4GenerationContractTest, OpenCodeQuestionSchemaIsEnforcedForAutoAndRequired) { + for (const std::string choice : {"auto", "required"}) { + OpenAIRequest request; + request.toolChoice = choice; + request.toolNameSchemaMap.emplace("question", ToolSchemaWrapper{nullptr, openCodeQuestionSchema}); + GenerationConfigBuilder builder({}, "gemma4", true, STANDARD); + builder.parseConfigFromRequest(request); + + if (choice == "auto") { + const auto& triggered = autoGrammar(builder.getConfig()); + ASSERT_EQ(triggered.tags.size(), 1u); + EXPECT_EQ(triggered.tags[0].begin, "<|tool_call>call:question"); + EXPECT_EQ(std::get(triggered.tags[0].content).value, openCodeQuestionSchema); + EXPECT_FALSE(triggered.at_least_one); + } else { + const auto& tags = grammar(builder.getConfig()); + ASSERT_EQ(tags.tags.size(), 1u); + EXPECT_EQ(tags.tags[0].begin, "<|tool_call>call:question"); + EXPECT_EQ(std::get(tags.tags[0].content).value, openCodeQuestionSchema); + } + } +} + +TEST(Gemma4GenerationContractTest, ImpossibleHardChoiceIsRejected) { + for (bool guided : {false, true}) { + for (const std::string choice : {"required", "missing"}) { + OpenAIRequest request; + request.toolChoice = choice; + GenerationConfigBuilder builder({}, "gemma4", guided, STANDARD); + EXPECT_THROW(builder.parseConfigFromRequest(request), std::invalid_argument); + } + auto request = requestWithTools("missing"); + GenerationConfigBuilder builder({}, "gemma4", guided, STANDARD); + EXPECT_THROW(builder.parseConfigFromRequest(request), std::invalid_argument); + } +} + +TEST(Gemma4GenerationContractTest, ResponseFormatAndActiveToolsCannotSilentlyReplaceConstraints) { + for (bool guided : {false, true}) { + for (const std::string choice : {"auto", "required", "second"}) { + auto request = requestWithTools(choice); + request.responseFormat = responseSchema; + GenerationConfigBuilder builder({}, "gemma4", guided, STANDARD); + EXPECT_THROW(builder.parseConfigFromRequest(request), std::invalid_argument); + } + } +} + +TEST(Gemma4GenerationContractTest, ObjectSchemaIsPassedWithoutLosingNestedConstraints) { + const std::string schema = R"({"type":"object","properties":{"nested":{"type":"object","properties":{"x":{"enum":[1,2]}}},"array":{"type":"array","items":{"type":["number","boolean","null"]}},"optional":{"type":"string"}},"required":["nested"],"additionalProperties":false})"; + for (const std::string choice : {"auto", "first"}) { + auto request = requestWithTools(choice); + request.toolNameSchemaMap.erase("second"); + request.toolNameSchemaMap["first"].stringRepr = schema; + GenerationConfigBuilder builder({}, "gemma4", false, STANDARD); + builder.parseConfigFromRequest(request); + if (choice == "auto") { + const auto& triggered = autoGrammar(builder.getConfig()); + ASSERT_EQ(triggered.tags.size(), 1u); + EXPECT_EQ(std::get(triggered.tags[0].content).value, schema); + } else { + const auto& tags = grammar(builder.getConfig()); + ASSERT_EQ(tags.tags.size(), 1u); + EXPECT_EQ(std::get(tags.tags[0].content).value, schema); + } + } +} + +TEST(Gemma4GenerationContractTest, RejectsToolNamesThatItsParserCannotExecute) { + for (const std::string choice : {"auto", "required"}) { + for (const std::string name : {"bad name", "", "bad:name"}) { + OpenAIRequest request; + request.toolChoice = choice; + request.toolNameSchemaMap.emplace(name, ToolSchemaWrapper{nullptr, emptySchema}); + GenerationConfigBuilder builder({}, "gemma4", false, STANDARD); + EXPECT_THROW(builder.parseConfigFromRequest(request), std::invalid_argument) << choice << ":" << name; + } + } +} + +TEST(Gemma4GenerationContractTest, RejectsEmptyToolSchemasForAutoAndHardChoices) { + for (const std::string choice : {"auto", "required", "first"}) { + auto request = requestWithTools(choice); + request.toolNameSchemaMap["first"].stringRepr.clear(); + if (choice == "first") + request.toolNameSchemaMap.erase("second"); + GenerationConfigBuilder builder({}, "gemma4", false, STANDARD); + EXPECT_THROW(builder.parseConfigFromRequest(request), std::invalid_argument) << choice; + } +} + +TEST(Gemma4GenerationContractTest, GeneratedToolGrammarsNeverUseEmptyConstString) { + for (const std::string choice : {"auto", "required", "second"}) { + auto request = requestWithTools(choice); + GenerationConfigBuilder builder({}, "gemma4", false, STANDARD); + builder.parseConfigFromRequest(request); + ASSERT_TRUE(builder.getConfig().structured_output_config.has_value()); + EXPECT_EQ(grammarString(builder.getConfig()).find("ConstString(\"\")"), std::string::npos) << choice; + } +} + +TEST(Gemma4GenerationContractTest, SingleToolParallelEnabledAllowsRepeatedSameToolGrammar) { + for (const std::string choice : {"auto", "required"}) { + SCOPED_TRACE(choice); + OpenAIRequest request; + request.toolChoice = choice; + request.parallelToolCalls = true; + request.toolNameSchemaMap.emplace("weather", ToolSchemaWrapper{nullptr, emptySchema}); + GenerationConfigBuilder builder({}, "gemma4", false, STANDARD); + builder.parseConfigFromRequest(request); + ASSERT_TRUE(builder.getConfig().structured_output_config.has_value()); + + if (choice == "auto") { + const auto& triggered = autoGrammar(builder.getConfig()); + EXPECT_FALSE(triggered.stop_after_first) + << "parallel enabled must not stop after first trigger"; + EXPECT_EQ(triggered.tags.size(), 1u); + } else { + const auto& tags = grammar(builder.getConfig()); + EXPECT_FALSE(tags.stop_after_first) + << "parallel enabled must not stop after first tag"; + EXPECT_TRUE(tags.at_least_one); + EXPECT_EQ(tags.tags.size(), 1u); + } + } +} + +TEST(Gemma4GenerationContractTest, PinnedXGrammarControlsSameToolMultiplicityWithStopAfterFirst) { + if (!std::getenv("GEMMA4_XGRAMMAR_PYTHON") || !std::getenv("GEMMA4_XGRAMMAR_PYTHONPATH")) + GTEST_SKIP() << "pinned xgrammar v0.1.31 probe environment is not configured"; + const std::string call = R"(<|tool_call>call:weather{})"; + for (const std::string choice : {"auto", "required"}) { + for (bool parallel : {true, false}) { + SCOPED_TRACE(choice + std::string(parallel ? ":parallel" : ":single")); + OpenAIRequest request; + request.toolChoice = choice; + request.parallelToolCalls = parallel; + request.toolNameSchemaMap.emplace("weather", ToolSchemaWrapper{nullptr, emptySchema}); + GenerationConfigBuilder builder({}, "gemma4", false, STANDARD); + builder.parseConfigFromRequest(request); + const auto format = std::visit([](const auto& value) { + return Structured::structural_tag_to_json(value); + }, rootGrammar(builder.getConfig())); + EXPECT_EQ(pinnedXGrammarAccepts(format, call + call), parallel); + } + } +} + +TEST(Gemma4GenerationContractTest, SingleToolParallelDisabledPermitsSingleCall) { + for (const std::string choice : {"auto", "required"}) { + SCOPED_TRACE(choice); + OpenAIRequest request; + request.toolChoice = choice; + request.parallelToolCalls = false; + request.toolNameSchemaMap.emplace("weather", ToolSchemaWrapper{nullptr, emptySchema}); + GenerationConfigBuilder builder({}, "gemma4", false, STANDARD); + builder.parseConfigFromRequest(request); + ASSERT_TRUE(builder.getConfig().structured_output_config.has_value()); + + if (choice == "auto") { + const auto& triggered = autoGrammar(builder.getConfig()); + EXPECT_TRUE(triggered.stop_after_first); + EXPECT_EQ(triggered.tags.size(), 1u) + << "single tool without parallel must have exactly one tag"; + } else { + const auto& tags = grammar(builder.getConfig()); + EXPECT_TRUE(tags.stop_after_first); + EXPECT_EQ(tags.tags.size(), 1u) + << "single tool without parallel must have exactly one tag"; + } + } +} diff --git a/src/test/llm/generation_config/gemma4_prompt_state_generation_contract_test.cpp b/src/test/llm/generation_config/gemma4_prompt_state_generation_contract_test.cpp new file mode 100644 index 0000000000..b754ebf7d4 --- /dev/null +++ b/src/test/llm/generation_config/gemma4_prompt_state_generation_contract_test.cpp @@ -0,0 +1,82 @@ +// Copyright 2026 Intel Corporation +// Licensed under the Apache License, Version 2.0. + +#include +#include + +#include +#include +#include + +#include "src/llm/io_processing/generation_config_builder.hpp" +#include "src/llm/io_processing/input_processors/chat_template_processor.hpp" + +using namespace ovms; +using Structured = ov::genai::StructuredOutputConfig; + +namespace { +const std::string emptySchema = R"({"type":"object","properties":{},"additionalProperties":false})"; + +OpenAIRequest requestWithTools(const std::string& choice) { + OpenAIRequest request; + request.toolChoice = choice; + request.toolNameSchemaMap.emplace("first", ToolSchemaWrapper{nullptr, emptySchema}); + request.toolNameSchemaMap.emplace("second", ToolSchemaWrapper{nullptr, emptySchema}); + return request; +} + +const Structured::StructuralTag& rootGrammar(const ov::genai::GenerationConfig& config) { + return std::get( + config.structured_output_config.value().structural_tags_config.value()); +} +} // namespace + +TEST(Gemma4PromptStateGenerationContractTest, OpenPromptReasoningUsesRequiredTriggeredNativeToolGrammar) { + for (const std::string choice : {"required", "second"}) { + SCOPED_TRACE(choice); + auto request = requestWithTools(choice); + GenerationConfigBuilder builder({}, "gemma4", false, STANDARD); + builder.parseConfigFromRequest(request); + + ASSERT_TRUE(std::holds_alternative>(rootGrammar(builder.getConfig()))); + + const bool changed = adaptGemma4HardToolGrammarForRenderedPrompt( + builder.getConfig(), "<|turn>model\n<|channel>thought\n"); + EXPECT_TRUE(changed); + + const auto& root = rootGrammar(builder.getConfig()); + ASSERT_TRUE(std::holds_alternative>(root)); + const auto& triggered = *std::get>(root); + ASSERT_EQ(triggered.triggers.size(), 1u); + EXPECT_EQ(triggered.triggers[0], "<|tool_call>"); + EXPECT_TRUE(triggered.at_least_one); + EXPECT_FALSE(triggered.stop_after_first); + ASSERT_EQ(triggered.tags.size(), choice == "second" ? 1u : 2u); + if (choice == "second") + EXPECT_EQ(triggered.tags[0].begin, "<|tool_call>call:second"); + } +} + +TEST(Gemma4PromptStateGenerationContractTest, OrdinaryPromptKeepsImmediateHardGrammar) { + auto request = requestWithTools("required"); + GenerationConfigBuilder builder({}, "gemma4", false, STANDARD); + builder.parseConfigFromRequest(request); + + const bool changed = adaptGemma4HardToolGrammarForRenderedPrompt( + builder.getConfig(), "<|turn>user\nUse a tool\n<|turn>model\n"); + EXPECT_FALSE(changed); + EXPECT_TRUE(std::holds_alternative>(rootGrammar(builder.getConfig()))); +} + +TEST(Gemma4PromptStateGenerationContractTest, OpenPromptReasoningPreservesSingleCallPolicy) { + auto request = requestWithTools("required"); + request.parallelToolCalls = false; + GenerationConfigBuilder builder({}, "gemma4", false, STANDARD); + builder.parseConfigFromRequest(request); + + ASSERT_TRUE(adaptGemma4HardToolGrammarForRenderedPrompt( + builder.getConfig(), "prefix<|channel>thought\n")); + const auto& triggered = *std::get>(rootGrammar(builder.getConfig())); + EXPECT_TRUE(triggered.at_least_one); + EXPECT_TRUE(triggered.stop_after_first); +} diff --git a/src/test/llm/generation_config/openai_parallel_tool_calls_contract_test.cpp b/src/test/llm/generation_config/openai_parallel_tool_calls_contract_test.cpp new file mode 100644 index 0000000000..89a8fea4fd --- /dev/null +++ b/src/test/llm/generation_config/openai_parallel_tool_calls_contract_test.cpp @@ -0,0 +1,187 @@ +// Copyright 2026 Intel Corporation +// Licensed under the Apache License, Version 2.0. + +#include +#include +#include +#include +#include + +#include +#include + +#include "src/llm/apis/openai_completions.hpp" +#include "src/llm/apis/openai_responses.hpp" +#include "src/test/platform_utils.hpp" + +using namespace ovms; + +namespace { + +std::string requestJson(const std::string& parallelField) { + return std::string(R"({ + "model": "gemma4", + "messages": [{"role": "user", "content": "Use tools if needed"}], + "tools": [{ + "type": "function", + "function": { + "name": "first", + "parameters": {"type": "object", "properties": {}, "additionalProperties": false} + } + }])") + parallelField + "}"; +} + +struct ParsedParallelPolicy { + absl::Status status; + bool parallelToolCalls{true}; +}; + +ov::genai::Tokenizer makeTokenizer() { + return ov::genai::Tokenizer(getGenericFullPathForSrcTest( + "/ovms/src/test/llm_testing/facebook/opt-125m")); +} + +ParsedParallelPolicy parseParallelPolicy(const std::string& parallelField) { + rapidjson::Document doc; + const std::string json = requestJson(parallelField); + doc.Parse(json.c_str()); + if (doc.HasParseError()) { + return {absl::InvalidArgumentError("test JSON failed to parse"), true}; + } + + OpenAIChatCompletionsHandler handler( + doc, + Endpoint::CHAT_COMPLETIONS, + std::chrono::system_clock::now(), + makeTokenizer()); + + auto status = handler.parseRequest( + /*maxTokensLimit=*/std::nullopt, + /*bestOfLimit=*/0, + /*maxModelLength=*/std::nullopt); + return {status, handler.getRequest().parallelToolCalls}; +} + +absl::Status parseChatRequestWithoutTools(const std::string& toolChoiceJson) { + rapidjson::Document doc; + const std::string json = std::string(R"({ + "model": "gemma4", + "messages": [{"role": "user", "content": "You must use a tool"}], + "tool_choice": )") + toolChoiceJson + "}"; + doc.Parse(json.c_str()); + if (doc.HasParseError()) + return absl::InvalidArgumentError("test JSON failed to parse"); + + OpenAIChatCompletionsHandler handler( + doc, + Endpoint::CHAT_COMPLETIONS, + std::chrono::system_clock::now(), + makeTokenizer()); + return handler.parseRequest( + /*maxTokensLimit=*/std::nullopt, + /*bestOfLimit=*/0, + /*maxModelLength=*/std::nullopt); +} + +absl::Status parseResponsesRequestWithoutTools(const std::string& toolChoiceJson) { + rapidjson::Document doc; + const std::string json = std::string(R"({ + "model": "gemma4", + "input": "You must use a tool", + "tool_choice": )") + toolChoiceJson + "}"; + doc.Parse(json.c_str()); + if (doc.HasParseError()) + return absl::InvalidArgumentError("test JSON failed to parse"); + + OpenAIResponsesHandler handler( + doc, + Endpoint::RESPONSES, + std::chrono::system_clock::now(), + makeTokenizer()); + return handler.parseRequest( + /*maxTokensLimit=*/std::nullopt, + /*bestOfLimit=*/0, + /*maxModelLength=*/std::nullopt); +} + +void expectInvalidArgument(const absl::Status& status) { + EXPECT_FALSE(status.ok()); + EXPECT_EQ(status.code(), absl::StatusCode::kInvalidArgument); +} + +} // namespace + +TEST(OpenAIParallelToolCallsContractTest, DefaultsToEnabledWhenFieldIsAbsent) { + const auto result = parseParallelPolicy(""); + ASSERT_TRUE(result.status.ok()) << result.status; + EXPECT_TRUE(result.parallelToolCalls); +} + +TEST(OpenAIParallelToolCallsContractTest, ParsesExplicitBooleanPolicy) { + const auto enabled = parseParallelPolicy(", \"parallel_tool_calls\": true"); + ASSERT_TRUE(enabled.status.ok()) << enabled.status; + EXPECT_TRUE(enabled.parallelToolCalls); + + const auto disabled = parseParallelPolicy(", \"parallel_tool_calls\": false"); + ASSERT_TRUE(disabled.status.ok()) << disabled.status; + EXPECT_FALSE(disabled.parallelToolCalls); +} + +TEST(OpenAIParallelToolCallsContractTest, RejectsNonBooleanValues) { + for (const std::string value : {"1", "\"no\"", "[]", "{}"}) { + SCOPED_TRACE(value); + const auto result = parseParallelPolicy(", \"parallel_tool_calls\": " + value); + EXPECT_FALSE(result.status.ok()); + EXPECT_EQ(result.status.code(), absl::StatusCode::kInvalidArgument); + } +} + +TEST(OpenAIParallelToolCallsContractTest, HardToolChoiceWithoutToolsFailsClosedForChatCompletions) { + expectInvalidArgument(parseChatRequestWithoutTools("\"required\"")); + expectInvalidArgument(parseChatRequestWithoutTools( + R"({"type":"function","function":{"name":"first"}})")); +} + +TEST(OpenAIParallelToolCallsContractTest, HardToolChoiceWithoutToolsFailsClosedForResponses) { + expectInvalidArgument(parseResponsesRequestWithoutTools("\"required\"")); + expectInvalidArgument(parseResponsesRequestWithoutTools( + R"({"type":"function","name":"first"})")); +} + +TEST(OpenAIParallelToolCallsContractTest, ResponsesPreservesPolicyInRequestAndResponseObject) { + rapidjson::Document doc; + const std::string json = R"({ + "model": "gemma4", + "input": "Use tools if needed", + "parallel_tool_calls": false, + "tools": [{ + "type": "function", + "name": "first", + "parameters": {"type": "object", "properties": {}, "additionalProperties": false} + }] + })"; + doc.Parse(json.c_str()); + ASSERT_FALSE(doc.HasParseError()); + + OpenAIResponsesHandler handler( + doc, + Endpoint::RESPONSES, + std::chrono::system_clock::now(), + makeTokenizer()); + + ASSERT_TRUE(handler.parseRequest( + /*maxTokensLimit=*/std::nullopt, + /*bestOfLimit=*/0, + /*maxModelLength=*/std::nullopt).ok()); + EXPECT_FALSE(handler.getRequest().parallelToolCalls); + + const std::vector deltas; + const std::string response = handler.serializeUnaryResponse( + deltas, ov::genai::GenerationFinishReason::STOP); + rapidjson::Document responseDoc; + responseDoc.Parse(response.c_str()); + ASSERT_FALSE(responseDoc.HasParseError()) << response; + ASSERT_TRUE(responseDoc.HasMember("parallel_tool_calls")); + ASSERT_TRUE(responseDoc["parallel_tool_calls"].IsBool()); + EXPECT_FALSE(responseDoc["parallel_tool_calls"].GetBool()); +} From ff6899dbf77099dfb9381277d1aa571bb0a472b9 Mon Sep 17 00:00:00 2001 From: DassaultFalconKing Date: Tue, 15 Sep 2026 01:10:55 +0200 Subject: [PATCH 2/2] docs(gemma4): recover upstream rationale and prepare Jenkins license gates --- docs/gemma4/.gitattributes | 1 + docs/gemma4/UPSTREAM-HANDOFF.md | 8 +- .../genai-gemma4-bounded-whitespace.patch | 3 + .../gemma4/evidence/benchmark/requests.ndjson | 14 -- docs/gemma4/evidence/benchmark/summary.csv | 15 -- docs/gemma4/evidence/pr-preflight.json | 27 +++ docs/gemma4/evidence/raw-traces.tar.gz | Bin 0 -> 13907 bytes .../evidence/tools/parallel-response.txt | 1 - .../gemma4/evidence/tools/single-response.txt | 1 - .../gemma4/evidence/tools/stream-response.txt | 9 - .../evidence/transfer-source-inventory.json | 216 ++++++++++++------ src/test/llm/gemma4_fast/BUILD | 15 ++ src/test/llm/gemma4_overlay/BUILD | 15 ++ src/test/llm/generation_config/BUILD | 15 ++ 14 files changed, 225 insertions(+), 115 deletions(-) delete mode 100644 docs/gemma4/evidence/benchmark/requests.ndjson delete mode 100644 docs/gemma4/evidence/benchmark/summary.csv create mode 100644 docs/gemma4/evidence/pr-preflight.json create mode 100644 docs/gemma4/evidence/raw-traces.tar.gz delete mode 100644 docs/gemma4/evidence/tools/parallel-response.txt delete mode 100644 docs/gemma4/evidence/tools/single-response.txt delete mode 100644 docs/gemma4/evidence/tools/stream-response.txt diff --git a/docs/gemma4/.gitattributes b/docs/gemma4/.gitattributes index e6692865b2..487b993cf6 100644 --- a/docs/gemma4/.gitattributes +++ b/docs/gemma4/.gitattributes @@ -1,3 +1,4 @@ +# Copyright (c) 2026 Intel Corporation # Preserve upstream patch context and raw SSE framing without whitespace rewriting. dependencies/*.patch -text whitespace=-blank-at-eol,-blank-at-eof evidence/tools/*-response.txt -text whitespace=-blank-at-eol,-blank-at-eof diff --git a/docs/gemma4/UPSTREAM-HANDOFF.md b/docs/gemma4/UPSTREAM-HANDOFF.md index f49d2afbdb..4daee47888 100644 --- a/docs/gemma4/UPSTREAM-HANDOFF.md +++ b/docs/gemma4/UPSTREAM-HANDOFF.md @@ -12,7 +12,7 @@ The upstream logprob fix and all existing upstream Gemma4 parser tests are retai ## Dependency blocker -The target GenAI dependency lacks JSONSchema(schema, optional whitespace_bound). The frozen GenAI patch is attached under dependencies for review; it is NOT applied by the upstream build. A companion GenAI API/serialization/matcher change and dependency update are required before this draft can compile. An unbounded fallback would invalidate the repair. +The target GenAI dependency lacks JSONSchema(schema, optional whitespace_bound). Companion draft [GenAI #4477](https://github.com/openvinotoolkit/openvino.genai/pull/4477) carries the API and frozen XGrammar revision, with four native tests. The attached dependency patch has only a license-comment preamble added; it is NOT applied by the upstream build. Review/merge and a coordinated dependency update are required before this OVMS draft can compile. An unbounded fallback would invalidate the repair. Downstream runtime tuple: OpenVINO `227c33757d1ef95d4da506d00686f923fdd2a535`, GenAI base `7ea2546852a382cd16bd22dea0cfad2db70ed744` plus attached patch, Tokenizers `a04accf6282d9b304214b492694b18c3979f667a`, XGrammar `9aa840b6d16abf094f3e8e2ac9c10465b77656c9`. @@ -31,7 +31,7 @@ Downstream runtime tuple: OpenVINO `227c33757d1ef95d4da506d00686f923fdd2a535`, G | Two same-name parallel echo calls | PASS, exact arguments, 2 calls, tool_calls finish | | Named SSE echo | PASS, reconstructed arguments, 1 call, tool_calls finish | -Raw synthetic requests/responses and summaries are under evidence. Tool cases: temperature=0, seed=170644, max_tokens=256. Benchmark: temperature=1, top_k=64, top_p=0.95, preserved seeds/prompts; timing includes prefill/HTTP/possible queueing. Long outputs stopped before max_tokens and metrics use actual usage. Cold startup, TTFT, concurrency, factual accuracy of free benchmark texts, multi-turn/real-tool execution and session persistence were NOT RUN in this campaign. +Raw synthetic requests/responses and summaries are under evidence. All original traces, including CSV/NDJSON and SSE, are preserved byte-for-byte in evidence/raw-traces.tar.gz; JSON requests/responses remain browsable. Tool cases: temperature=0, seed=170644, max_tokens=256. Benchmark: temperature=1, top_k=64, top_p=0.95, preserved seeds/prompts; timing includes prefill/HTTP/possible queueing. Long outputs stopped before max_tokens and metrics use actual usage. Cold startup, TTFT, concurrency, factual accuracy of free benchmark texts, multi-turn/real-tool execution and session persistence were NOT RUN in this campaign. These results belong to the frozen downstream RC, NOT the assembled upstream head. The original candidate manifest's historical live acceptance NOT_RUN is not overwritten. @@ -52,4 +52,6 @@ These results belong to the frozen downstream RC, NOT the assembled upstream hea //src/test/llm/gemma4_overlay:gemma4_google_jinja_contract_test ``` -Build, executable tests and broad CI on the transferred head: NOT RUN. This remains draft while gates are open. +Local product build/executable tests on the assembled OVMS head: NOT RUN. Jenkins job 1 reported ERROR on the initial head dd7ac8de89fe76cfe00d04d4f69620e00e5aa37e; a successful complete pipeline on the updated head remains required. License scanner preflight with Linux paths/UTF-8 passed; the three new BUILD files have Apache headers. ownsToolCallBoundaries is present in OutputParsingConfig, avoiding the earlier #4525 missing-field wiring error. These checks do not constitute a product build. + +Companion GenAI real-header standalone contracts: original API RED at compilation; transferred API GREEN for legacy/bounded/zero/equality assertions. Full GenAI/native tests remain NOT RUN. This remains draft while gates are open. diff --git a/docs/gemma4/dependencies/genai-gemma4-bounded-whitespace.patch b/docs/gemma4/dependencies/genai-gemma4-bounded-whitespace.patch index 9d6b0e40c8..507e2981cc 100644 --- a/docs/gemma4/dependencies/genai-gemma4-bounded-whitespace.patch +++ b/docs/gemma4/dependencies/genai-gemma4-bounded-whitespace.patch @@ -1,3 +1,6 @@ +# Copyright (c) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# Frozen dependency diff; comment preamble added for the upstream license scanner. diff --git a/src/cpp/CMakeLists.txt b/src/cpp/CMakeLists.txt index 463542c9..42bbe6a3 100644 --- a/src/cpp/CMakeLists.txt diff --git a/docs/gemma4/evidence/benchmark/requests.ndjson b/docs/gemma4/evidence/benchmark/requests.ndjson deleted file mode 100644 index d77ebf670f..0000000000 --- a/docs/gemma4/evidence/benchmark/requests.ndjson +++ /dev/null @@ -1,14 +0,0 @@ -{"timestamp_utc":"2026-09-14T22:44:50.4495236Z","case":"warmup","transport":"curl","endpoint":"http://127.0.0.1:18091/v3/chat/completions","requested_model":"gemma4-26-heretic","response_model":"gemma4-26-heretic","max_tokens":16,"seed":1,"do_sample":true,"temperature":1.0,"top_k":64,"top_p":0.95,"http_status":200,"elapsed_s":4.165,"prompt_tokens":21,"completion_tokens":5,"total_tokens":26,"tokens_per_s":1.201,"finish_reason":"stop","request_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\warmup-curl-request.json","response_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\warmup-curl-response.json"} -{"timestamp_utc":"2026-09-14T22:44:51.0761088Z","case":"warmup","transport":"InvokeWebRequest","endpoint":"http://127.0.0.1:18091/v3/chat/completions","requested_model":"gemma4-26-heretic","response_model":"gemma4-26-heretic","max_tokens":16,"seed":1,"do_sample":true,"temperature":1.0,"top_k":64,"top_p":0.95,"http_status":200,"elapsed_s":0.588,"prompt_tokens":21,"completion_tokens":5,"total_tokens":26,"tokens_per_s":8.506,"finish_reason":"stop","request_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\warmup-InvokeWebRequest-request.json","response_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\warmup-InvokeWebRequest-response.json"} -{"timestamp_utc":"2026-09-14T22:45:21.2809181Z","case":"long-1024","transport":"curl","endpoint":"http://127.0.0.1:18091/v3/chat/completions","requested_model":"gemma4-26-heretic","response_model":"gemma4-26-heretic","max_tokens":1024,"seed":8024,"do_sample":true,"temperature":1.0,"top_k":64,"top_p":0.95,"http_status":200,"elapsed_s":30.188,"prompt_tokens":68,"completion_tokens":728,"total_tokens":796,"tokens_per_s":24.115,"finish_reason":"stop","request_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\long-1024-curl-request.json","response_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\long-1024-curl-response.json"} -{"timestamp_utc":"2026-09-14T22:45:53.1548996Z","case":"long-1024","transport":"InvokeWebRequest","endpoint":"http://127.0.0.1:18091/v3/chat/completions","requested_model":"gemma4-26-heretic","response_model":"gemma4-26-heretic","max_tokens":1024,"seed":8024,"do_sample":true,"temperature":1.0,"top_k":64,"top_p":0.95,"http_status":200,"elapsed_s":31.87,"prompt_tokens":68,"completion_tokens":789,"total_tokens":857,"tokens_per_s":24.757,"finish_reason":"stop","request_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\long-1024-InvokeWebRequest-request.json","response_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\long-1024-InvokeWebRequest-response.json"} -{"timestamp_utc":"2026-09-14T22:46:35.9477569Z","case":"long-2048","transport":"curl","endpoint":"http://127.0.0.1:18091/v3/chat/completions","requested_model":"gemma4-26-heretic","response_model":"gemma4-26-heretic","max_tokens":2048,"seed":9048,"do_sample":true,"temperature":1.0,"top_k":64,"top_p":0.95,"http_status":200,"elapsed_s":42.791,"prompt_tokens":68,"completion_tokens":1069,"total_tokens":1137,"tokens_per_s":24.982,"finish_reason":"stop","request_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\long-2048-curl-request.json","response_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\long-2048-curl-response.json"} -{"timestamp_utc":"2026-09-14T22:47:30.7191435Z","case":"long-2048","transport":"InvokeWebRequest","endpoint":"http://127.0.0.1:18091/v3/chat/completions","requested_model":"gemma4-26-heretic","response_model":"gemma4-26-heretic","max_tokens":2048,"seed":9048,"do_sample":true,"temperature":1.0,"top_k":64,"top_p":0.95,"http_status":200,"elapsed_s":54.769,"prompt_tokens":68,"completion_tokens":1397,"total_tokens":1465,"tokens_per_s":25.507,"finish_reason":"stop","request_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\long-2048-InvokeWebRequest-request.json","response_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\long-2048-InvokeWebRequest-response.json"} -{"timestamp_utc":"2026-09-14T22:47:35.9176438Z","case":"short-128-101","transport":"curl","endpoint":"http://127.0.0.1:18091/v3/chat/completions","requested_model":"gemma4-26-heretic","response_model":"gemma4-26-heretic","max_tokens":128,"seed":101,"do_sample":true,"temperature":1.0,"top_k":64,"top_p":0.95,"http_status":200,"elapsed_s":5.196,"prompt_tokens":43,"completion_tokens":128,"total_tokens":171,"tokens_per_s":24.634,"finish_reason":"length","request_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\short-128-101-curl-request.json","response_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\short-128-101-curl-response.json"} -{"timestamp_utc":"2026-09-14T22:47:40.8721180Z","case":"short-128-101","transport":"InvokeWebRequest","endpoint":"http://127.0.0.1:18091/v3/chat/completions","requested_model":"gemma4-26-heretic","response_model":"gemma4-26-heretic","max_tokens":128,"seed":101,"do_sample":true,"temperature":1.0,"top_k":64,"top_p":0.95,"http_status":200,"elapsed_s":4.952,"prompt_tokens":43,"completion_tokens":128,"total_tokens":171,"tokens_per_s":25.851,"finish_reason":"length","request_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\short-128-101-InvokeWebRequest-request.json","response_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\short-128-101-InvokeWebRequest-response.json"} -{"timestamp_utc":"2026-09-14T22:47:45.8146549Z","case":"short-128-102","transport":"curl","endpoint":"http://127.0.0.1:18091/v3/chat/completions","requested_model":"gemma4-26-heretic","response_model":"gemma4-26-heretic","max_tokens":128,"seed":102,"do_sample":true,"temperature":1.0,"top_k":64,"top_p":0.95,"http_status":200,"elapsed_s":4.94,"prompt_tokens":43,"completion_tokens":128,"total_tokens":171,"tokens_per_s":25.91,"finish_reason":"length","request_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\short-128-102-curl-request.json","response_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\short-128-102-curl-response.json"} -{"timestamp_utc":"2026-09-14T22:47:50.7255474Z","case":"short-128-102","transport":"InvokeWebRequest","endpoint":"http://127.0.0.1:18091/v3/chat/completions","requested_model":"gemma4-26-heretic","response_model":"gemma4-26-heretic","max_tokens":128,"seed":102,"do_sample":true,"temperature":1.0,"top_k":64,"top_p":0.95,"http_status":200,"elapsed_s":4.9,"prompt_tokens":43,"completion_tokens":128,"total_tokens":171,"tokens_per_s":26.12,"finish_reason":"length","request_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\short-128-102-InvokeWebRequest-request.json","response_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\short-128-102-InvokeWebRequest-response.json"} -{"timestamp_utc":"2026-09-14T22:47:55.6742442Z","case":"short-128-103","transport":"curl","endpoint":"http://127.0.0.1:18091/v3/chat/completions","requested_model":"gemma4-26-heretic","response_model":"gemma4-26-heretic","max_tokens":128,"seed":103,"do_sample":true,"temperature":1.0,"top_k":64,"top_p":0.95,"http_status":200,"elapsed_s":4.945,"prompt_tokens":43,"completion_tokens":128,"total_tokens":171,"tokens_per_s":25.884,"finish_reason":"length","request_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\short-128-103-curl-request.json","response_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\short-128-103-curl-response.json"} -{"timestamp_utc":"2026-09-14T22:48:00.8437842Z","case":"short-128-103","transport":"InvokeWebRequest","endpoint":"http://127.0.0.1:18091/v3/chat/completions","requested_model":"gemma4-26-heretic","response_model":"gemma4-26-heretic","max_tokens":128,"seed":103,"do_sample":true,"temperature":1.0,"top_k":64,"top_p":0.95,"http_status":200,"elapsed_s":5.166,"prompt_tokens":43,"completion_tokens":128,"total_tokens":171,"tokens_per_s":24.778,"finish_reason":"length","request_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\short-128-103-InvokeWebRequest-request.json","response_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\short-128-103-InvokeWebRequest-response.json"} -{"timestamp_utc":"2026-09-14T22:48:20.7626798Z","case":"gpu-512","transport":"curl","endpoint":"http://127.0.0.1:18091/v3/chat/completions","requested_model":"gemma4-26-heretic","response_model":"gemma4-26-heretic","max_tokens":512,"seed":777,"do_sample":true,"temperature":1.0,"top_k":64,"top_p":0.95,"http_status":200,"elapsed_s":19.917,"prompt_tokens":47,"completion_tokens":512,"total_tokens":559,"tokens_per_s":25.707,"finish_reason":"length","request_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\gpu-512-curl-request.json","response_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\gpu-512-curl-response.json"} -{"timestamp_utc":"2026-09-14T22:48:40.5059187Z","case":"gpu-512","transport":"InvokeWebRequest","endpoint":"http://127.0.0.1:18091/v3/chat/completions","requested_model":"gemma4-26-heretic","response_model":"gemma4-26-heretic","max_tokens":512,"seed":777,"do_sample":true,"temperature":1.0,"top_k":64,"top_p":0.95,"http_status":200,"elapsed_s":19.741,"prompt_tokens":47,"completion_tokens":512,"total_tokens":559,"tokens_per_s":25.935,"finish_reason":"length","request_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\gpu-512-InvokeWebRequest-request.json","response_path":"C:\\git\\artifacts\\rc-17064400-protocol-20260915\\benchmarks\\20260914T224446Z-gemma4-26-heretic\\gpu-512-InvokeWebRequest-response.json"} diff --git a/docs/gemma4/evidence/benchmark/summary.csv b/docs/gemma4/evidence/benchmark/summary.csv deleted file mode 100644 index a6b8c0d6a1..0000000000 --- a/docs/gemma4/evidence/benchmark/summary.csv +++ /dev/null @@ -1,15 +0,0 @@ -"timestamp_utc","case","transport","endpoint","requested_model","response_model","max_tokens","seed","do_sample","temperature","top_k","top_p","http_status","elapsed_s","prompt_tokens","completion_tokens","total_tokens","tokens_per_s","finish_reason","request_path","response_path" -"2026-09-14T22:44:50.4495236Z","warmup","curl","http://127.0.0.1:18091/v3/chat/completions","gemma4-26-heretic","gemma4-26-heretic","16","1","True","1","64","0.95","200","4.165","21","5","26","1.201","stop","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\warmup-curl-request.json","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\warmup-curl-response.json" -"2026-09-14T22:44:51.0761088Z","warmup","InvokeWebRequest","http://127.0.0.1:18091/v3/chat/completions","gemma4-26-heretic","gemma4-26-heretic","16","1","True","1","64","0.95","200","0.588","21","5","26","8.506","stop","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\warmup-InvokeWebRequest-request.json","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\warmup-InvokeWebRequest-response.json" -"2026-09-14T22:45:21.2809181Z","long-1024","curl","http://127.0.0.1:18091/v3/chat/completions","gemma4-26-heretic","gemma4-26-heretic","1024","8024","True","1","64","0.95","200","30.188","68","728","796","24.115","stop","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\long-1024-curl-request.json","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\long-1024-curl-response.json" -"2026-09-14T22:45:53.1548996Z","long-1024","InvokeWebRequest","http://127.0.0.1:18091/v3/chat/completions","gemma4-26-heretic","gemma4-26-heretic","1024","8024","True","1","64","0.95","200","31.87","68","789","857","24.757","stop","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\long-1024-InvokeWebRequest-request.json","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\long-1024-InvokeWebRequest-response.json" -"2026-09-14T22:46:35.9477569Z","long-2048","curl","http://127.0.0.1:18091/v3/chat/completions","gemma4-26-heretic","gemma4-26-heretic","2048","9048","True","1","64","0.95","200","42.791","68","1069","1137","24.982","stop","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\long-2048-curl-request.json","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\long-2048-curl-response.json" -"2026-09-14T22:47:30.7191435Z","long-2048","InvokeWebRequest","http://127.0.0.1:18091/v3/chat/completions","gemma4-26-heretic","gemma4-26-heretic","2048","9048","True","1","64","0.95","200","54.769","68","1397","1465","25.507","stop","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\long-2048-InvokeWebRequest-request.json","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\long-2048-InvokeWebRequest-response.json" -"2026-09-14T22:47:35.9176438Z","short-128-101","curl","http://127.0.0.1:18091/v3/chat/completions","gemma4-26-heretic","gemma4-26-heretic","128","101","True","1","64","0.95","200","5.196","43","128","171","24.634","length","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\short-128-101-curl-request.json","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\short-128-101-curl-response.json" -"2026-09-14T22:47:40.8721180Z","short-128-101","InvokeWebRequest","http://127.0.0.1:18091/v3/chat/completions","gemma4-26-heretic","gemma4-26-heretic","128","101","True","1","64","0.95","200","4.952","43","128","171","25.851","length","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\short-128-101-InvokeWebRequest-request.json","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\short-128-101-InvokeWebRequest-response.json" -"2026-09-14T22:47:45.8146549Z","short-128-102","curl","http://127.0.0.1:18091/v3/chat/completions","gemma4-26-heretic","gemma4-26-heretic","128","102","True","1","64","0.95","200","4.94","43","128","171","25.91","length","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\short-128-102-curl-request.json","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\short-128-102-curl-response.json" -"2026-09-14T22:47:50.7255474Z","short-128-102","InvokeWebRequest","http://127.0.0.1:18091/v3/chat/completions","gemma4-26-heretic","gemma4-26-heretic","128","102","True","1","64","0.95","200","4.9","43","128","171","26.12","length","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\short-128-102-InvokeWebRequest-request.json","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\short-128-102-InvokeWebRequest-response.json" -"2026-09-14T22:47:55.6742442Z","short-128-103","curl","http://127.0.0.1:18091/v3/chat/completions","gemma4-26-heretic","gemma4-26-heretic","128","103","True","1","64","0.95","200","4.945","43","128","171","25.884","length","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\short-128-103-curl-request.json","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\short-128-103-curl-response.json" -"2026-09-14T22:48:00.8437842Z","short-128-103","InvokeWebRequest","http://127.0.0.1:18091/v3/chat/completions","gemma4-26-heretic","gemma4-26-heretic","128","103","True","1","64","0.95","200","5.166","43","128","171","24.778","length","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\short-128-103-InvokeWebRequest-request.json","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\short-128-103-InvokeWebRequest-response.json" -"2026-09-14T22:48:20.7626798Z","gpu-512","curl","http://127.0.0.1:18091/v3/chat/completions","gemma4-26-heretic","gemma4-26-heretic","512","777","True","1","64","0.95","200","19.917","47","512","559","25.707","length","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\gpu-512-curl-request.json","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\gpu-512-curl-response.json" -"2026-09-14T22:48:40.5059187Z","gpu-512","InvokeWebRequest","http://127.0.0.1:18091/v3/chat/completions","gemma4-26-heretic","gemma4-26-heretic","512","777","True","1","64","0.95","200","19.741","47","512","559","25.935","length","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\gpu-512-InvokeWebRequest-request.json","C:\git\artifacts\rc-17064400-protocol-20260915\benchmarks\20260914T224446Z-gemma4-26-heretic\gpu-512-InvokeWebRequest-response.json" diff --git a/docs/gemma4/evidence/pr-preflight.json b/docs/gemma4/evidence/pr-preflight.json new file mode 100644 index 0000000000..0672ae13b6 --- /dev/null +++ b/docs/gemma4/evidence/pr-preflight.json @@ -0,0 +1,27 @@ +{ + "license_scanner": { + "result": "PASS", + "method": "Unchanged upstream ci/lib_search.py check_dir, Linux path separators and UTF-8; only upstream exclusions", + "not_claimed": "Complete Jenkins/style pipeline" + }, + "raw_trace_archive": { + "result": "PASS", + "files": 40, + "method": "SHA256 equality of each archived member and original bytes" + }, + "genai_companion": { + "url": "https://github.com/openvinotoolkit/openvino.genai/pull/4477", + "head": "33bfec6e44d5c2a1bc5ad12c375f65d59929da85", + "base": "7ea2546852a382cd16bd22dea0cfad2db70ed744", + "original_header": "RED: compile fails, field and two-argument constructor absent", + "transferred_header": "GREEN: MSVC real-header executable, legacy/bounded/zero/equality assertions", + "full_build_and_native_gtests": "NOT RUN" + }, + "jenkins_initial_ovms_head": { + "head": "dd7ac8de89fe76cfe00d04d4f69620e00e5aa37e", + "job": "https://ci.iotg.sclab.intel.com/job/ovmsc/job/Oncommit_ovms_build/job/PR-4563/1/display/redirect", + "reported_status": "ERROR", + "console_access": "BLOCKED: hostname resolution timeout on preparation host", + "failure_cause": "UNCLASSIFIED" + } +} diff --git a/docs/gemma4/evidence/raw-traces.tar.gz b/docs/gemma4/evidence/raw-traces.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..00912dc37e6feacb7911bb21e7eb5470a77bbecb GIT binary patch literal 13907 zcmV-ZHmu1XiwFRx1MPk3jvLptZhi~!I~0fjR{}_i#ko5_E^H@G49B*PtvEs4 zzCw}QBr9F4Vru9XBM3MTah~uz$z8)9tBO+(?3T?sCzeDOYuI~@-Oo-S8GdY!%Q zcDlpPh2MYM?e_cq;Xhjov04TFR(IH1id5v`^!4xVbaXnM(P-r1pKz4_-fWN-~XLAxZCZHJBln1Qh1d6UZ?zWpU5#eYX5`9A{C1u7wvy#Nqp!%U_FMz z;r9LSk9zL@cYXN5@4S2O$sxyYeE)y`?1krj!2ft&BTs`kTPJDWxbzy+B8?hN`OP%Q zLVsxRtxfS7JnZsB7yLW4m z(plYp8`vIC`q3cY8iP;Aby> z`Rv70Z{33&$MOGSU9<*%w}rIPN_h+Ri-LXp53H~Gf7nO9M7SJNcUMGQ>Y_Pl>_d zGOI9eH$MIX?;E_=(gNtb*~rBT*gwdN6kgqLci@v`{qDMPIqdUa*Nw|g`(n^+WJ1gu zmtg07{G=qqz4#x*a-8kA1@`d&us1OF-*DIm{_lZJaOVG093L9fWfD#$i_gP2%$Dy` z5x|xLYeynpO>!CyQy9SRU> z`7%w4#S;GB^s=>>hI5o;(Z-u6(}JE4o*r*713?tV)7)Dx(F~!tg^z7n_$c9x z@D_02l-EPPBgnTRRWfRwig_4Ce37Pii%iGD3j`MYHH_g&@RzG~?uB}t;N?TOV35M2 z36N`v_5ApwFc0Cquf>Mm?Iwr{;bpXlO*$i4tA~PX%Ad9-39MNYmM4hKG7{ikkm?nOB}!u9O*Z%kzcs%o-3nF*mJ*g8 z)@2#af#7Co01qWMJ5O$dbmmP2-iYa?W$1+#%9|B31U^J|dV0?j%xEm(ShYZl$W-ppAxN`AHk zjO5(F!|&k>v;pO1;A46|7Fx-TNYTu*By@^=)&fDX27(s(tq?4F7f~{?NCw`MnQ##j zn;=(bXc5v~=42p}oCBMcXbnV0400rh@R4uBe94j!tZj6ca=K;OCdlP&O#&ykdLrhcuD|#84IK;r7rnalVEy{ zs|4~uy(O|M3z#tH@8@I+XT^kRCHH4i-G{q*-*lGDzZ>)EF&>5-OU!{#^id zt97=dR)#V#$h@!KjGNxq-@P4^{V#4{HQ--;RHD<6c3JTgyYsw#W?z9Q{>#1vQ=im0 zOic46$wBGGxf0-{i7Xig%)fxwqK`A&oH;BJ(vVydkZmX|;0hUUtws0v&~Y${tm_zl z%}|w*J_?bGf>?ljEVH_@O;9L1yqza#=maa^7OBgy83Yid$OdBQ`ZIVM=L@SB+F9|cxNHBoY+DE_}*YB6l7(= z>NxKv(i9(J#_ne@offbt$g06Kg}0LC2l$y%21pXIjocswtqgB~&STsAsz{LufU=|o zG85}4*^o+sAESMVH$cIy%^GWggAzMxEyOlYpp1d0G7zpBV8zVytA;e7U{_}06jwoO z5_)Jhme^{B3mM@7^afiFdIt6eNDLks`~Zmo1YppDJ_TDJz2GEzhbjzRh+jUU8;w+& zqdV-6#ut6?D#zeB?7|t?ri3r2@EmOe-N9Fd@-Eo$)$VzJ)U3n!*lR})Fdq#rK7M>^ zKkK<3ck}Yx4pQygdf?~MOF2>aD7}y07K8!TgN>CsfV&ETwA(l&~m-ZNWS-|=mZ48zX zzh*7mryNNeIs{K_DI_k0_E$lcok9CFin0Qs7r;WU!1c+cCy#yc=3#?3r4qLCKah01=@_a^o01YJ%a2HZz`zb}q<6 zuv%+9vM9)w8cI^n>LRfUJ|oa9Cb3#wKI`Go@HEH_{smu!zzq8=L@j`FtElJ zTDAOi<)(r638%sHHVxQ=IHjU1-fm~9EsU34dv+vO!z zTpYvZ@-*b~S$Gr96xfj8CS~;evI>UleQ^O_pw`Nn8ilf=2sRFGgXorPibN4@G>Q&y zK8b`K3}~Swi4n$EzIpX|%*_?jk2hxr%f_GeT@_Rbca}~UJ|f=L**yzghP?-&0hF~u zeeB7H$~U-23U0PS2H%KH>nFkzUw-q`D+M7 z$`TK?J>ew^ZA8BXQOd*6GlO19(%*_7f?9xe0Y;(6^oOkpNc}l*A^y9yP7pbK`PCbL zh#txpU-`pV`daV`V>0ZB&y z!PdZk3Nu$!PnVZEgMY2nDgo%QWN15xX;7dsM`Vu)3btWtZzV{k8pB{l8T7?7xLX|<`#`FuieQEve*n_+l-xdD^o`VgX zS@0%^f|elwOVA>PhiL3gfy99p?+k_cohOIF9RKt<{|_Zq%kOmiM`eKR!+(Z7lmF4{ z_b~s%_s{uXCpjJv|Iwh2r7aBp16JO&$dq73{TFfIqZI}5qJX=!Sm9}@(X8>+q~$>R z$7mvyb_mYL)6H(gXaa&nxl^~5a73hdu6mqW&XoTc1kt0x3s6w^?NOR6BU zIcrTy%aV4fyad>d)9Ywc&ICeRnpZO51!LEMFFZA3cjFDHq<#kO(b}bp{ZP{Z)6fCLoy~+D3fUu z)!WW6Xiw=8447D;B9f%au%R{9IB~6?$POJyKo|bac7TYI#4xRnDX^S`AcZsHPhplZ zN~OUZls;4Z3g6gJ4xOkm4pJKL5iE^oz$}Y2xs@cRF(Ud5p95=v=)TdWN$xTx9>Oz6 ziiq*%gohz;Y$3kJ309sX(?7|-WfGCAoXKp=O#99|PL_10-Fcv?Od#?nw?aJyt1x)d z6jw{fFv3`BK$8PPk`+0qSP2a#Ed$IVm45M0D!bFd=S{$(29Q_waS4sn=}WdAu{D8C z+|!U$DG#ab{hbg{eFKTCNGj+2BlB?o```cL!rPL-O>(aB#Z`I@@*z4J8T=UY&Ngcq z=3p5SsIW>#+p6xby#Tr*kQ74An4PLnPTZSh3bt#efD0%2ikO6U+d~k$D)wNNvSl%! zN0{?HpL3pRCOE%KQ7luCmY?e+XL;t5D&0WZObUyX(7rOONy=n8kqyifS`&u7QxP!( zW1K=!dnx5ptp@40xW$=;e?Z#UB!ylz>sL+sLPK+PDcuIXQ8}ZedRX=0uPx35n-A#! zsv+0$7CyXcSc<^O^F-`dXT$0JNak}|U$V$R4`iHgsA#JcbdccFw&X#=G8tooHDv@& z>Rl9NWV=Z8n(`r0qQEQOgi?KEnk1~$zI_RSq^%Waw-#wIGi;|g>E8#ZBI33hcC(Xx zwQ4Q_nk5L3bN`S(h$%=#IeBK7Gc8)M?zGadvPwHCR17U`X{zTUioyk3!*44&Z7k#D zYM@X7#VewXF+Whhk?|yC8lQ>WW*Da^Qg02`a*$@1D!|9f%w)~Tu;dn81mq6%-bETp z0KqO#(2y?$K<1zy@hs3KQlR2lTI*P$2$cKq(ve`GUaG8&VFyjNb+STd7g~Z=+L~b5 z5^NmebvdA91Ky0*0{NvWzL;bGk@9pEXC^TVRjjZCMGr(Etw<2s@Lgs^Z?i_ftti4# z&e-Ry^{M1%@qQ{qy>e7P+@=_DFRm#P;v{{p)#6Fh1FigBNHa6IWvv~S-s>91j>>Z z$P8#7!aZ4`QpR*tEEeE;lNrD!<_ai3lv*)cBOwCo;tLrdoV-eTN>&1=EE09oIFl(uR5mth#TziS0v=R8TpHZgLiQWW*=7&> zfB*Y`R> zi|UG4k&n7CNi8VLsTgb_qEk^IE~?`N^B`N2nXgLD!`zfCQGOyxTr?U976HeyywKtuMZW4cuC_x@7tQ;f%_^xibsjiSOUEVg;F#8BDvc z0uE?rpdiAPhb>K2+}dFPxi~Qq*p21MbjbqDq?UkpURGc9E&Yk`IdZ063o1-DeeRhTGi zb`}{}?O;kxaoFH3moGFqyvz0rr{}wEd#pIlma+w{VpTMQVGkshlD;X$$&Lw-$ux9Z ze3HCJ7N&kD?>Q1l7ZvmJ770ZkDq#rpdP7sCeAUQ+ zfsuKV+zFIA`xSw9h$v$oG&vsC%N6Y_b{IBQx=l+i; zInMb1vG_ld|3Uaao_IX|e*wRG-uS=G{|}M>-|Kfrn*Rqr)&F!yXZ-&p$7A9DcgpzJ z3Aq_ne42E9%F#yys@%3SM3jYGZ)y4mx536>l8CNvuLyjJ5PJ3s6Eo2$0oM;vFas8o zEyu1fAq#=Z!=EMGhmzp1O5%?dnW)4L_`*6wpt3`#3kEq!f=bRgSxhyGv!T83-u!J> zWFd$M!VRKZEP}mzJV04~GOb+Ya8E3}dRre0hsS@ZZmAY>Qt807Z z0`o4MUb!{xaVrg937)VsO07x@`wdm&G`$y4V26U*%(vVj#R?7%(Vc4jqyeu zZ-Pl)3D@bY1SNmbEi)}~KxE3objyRXLJkB|Q7ww!?|6UyLa)oe@C(KZsmff#Y;0{w zB+dvu>CxM1j!6ugFzDcAUrRVLm}J;r*j+k>RkNgWE5V5l)gE~%_ex#BO(Gdw)`j2b zSc0gRok^*C%Q``}&9nfkRS^GC8sg6ie#87MslU@!nOk;si!YO~Fxxm%_{hlFBrBAhcU#DlQ99 zt#(ufw7m0{;MQ$WiEfo^Mu4zxlOmcCbm(?5(vd?aYPbw1q2lQ%xuxJ;hK*J{?of0q zICN%fu32TNN-;7S91AsEx<5>&fwBVwW-7hCChTSPVk(!*NGw@{`{h7r3E?jRu_`*& zlvacW#5ip`RVl<)yvr^^VRV)raSBSEIuj>FI+e(AAT2l(|AH45C@nb}Wk==QfV&E9 zDc)9L5RFqNx~ho?kXDm!p-asYi7{6NmVX1!wkNN?SvKbJ7qzufn3aRZ;x<^YFv_YLAXvlACD`~T3}k+EyV73=vCgcniF&2O zj55a*nQnoS`C|28GRHj?`bTzMv>{zzDz>Vt{3?^L1P!G~o}fjSd4(2mI?KBn*Q_sY zQBs!d6An76ZX?b8@hAc$MOGg0prTBzJZa6tG|OfD>*d?GU%fSL!srx?jxi9RPkQtH z+gIFE$yCy~5mPHfYK5D*(+dm75Sx-C@Stjx9CR9O1&%K&sh8z;4=9)UP!O?}M71UH zQPs3Z`Y29CP&t5(Us49Qs?Cfxie)q`O*3RiCg9)(%qeiKq`K>k$L zs3bH#?*H|^M?wrX)v#1(CVA9$g@UVux%3WN9IC)xUb7;ynnu|@@trnLx64Gwq$#F> zcmzy9EB+-(UqKu*r3ej;{^lmBj`*7KG<$M?VQ7FxB&NOMaKbwv@WL zQNODO7>;UY!d0UZ07RkGtcxtFXQ>pan$lF(fZMC(T*HU^TdC@3Zdr+-8rAAM0&90w z(G3Vkw2v^i&+b{u8BrvhT^!urE%=Qc8LKbhajNRU@=~(C3|bLJ^=&Za7cFA$4DSXO zO_4VvAj)RY76Eo=;NcK@xxSQ+GXDMhA58)%2L~wW%gKb{@NS4k3eL=+_FwdD?4|>I ze(rZV&a1fa+7EC4;q^Ng9ad`;gjb%vMOq5ir_y_V`X&mgDX$5mrh2njLzg4P7(=c^ zmfPOnbSWgSTRAFhrJ;mdiX?$noT6DKmbyrEr8z1GVsq+>kx8-3h6@@M>ov+LE=aUW z=qi=^CiV;8o8a$+#LT>i?h#-e^iCCE^*h5#gw^+Zqo*#)I^l8G_|Ie2e{uY0(D%>% z|4(w9Mk?((%F6O zRIqJ(Ty-KAVZ6I1yB&9~v^rlhwnfmgj^XUV3nr_JtCkAbbaYRZvYA&q zx{S?qiLlBGOVOcHC}?lYk@YX3UglA$4PPQ_6-&LbWs*5N>h}ys>R>^Ii{S};@2w6! z)AVH{Z}RCD_~29iFyCBzf#Dx??8FQ7RrZ$fs&GZwlLpZFCzI3m7kh z?zB4^7Pp#i<110OC)t9BeCh4IUnm@Bz)myept7_qVdwBtV z!^y3nTDa-}E6BLPq$>E3>(AZrRZ&T%3Xh!81n_{mcMq=j?pX_m_C>>otf_BeWnncs zD-Gns0C5Qd(uU|4=2z936nmSj=>aMnow1l?ANF4b<{1B1cT{yo6gSbYW`ZW$u4!di zNDN1oSp5FG-<#Li39rf4aAj!CO&y>~rs7QT9phB=! zT$CAH<1I>bQ)I{F7~KYO&Ux}~?j>s$J!)`64XE||nc8FUw%q8hhlOrN;O(sb;O^FN+ zE{8rMVPr=Uu-Y$lSrvHctg+0=ifE=e$T5tEwe60y8B1+djfM7%K|^JhiM!-H@IJG$ zN;2A+O{sbm!$xLGGw{?pJE9;PYuVCSUGF}HV^` zH`0-q`SDhY$)QD!lO(0RmYHjkIV;I1ZW2mU$$GN$bk3EEyiNJ1J8C4kIMS-l6oGdm*QM$OpL+S z@H1@c5gYG^eRgadzuAH**ma+#&YZpFt!ZFxzkHM;)wduyXg^KG53z4J7f2J^LeYCY z?rOT7l#E1LrBpkWk`^d})EWC!)NM*qmC#%Y_^yvK5@* ze|@xHP83|qz8ckuiAIv6VDCqsC{A~3zPaTMCizhKtF7n(A^rC+@Al+lKa+nUnY3tTj* zZ}ED6@jD&E?<(c{&wu>m4@3QHz22~xGwZi1AwgEH%&8fY_qlsMs0J*fpmYVswMxQ< z3=XQvx1W-J%6oewxZMscIR0Ep4P1f})?LmNWcM_48h+3<(}H*re?u0Pc4(EoEy-8r zZ&@DXlDf~*R#RJWm1bH~JatQIXcETCxtIs!TyWmEkX%5btVpvWB**Zo!LsA3Vd-Xf zw2W$MtgKhcj)8|sFUilgTXK~8AlW2nH3oa>R1&>vRxV>hh)TY`ECI)_SZh?CKqb&5 z4%+!reJ+0tRHK}KE1iVWHgPm;d7rO#OGSH}pE)Zg+T||8|n&LjxI~g8$)cA0sfg)O1lF588bsEl#k>1Uq?sc*Up-NfH%hk)n722b3cG@X0dI*O#wf``uBy1OMk=`ePum*EhY_s8nAYU74XCXX*ke zvP&&1<+Jl7rw{v3<<|Ml#w>Z41$fiOWu6w|nEX6^mvi9@`1g14FnB$`-R;1Swl4YBFwL@us#Oe zqwMM`owoc@XV~v|IxTprJeej@OS1Uj>dNr_)zyB+zq;c6uC{%$H{_|F#xh1w#~eb^ zu+gKT-x-hhk#srF@i$7|oi-@>2PE%td(as^Kl0w0)&8WUzB}LcAhI98ezm(upkv=6 zdp*ZzMqcb{q}ftFu~V0r>r6P5q|m%3H}+%`a{R-9K;C2%if@U(I1Tl!;3N_ zUKBeeQxw$-@)|>X?{yf$%E*8gqnKf!)QGJk*7>QmGI1IYXl$$Wn_?DuTXHd_MixBTuH zl#BmK$bG-7S-|f+qF*~``)H+?7@*&)XMn1W?~nW%kw5J9tK`qg7f-?f&i+3a!C(7O z#}j`D}>)8ESyN4|oTrfNen4+`rhu z6x~lx3v?fq5&93#2N%zm4Z6=!2XybQ0|wv#bO(d}sDB{2e_}e|aY_GyDSz1ZYaHHZ zL;25K0n|R=9XMjp9*+8O%kCDA=zV$$p!c{6pc=iswO?E@aq6kV{3hTY-l!UlsD>!LOAKMe<9 zfTjeqz-TmjJQnaT5H+jvz^FdI0iVj{fx+OSMgffKV9+Pz0lD?hMf8^*=Xm<=ciMx_ z;KCn|_R{zB7xyQm^-;eX=(;bhU-WA78lC~IKSN2sbjUjAUD*h zAW*!Y@e8l+C-}uL{5z75VZ4CzKgjbfsp*l}l%G4o(?jX0zF-;Zfe^Jm0a5CqEX9vL z^36<@7QRYFRC=$yX+%|CRL@A&_uSnmgGYY0r4PsP|DHh7mV4~u|Nf{~>i^jr^v?5N zPjWm!|KC5$f=?=jtm*tK-mjw|)~%52F<91^C%X&%t`@$gcQ3Y&qNX+6#5bd<3SaUQ z2&-YQ>?%>;q?i5KBYX4$@jqLJKJs_~{@+^*xQG8y+gtoULjE6}=Rcq3*xmnE+wXVi z0BmvecDAof16^Qyw%G2!>GlF!sgWA0#9Ku3R$YrURe4ndc|6BYjHX6jC<)=Z7i{2lRJw_j|HW+DAo2F=5S03I*sXAD+XY)qa-G(|dco{Ttkm*THS{g@ zn^R9*KS_~V0PtGj7$rTh_dmROG;C!1pkJ%gB+dN&@s->8)E@e(+soAuTMeDvwfub8 z8FU`J{QM})mWN#2NB$o=1L8RQ59R-`)A!Hv{}jjB{yW=$56J(uw%{KA2e!BP-|r*; zcRRf^|DWP`Jp0eJ+-93NScTJUy9B@{HM?!W{p}oOe>G)Qm@eMjGPzVapIT)MqQPFW2k|aA_`Pa!$%h3= z+)}2urYQNms?DUcaH=U;kkf<+s}-Q!4cRD4+uK$nYxE2($DnRTgzfLt82;9v4(U0B z**%QMhwoi&uCXBr09!J^#r2l(^{#pN@MR>YZr;iw z7X#A9kj`XBPsf442_~wCGL2cOaTe_E*%C3fhIJYXn%x3y0PCz3dMTDuq(^w6K^{=R zTzU)k4mrdjZE0#KYAkG8M%O{JTo>&7YI}3$D-GZ)#{e6w){G@Af%;qQd_9+^Ht0|p zQs_YZ*s&FR=z7+erMvqE!)F03pN~)CxR3sS6#Sp`|FF|L=YO5#INSec`~Ly?|4#V7 z-yK=}KVod0o><81$*?fW!vH9ba9&FEJvi#z*TF;vz~RmP6ZA$ z#Pv0(U~F<;D?|Ak*d0X7WMw97s-Ns!JRC zkDfwlrw%Q{*-TB8=UgCkfcVj+?^JSr)cjQ%1!wZcwz@%9Qd3Pj`rS?U(HXiwA16KT zqyHZT|0n%F?DWs`-%fIz?fHje7>|Fivnw*Q}v z{eOn;&&NrRBlUl=!WdY4n%x|6kKOv;9}Gu>()?F&3C{T6NsdzMC)q{M?ww~f%B%Li ztMs{ZtZQj6+r8{uV_O;8%F1?I*V%3B%Khr^GefDDCBwRf& zRQAq1s)=Bobg%o&AL4)De{a)5$iEJ8h#mSP@JF}Pf&bFa1RlKe z z#3y=&-J4vVa%vBLd{jo|Ta5XSog2)PMe!^A&!|iPy}-@o`JgA-<=wMCPkZZqd+N^g zwD5*X1WS9Q-(#|}yu}6ov0L%d z)JV10I298A@OyH%FUH+d?)KKXj;Fo#?p-abc7dy4U)miGax2J@c!P&+I(x;prL_ebjZ{KeO-j zx8JwD?kv9Y_MS{rAG*~v|E_kcX^PzwZS|9K_ffOxYDvJEJXfBT{W-?zE$oVD`io|?Z8+34DR#tCIlw9n6mx%W;VsNH{Os#tlt(kYm`cL;M2 zw#oGI)2Q$ISzD*S{qF58&(JDw@2S~)?^e^?sx6kNCgFmsQ{5j)L}@0z1@${X+7O?N($+09e2@-aI*>ib)E z_M$g9`OZEm7auqu=s`mD$HjlQMR|7LV_*Jf*NXqr{GV>m_dDnK?`?X8aomUh_ZEK1V{iU<&(weQ z`ZWIsp5V;?r#N=!e@a~Y$n4Lryk+?iuF1Q#muADY&RmE z1bMhZg@UxX61hk-yqLU|R0mWh0Cr>^3jP*;Y@G#fhvT9@erz^^*(~IWxi{7?H)8bcXTY=&Vk1Dmsq-nm+-{GT{~pQz;o1K`$+6r1w^;sYFW>;v|3f1L5!?9N*Q4;OFzQjs))AT` zNEZbSaD|UPT#=x@YFu75%ChjP@eu{EyxSiqKYjkE)%s7@4}Uk7s(icp3Sa*bn)Cae zPQ72#t5o<7dS%aM3;zS#7g3(w%k0y~KKrlR^-K1De>gbLe>utV^JgzS?*snF^SJi& z(raX3Bw?+l{8*M_!WVBo|M5rtF{TW_hg~2){rd=*J1LO=JzsrV5-Oe}RpdFyO?Zc& z`QPw~%~t$4{M9e&C#E(O?uEHs`1!|X-TNqQcF#BJKG++PSW>5!UY2CYu9xk$hr{vq zmz5U$r?h$99=*9UBg8y{c1*!|39<5a&xo$0xWzWB%F+fDv?t?6kl`hZ60^IxgzBid{CsE_*d hAHV