Add video frames input support via video_url in chat/completions

docs/model_server_rest_api_chat.md

@@ -195,6 +195,41 @@ curl http://localhost/v3/chat/completions \
 ```
 **Note**: check `--allowed_local_media_path` parameter described [here](parameters.md)
 
+In case of VLM models that support video understanding (e.g. Qwen3-VL), a video can be sent as a `video_url` content part. Because the server does not decode compressed video containers (mp4 etc.), the client is responsible for extracting frames from the source video and sending them as an **array of already-decoded frames**. Each element of the `url` array is a single frame in the same formats accepted by `image_url` (base64 data URI, HTTP(S) URL, or local filesystem path). Frames are sent in the order given, so extract them in temporal order:
+
+```
+curl http://localhost/v3/chat/completions \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "qwen3-vl",
+    "messages": [
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "text",
+                    "text": "Describe this video."
+                },
+                {
+                    "type": "video_url",
+                    "video_url": {
+                        "url": [
+                            "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBD ...",
+                            "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBD ...",
+                            "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBD ..."
+                        ]
+                    }
+                }
+            ]
+        }
+    ],
+    "temperature": 0.0,
+    "max_completion_tokens": 128
+}'
+```
+
+**Note**: Unlike `image_url.url` which is a single string, `video_url.url` is an **array of strings** (one per frame). All frames of a single video must share the same height, width and channel count. The same `--allowed_media_domains` / `--allowed_local_media_path` restrictions apply to each frame reference. Multiple `video_url` parts, and mixing `image_url` and `video_url` in one message, are supported.
+
 ### Request
 
 Below we listed request parameters specified in the body as defined in OpenAI API specification. 

src/llm/BUILD

@@ -140,6 +140,19 @@ ovms_cc_library(
     visibility = ["//visibility:public"],
 )
 
+ovms_cc_library(
+    name = "video_utils",
+    hdrs = ["io_processing/video_utils.hpp"],
+    srcs = ["io_processing/video_utils.cpp"],
+    deps = [
+        "@mediapipe//mediapipe/framework:calculator_framework",  # required for absl status
+        ":image_utils",
+        "//third_party:genai",
+        "//src:libovmslogging",
+    ],
+    visibility = ["//visibility:public"],
+)
+
 ovms_cc_library(
     name = "io_processing_input_processor_context",
     hdrs = ["io_processing/input_processor_context.hpp",
@@ -160,6 +173,7 @@ ovms_cc_library(
     name = "io_processing_input_processors",
     hdrs = ["io_processing/input_processors/image_decoding_processor.hpp",
             "io_processing/input_processors/audio_decoding_processor.hpp",
+            "io_processing/input_processors/video_frames_processor.hpp",
             "io_processing/input_processors/chat_template_processor.hpp",
             "io_processing/input_processors/chat_template_adapter.hpp",
             "io_processing/chat_template/caps.hpp",
@@ -170,6 +184,7 @@ ovms_cc_library(
             "io_processing/input_processors/tokenization_processor.hpp"],
     srcs = ["io_processing/input_processors/image_decoding_processor.cpp",
             "io_processing/input_processors/audio_decoding_processor.cpp",
+            "io_processing/input_processors/video_frames_processor.cpp",
             "io_processing/input_processors/chat_template_processor.cpp",
             "io_processing/input_processors/chat_template_adapter.cpp",
             "io_processing/input_processors/empty_content_array_normalization_processor.cpp",
@@ -180,6 +195,7 @@ ovms_cc_library(
         "@mediapipe//mediapipe/framework:calculator_framework",
         "//src:libovmsprofiler",
         ":image_utils",
+        ":video_utils",
         ":io_processing_input_request",
         ":runtime_chat_template_loader",
         "//third_party:genai",

src/llm/apis/openai_completions.cpp

@@ -271,6 +271,26 @@ absl::Status OpenAIChatCompletionsHandler::parseMessages(std::optional<std::stri
                         if (!inputAudio.HasMember("data") || !inputAudio["data"].IsString()) {
                             return absl::InvalidArgumentError("Invalid message structure - input_audio does not have a valid data field");
                         }
+                    } else if (entryType == "video_url") {
+                        // Video is passed as a sequence of already-decoded frames; the client
+                        // is responsible for extracting frames from the source video.
+                        // VideoFramesProcessor decodes them later.
+                        if (!entry.HasMember("video_url") || !entry["video_url"].IsObject()) {
+                            return absl::InvalidArgumentError("Invalid message structure - content video_url missing");
+                        }
+                        const auto videoUrl = entry["video_url"].GetObject();
+                        if (!videoUrl.HasMember("url") || !videoUrl["url"].IsArray()) {
+                            return absl::InvalidArgumentError("Invalid message structure - video_url does not have a url array field");
+                        }
+                        const auto frames = videoUrl["url"].GetArray();
+                        if (frames.Size() == 0) {
+                            return absl::InvalidArgumentError("Invalid message structure - video_url url array cannot be empty");
+                        }
+                        for (const auto& frame : frames) {
+                            if (!frame.IsString()) {
+                                return absl::InvalidArgumentError("Invalid message structure - video_url url array must contain only strings");
+                            }
+                        }
                     } else {
                         return absl::InvalidArgumentError("Unsupported content type");
                     }

src/llm/io_processing/input_processor.cpp

@@ -26,6 +26,7 @@
 #include "input_processors/empty_content_array_normalization_processor.hpp"
 #include "input_processors/empty_tool_calls_array_removing_processor.hpp"
 #include "input_processors/image_decoding_processor.hpp"
+#include "input_processors/video_frames_processor.hpp"
 #include "input_processors/audio_decoding_processor.hpp"
 #include "input_processors/chat_template_adapter.hpp"
 #include "input_processors/raw_prompt_extractor.hpp"
@@ -52,6 +53,11 @@ InputProcessor::InputProcessor(InputProcessorContext& context,
             processors.emplace_back(std::make_unique<ImageDecodingProcessor>(
                 settings.allowedLocalMediaPath,
                 settings.allowedMediaDomains));
+            // Runs after ImageDecodingProcessor and before TextContentNormalizationProcessor:
+            // rewrites video_url parts into <ov_genai_video_N> text tags and fills inputVideos.
+            processors.emplace_back(std::make_unique<VideoFramesProcessor>(
+                settings.allowedLocalMediaPath,
+                settings.allowedMediaDomains));
         }
 
         if (context.config.isOmni) {

src/llm/io_processing/input_processors/video_frames_processor.cpp

@@ -0,0 +1,105 @@
+//*****************************************************************************
+// Copyright 2026 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//*****************************************************************************
+
+#include "video_frames_processor.hpp"
+
+#include <string>
+#include <string_view>
+#include <utility>
+#include <variant>
+#include <vector>
+
+#include <openvino/genai/json_container.hpp>
+
+#include "../video_utils.hpp"
+#include "../../../logging.hpp"
+
+namespace ovms {
+
+namespace {
+constexpr std::string_view VIDEO_TAG_PREFIX = "<ov_genai_video_";
+}  // namespace
+
+VideoFramesProcessor::VideoFramesProcessor(
+    std::optional<std::string> allowedLocalMediaPath,
+    std::optional<std::vector<std::string>> allowedMediaDomains) :
+    allowedLocalMediaPath(std::move(allowedLocalMediaPath)),
+    allowedMediaDomains(std::move(allowedMediaDomains)) {}
+
+absl::Status VideoFramesProcessor::process(InputRequest& req) {
+    if (!std::holds_alternative<ov::genai::ChatHistory>(req.input)) {
+        return absl::Status(absl::StatusCode::kInternal,
+            "VideoFramesProcessor received input that is not a ChatHistory");
+    }
+    auto& chatHistory = std::get<ov::genai::ChatHistory>(req.input);
+
+    // Injection guard: reject requests that already contain video tags to
+    // prevent prompt injection via pre-baked tags.
+    for (size_t i = 0; i < chatHistory.size(); i++) {
+        const auto content = chatHistory[i]["content"];
+        if (content.as_string().value_or("").find(VIDEO_TAG_PREFIX) != std::string::npos) {
+            return absl::InvalidArgumentError("Message contains restricted <ov_genai_video> tag");
+        }
+        if (content.is_array()) {
+            for (size_t j = 0; j < content.size(); j++) {
+                const auto part = content[j];
+                if (part["type"].as_string().value_or("") == "text") {
+                    if (part["text"].as_string().value_or("").find(VIDEO_TAG_PREFIX) != std::string::npos) {
+                        return absl::InvalidArgumentError("Message contains restricted <ov_genai_video> tag");
+                    }
+                }
+            }
+        }
+    }
+
+    size_t videoIndex = 0;
+    for (size_t i = 0; i < chatHistory.size(); i++) {
+        const auto content = chatHistory[i]["content"];
+        if (!content.is_array()) {
+            continue;
+        }
+
+        // Replace video_url parts with text tag parts in-place, mirroring
+        // ImageDecodingProcessor. Each video_url is decoded into a single
+        // {N,H,W,C} video tensor and replaced with a text part carrying the
+        // <ov_genai_video_N> tag consumed later by the VLM pipeline. All other
+        // parts (text, image_url) are preserved as-is. Flattening to a string is
+        // deferred to TextContentNormalizationProcessor.
+        for (size_t j = 0; j < content.size(); j++) {
+            const auto part = content[j];
+            if (part["type"].as_string().value_or("") == "video_url") {
+                const auto urls = part["video_url"]["url"];
+                std::vector<std::string> frameSources;
+                frameSources.reserve(urls.size());
+                for (size_t k = 0; k < urls.size(); k++) {
+                    frameSources.push_back(urls[k].as_string().value_or(""));
+                }
+                auto videoResult = loadVideoFrames(frameSources, allowedLocalMediaPath, allowedMediaDomains);
+                if (!videoResult.ok()) {
+                    return videoResult.status();
+                }
+                req.inputVideos.push_back(std::move(videoResult).value());
+                std::string tag = std::string(VIDEO_TAG_PREFIX) + std::to_string(videoIndex++) + ">";
+                ov::genai::JsonContainer textEntry({{"type", "text"}, {"text", tag}});
+                content[j] = textEntry;
+            }
+        }
+    }
+
+    return absl::OkStatus();
+}
+
+}  // namespace ovms

src/llm/io_processing/input_processors/video_frames_processor.hpp

@@ -0,0 +1,45 @@
+//*****************************************************************************
+// Copyright 2026 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//*****************************************************************************
+#pragma once
+
+#include <optional>
+#include <string>
+#include <vector>
+
+#include "../base_input_processor.hpp"
+
+namespace ovms {
+
+// Decodes video_url content entries from ChatHistory messages into video tensors
+// and rewrites each video_url part into a text part carrying an
+// <ov_genai_video_N> tag. Runs before ImageDecodingProcessor, which then flattens
+// the message content (text + image tags + video tags) into a single string.
+// Each video_url entry carries a "url" array of already-decoded frame references
+// (base64 data URIs, HTTP/HTTPS URLs or local file paths); mp4 decoding is done
+// on the client side.
+// Active when: config.isVLM && input is ChatHistory variant.
+class VideoFramesProcessor : public BaseInputProcessor {
+public:
+    VideoFramesProcessor(std::optional<std::string> allowedLocalMediaPath,
+        std::optional<std::vector<std::string>> allowedMediaDomains);
+    absl::Status process(InputRequest& req) override;
+
+private:
+    std::optional<std::string> allowedLocalMediaPath;
+    std::optional<std::vector<std::string>> allowedMediaDomains;
+};
+
+}  // namespace ovms

src/llm/io_processing/video_utils.cpp

@@ -0,0 +1,71 @@
+//*****************************************************************************
+// Copyright 2026 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//*****************************************************************************
+
+#include "video_utils.hpp"
+
+#include <cstring>
+#include <string>
+#include <vector>
+
+#include "image_utils.hpp"
+#include "../../logging.hpp"
+
+namespace ovms {
+
+absl::StatusOr<ov::Tensor> loadVideoFrames(const std::vector<std::string>& frameSources,
+    const std::optional<std::string>& allowedLocalMediaPath,
+    const std::optional<std::vector<std::string>>& allowedMediaDomains) {
+    if (frameSources.empty()) {
+        return absl::InvalidArgumentError("Video must contain at least one frame");
+    }
+    if (static_cast<int64_t>(frameSources.size()) > MAX_VIDEO_FRAMES) {
+        return absl::InvalidArgumentError("Number of video frames exceeds the allowed maximum of " + std::to_string(MAX_VIDEO_FRAMES));
+    }
+
+    std::vector<ov::Tensor> frames;
+    frames.reserve(frameSources.size());
+    ov::Shape frameShape;
+    for (size_t i = 0; i < frameSources.size(); i++) {
+        auto frameResult = loadImage(frameSources[i], allowedLocalMediaPath, allowedMediaDomains);
+        if (!frameResult.ok()) {
+            return frameResult.status();
+        }
+        const ov::Tensor& frame = frameResult.value();
+        // loadImage returns [1, H, W, C]; all frames must share the same H, W, C.
+        if (i == 0) {
+            frameShape = frame.get_shape();
+        } else if (frame.get_shape() != frameShape) {
+            return absl::InvalidArgumentError("All video frames must have the same height, width and channel count");
+        }
+        frames.push_back(frame);
+    }
+
+    const size_t numFrames = frames.size();
+    const size_t height = frameShape[1];
+    const size_t width = frameShape[2];
+    const size_t channels = frameShape[3];
+    ov::Tensor video(frames[0].get_element_type(), ov::Shape{numFrames, height, width, channels});
+
+    const size_t frameBytes = height * width * channels * video.get_element_type().size();
+    auto* dst = static_cast<uint8_t*>(video.data());
+    for (size_t i = 0; i < numFrames; i++) {
+        std::memcpy(dst + i * frameBytes, frames[i].data(), frameBytes);
+    }
+    SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Loaded video with {} frames of shape [{}, {}, {}]", numFrames, height, width, channels);
+    return video;
+}
+
+}  // namespace ovms