From 8764d2b061100b4f197a372fa2f98967b89ba96f Mon Sep 17 00:00:00 2001 From: luukunn <981429396@qq.com> Date: Fri, 27 Mar 2026 17:19:01 +0800 Subject: [PATCH 1/2] remove ENABLE_V1_DATA_PROCESSOR --- fastdeploy/engine/async_llm.py | 5 +- fastdeploy/engine/common_engine.py | 5 +- fastdeploy/entrypoints/engine_client.py | 4 +- fastdeploy/entrypoints/openai/serving_chat.py | 7 +- .../entrypoints/openai/serving_completion.py | 7 +- .../entrypoints/openai/serving_embedding.py | 29 +- .../entrypoints/openai/serving_reward.py | 30 +- fastdeploy/envs.py | 2 - fastdeploy/input/preprocess.py | 51 +- fastdeploy/input/v1/__init__.py | 15 - fastdeploy/input/v1/ernie4_5_processor.py | 641 -------- .../v1/ernie4_5_vl_processor/__init__.py | 28 - .../ernie4_5_vl_processor.py | 340 ---- .../image_preprocessor/__init__.py | 20 - .../get_image_preprocessor.py | 34 - .../image_preprocessor_adaptive.py | 587 ------- .../input/v1/ernie4_5_vl_processor/process.py | 751 --------- .../v1/ernie4_5_vl_processor/process_video.py | 205 --- .../utils/Roboto-Regular.ttf | Bin 146004 -> 0 bytes .../ernie4_5_vl_processor/utils/__init__.py | 15 - .../ernie4_5_vl_processor/utils/io_utils.py | 109 -- .../utils/render_timestamp.py | 103 -- .../utils/video_utils.py | 83 - .../v1/paddleocr_vl_processor/__init__.py | 20 - .../paddleocr_vl_processor/image_processor.py | 275 ---- .../paddleocr_vl_processor.py | 322 ---- .../v1/paddleocr_vl_processor/process.py | 622 ------- .../paddleocr_vl_processor/process_video.py | 82 - .../input/v1/qwen3_vl_processor/__init__.py | 23 - .../v1/qwen3_vl_processor/image_processor.py | 413 ----- .../input/v1/qwen3_vl_processor/process.py | 814 --------- .../qwen3_vl_processor/qwen3_vl_processor.py | 341 ---- .../input/v1/qwen_vl_processor/__init__.py | 23 - .../v1/qwen_vl_processor/image_processor.py | 442 ----- .../input/v1/qwen_vl_processor/process.py | 591 ------- .../v1/qwen_vl_processor/process_video.py | 100 -- .../v1/qwen_vl_processor/qwen_vl_processor.py | 338 ---- fastdeploy/input/v1/text_processor.py | 925 ----------- fastdeploy/inter_communicator/zmq_server.py | 21 +- tests/engine/test_common_engine.py | 2 - tests/entrypoints/openai/test_serving_chat.py | 68 +- tests/entrypoints/test_serving_completion.py | 36 +- tests/input/test_preprocess.py | 2 - tests/input/v1/test_ernie4_5_processor.py | 448 ----- tests/input/v1/test_ernie_processor.py | 162 -- tests/input/v1/test_ernie_vl_processor.py | 1460 ----------------- .../v1/test_image_preprocessor_adaptive.py | 499 ------ tests/input/v1/test_paddleocr_vl_processor.py | 1182 ------------- tests/input/v1/test_process_video.py | 386 ----- tests/input/v1/test_qwen3_vl_processor.py | 1172 ------------- tests/input/v1/test_qwen_vl_processor.py | 776 --------- tests/input/v1/test_text_processor.py | 586 ------- tests/input/v1/test_tokenizer_client.py | 101 -- tests/inter_communicator/test_zmq_server.py | 59 +- 54 files changed, 81 insertions(+), 15281 deletions(-) delete mode 100644 fastdeploy/input/v1/__init__.py delete mode 100644 fastdeploy/input/v1/ernie4_5_processor.py delete mode 100644 fastdeploy/input/v1/ernie4_5_vl_processor/__init__.py delete mode 100644 fastdeploy/input/v1/ernie4_5_vl_processor/ernie4_5_vl_processor.py delete mode 100644 fastdeploy/input/v1/ernie4_5_vl_processor/image_preprocessor/__init__.py delete mode 100644 fastdeploy/input/v1/ernie4_5_vl_processor/image_preprocessor/get_image_preprocessor.py delete mode 100644 fastdeploy/input/v1/ernie4_5_vl_processor/image_preprocessor/image_preprocessor_adaptive.py delete mode 100644 fastdeploy/input/v1/ernie4_5_vl_processor/process.py delete mode 100644 fastdeploy/input/v1/ernie4_5_vl_processor/process_video.py delete mode 100644 fastdeploy/input/v1/ernie4_5_vl_processor/utils/Roboto-Regular.ttf delete mode 100644 fastdeploy/input/v1/ernie4_5_vl_processor/utils/__init__.py delete mode 100644 fastdeploy/input/v1/ernie4_5_vl_processor/utils/io_utils.py delete mode 100644 fastdeploy/input/v1/ernie4_5_vl_processor/utils/render_timestamp.py delete mode 100644 fastdeploy/input/v1/ernie4_5_vl_processor/utils/video_utils.py delete mode 100644 fastdeploy/input/v1/paddleocr_vl_processor/__init__.py delete mode 100644 fastdeploy/input/v1/paddleocr_vl_processor/image_processor.py delete mode 100644 fastdeploy/input/v1/paddleocr_vl_processor/paddleocr_vl_processor.py delete mode 100644 fastdeploy/input/v1/paddleocr_vl_processor/process.py delete mode 100644 fastdeploy/input/v1/paddleocr_vl_processor/process_video.py delete mode 100644 fastdeploy/input/v1/qwen3_vl_processor/__init__.py delete mode 100644 fastdeploy/input/v1/qwen3_vl_processor/image_processor.py delete mode 100644 fastdeploy/input/v1/qwen3_vl_processor/process.py delete mode 100644 fastdeploy/input/v1/qwen3_vl_processor/qwen3_vl_processor.py delete mode 100644 fastdeploy/input/v1/qwen_vl_processor/__init__.py delete mode 100644 fastdeploy/input/v1/qwen_vl_processor/image_processor.py delete mode 100644 fastdeploy/input/v1/qwen_vl_processor/process.py delete mode 100644 fastdeploy/input/v1/qwen_vl_processor/process_video.py delete mode 100644 fastdeploy/input/v1/qwen_vl_processor/qwen_vl_processor.py delete mode 100644 fastdeploy/input/v1/text_processor.py delete mode 100644 tests/input/v1/test_ernie4_5_processor.py delete mode 100644 tests/input/v1/test_ernie_processor.py delete mode 100644 tests/input/v1/test_ernie_vl_processor.py delete mode 100644 tests/input/v1/test_image_preprocessor_adaptive.py delete mode 100644 tests/input/v1/test_paddleocr_vl_processor.py delete mode 100644 tests/input/v1/test_process_video.py delete mode 100644 tests/input/v1/test_qwen3_vl_processor.py delete mode 100644 tests/input/v1/test_qwen_vl_processor.py delete mode 100644 tests/input/v1/test_text_processor.py delete mode 100644 tests/input/v1/test_tokenizer_client.py diff --git a/fastdeploy/engine/async_llm.py b/fastdeploy/engine/async_llm.py index 3f99388d685..4afb3dc5c49 100644 --- a/fastdeploy/engine/async_llm.py +++ b/fastdeploy/engine/async_llm.py @@ -446,7 +446,7 @@ async def add_request( ) if envs.ZMQ_SEND_BATCH_DATA and self.connection_manager is not None: request["zmq_worker_pid"] = self.connection_manager.worker_pid - if not envs.ENABLE_V1_DATA_PROCESSOR and self.cfg.model_config.enable_mm: + if self.cfg.model_config.enable_mm: self.request_client.send_pyobj(request) else: self.request_client.send_json(request) @@ -543,8 +543,7 @@ async def generate( ) else: processed_output = response_item - if not envs.ENABLE_V1_DATA_PROCESSOR: - processed_output = RequestOutput.from_dict(processed_output) + processed_output = RequestOutput.from_dict(processed_output) # Enrich outputs with prompt metadata on the first packet if req_id: prompt_meta = self._prompt_metadata.get(req_id) diff --git a/fastdeploy/engine/common_engine.py b/fastdeploy/engine/common_engine.py index 28776b53ede..a2e72a1c844 100644 --- a/fastdeploy/engine/common_engine.py +++ b/fastdeploy/engine/common_engine.py @@ -1165,7 +1165,7 @@ def _insert_zmq_task_to_scheduler(self): while self.running: try: block = True if len(added_requests) == 0 else False - if not self.cfg.model_config.enable_mm and not envs.ENABLE_V1_DATA_PROCESSOR: + if not self.cfg.model_config.enable_mm: err, data = self.recv_request_server.receive_json_once(block) else: err, data = self.recv_request_server.receive_pyobj_once(block) @@ -1222,8 +1222,7 @@ def _insert_zmq_task_to_scheduler(self): continue err_msg = None try: - if not envs.ENABLE_V1_DATA_PROCESSOR: - request = Request.from_dict(data) + request = Request.from_dict(data) request.metrics.scheduler_recv_req_time = time.time() main_process_metrics.requests_number.inc() trace_carrier = data.get("trace_carrier") diff --git a/fastdeploy/entrypoints/engine_client.py b/fastdeploy/entrypoints/engine_client.py index 3f311e743fd..f03a18594de 100644 --- a/fastdeploy/entrypoints/engine_client.py +++ b/fastdeploy/entrypoints/engine_client.py @@ -437,7 +437,7 @@ async def add_requests(self, task): def _send_task(self, task): if envs.ZMQ_SEND_BATCH_DATA: task["zmq_worker_pid"] = self.worker_pid - if not self.enable_mm and not envs.ENABLE_V1_DATA_PROCESSOR: + if not self.enable_mm: self.zmq_client.send_json(task) else: if envs.FD_ENABLE_E2W_TENSOR_CONVERT: @@ -599,7 +599,7 @@ async def run_control_method(self, request: ControlRequest): req_dict = request.to_dict() if envs.ZMQ_SEND_BATCH_DATA: req_dict["zmq_worker_pid"] = self.worker_pid - if not self.enable_mm and not envs.ENABLE_V1_DATA_PROCESSOR: + if not self.enable_mm: self.zmq_client.send_json(req_dict) else: self.zmq_client.send_pyobj(req_dict) diff --git a/fastdeploy/entrypoints/openai/serving_chat.py b/fastdeploy/entrypoints/openai/serving_chat.py index 9d380b0db0c..09e06ffa0f6 100644 --- a/fastdeploy/entrypoints/openai/serving_chat.py +++ b/fastdeploy/entrypoints/openai/serving_chat.py @@ -26,7 +26,7 @@ import fastdeploy.envs as envs import fastdeploy.metrics.trace as tracing -from fastdeploy.engine.request import Request, RequestOutput +from fastdeploy.engine.request import RequestOutput from fastdeploy.entrypoints.openai.protocol import ( ChatCompletionRequest, ChatCompletionResponse, @@ -145,10 +145,7 @@ async def create_chat_completion(self, request: ChatCompletionRequest): prompt_tokens = None max_tokens = None try: - if not envs.ENABLE_V1_DATA_PROCESSOR: - current_req_dict = request.to_dict_for_infer(f"{request_id}_0") - else: - current_req_dict = Request.from_generic_request(request, request_id=f"{request_id}_0") + current_req_dict = request.to_dict_for_infer(f"{request_id}_0") if "chat_template" not in current_req_dict: current_req_dict["chat_template"] = self.chat_template current_req_dict["metrics"]["arrival_time"] = time.time() diff --git a/fastdeploy/entrypoints/openai/serving_completion.py b/fastdeploy/entrypoints/openai/serving_completion.py index 4caf9fe210a..9c2b386fce0 100644 --- a/fastdeploy/entrypoints/openai/serving_completion.py +++ b/fastdeploy/entrypoints/openai/serving_completion.py @@ -27,7 +27,7 @@ import fastdeploy.envs as envs import fastdeploy.metrics.trace as tracing -from fastdeploy.engine.request import Request, RequestOutput +from fastdeploy.engine.request import RequestOutput from fastdeploy.entrypoints.openai.protocol import ( CompletionLogprobs, CompletionRequest, @@ -178,10 +178,7 @@ async def create_completion(self, request: CompletionRequest): try: for idx, prompt in enumerate(request_prompts): request_id_idx = f"{request_id}_{idx}" - if not envs.ENABLE_V1_DATA_PROCESSOR: - current_req_dict = request.to_dict_for_infer(request_id_idx, prompt) - else: - current_req_dict = Request.from_generic_request(request, request_id=f"{request_id}_0") + current_req_dict = request.to_dict_for_infer(request_id_idx, prompt) current_req_dict["metrics"]["arrival_time"] = time.time() prompt_token_ids = await self.engine_client.format_and_add_data(current_req_dict) # tokenize if isinstance(prompt_token_ids, np.ndarray): diff --git a/fastdeploy/entrypoints/openai/serving_embedding.py b/fastdeploy/entrypoints/openai/serving_embedding.py index ec3223b3576..25f3f630510 100644 --- a/fastdeploy/entrypoints/openai/serving_embedding.py +++ b/fastdeploy/entrypoints/openai/serving_embedding.py @@ -15,20 +15,17 @@ """ import base64 -import time from collections.abc import AsyncGenerator from typing import Literal, Union import numpy as np from typing_extensions import assert_never, override -import fastdeploy.envs as envs from fastdeploy.engine.pooling_params import PoolingParams from fastdeploy.engine.request import ( EmbeddingOutput, EmbeddingRequestOutput, PoolingRequestOutput, - Request, ) from fastdeploy.entrypoints.openai.protocol import ( EmbeddingCompletionRequest, @@ -69,25 +66,13 @@ def __init__(self, engine_client, models, cfg, pid, ips, max_waiting_time, chat_ @override def _request_to_dict(self, ctx: ServeContext): request: EmbeddingRequest = ctx.request - if not envs.ENABLE_V1_DATA_PROCESSOR: - request_dict = super()._request_to_dict(ctx) - if hasattr(request, "to_pooling_params"): - pooling_params: PoolingParams = request.to_pooling_params() - pooling_params.verify("embed", self.cfg.model_config) - request_dict["pooling_params"] = pooling_params.to_dict() - request_dict["metrics"] = {} - return request_dict - else: - request_obj = None - if hasattr(request, "to_pooling_params"): - pooling_params: PoolingParams = request.to_pooling_params() - pooling_params.verify("embed", self.cfg.model_config) - request_obj = Request.from_generic_request( - req=request, request_id=ctx.request_id, pooling_params=pooling_params - ) - request_obj.metrics.arrival_time = time.time() - super()._process_chat_template_kwargs(request_obj) - return request_obj + request_dict = super()._request_to_dict(ctx) + if hasattr(request, "to_pooling_params"): + pooling_params: PoolingParams = request.to_pooling_params() + pooling_params.verify("embed", self.cfg.model_config) + request_dict["pooling_params"] = pooling_params.to_dict() + request_dict["metrics"] = {} + return request_dict @override def _request_to_batch_dicts(self, ctx: ServeContext): diff --git a/fastdeploy/entrypoints/openai/serving_reward.py b/fastdeploy/entrypoints/openai/serving_reward.py index cbde62deea5..cc3ed8a4729 100644 --- a/fastdeploy/entrypoints/openai/serving_reward.py +++ b/fastdeploy/entrypoints/openai/serving_reward.py @@ -14,14 +14,12 @@ # limitations under the License. """ -import time from collections.abc import AsyncGenerator from typing_extensions import override -import fastdeploy.envs as envs from fastdeploy.engine.pooling_params import PoolingParams -from fastdeploy.engine.request import PoolingRequestOutput, Request, RewardRequestOutput +from fastdeploy.engine.request import PoolingRequestOutput, RewardRequestOutput from fastdeploy.entrypoints.openai.protocol import ( ChatRewardData, ChatRewardRequest, @@ -46,25 +44,13 @@ def __init__(self, engine_client, models, cfg, pid, ips, max_waiting_time, chat_ @override def _request_to_dict(self, ctx: ServeContext): request: ChatRewardRequest = ctx.request - if not envs.ENABLE_V1_DATA_PROCESSOR: - request_dict = super()._request_to_dict(ctx) - if hasattr(request, "to_pooling_params"): - pooling_params: PoolingParams = request.to_pooling_params() - pooling_params.verify("reward", self.cfg.model_config) - request_dict["pooling_params"] = pooling_params.to_dict() - request_dict["metrics"] = {} - return request_dict - else: - request_obj: Request = None - if hasattr(request, "to_pooling_params"): - pooling_params: PoolingParams = request.to_pooling_params() - pooling_params.verify("reward", self.cfg.model_config) - request_obj = Request.from_generic_request( - req=request, request_id=ctx.request_id, pooling_params=pooling_params - ) - request_obj.metrics.arrival_time = time.time() - super()._process_chat_template_kwargs(request_obj) - return request_obj + request_dict = super()._request_to_dict(ctx) + if hasattr(request, "to_pooling_params"): + pooling_params: PoolingParams = request.to_pooling_params() + pooling_params.verify("reward", self.cfg.model_config) + request_dict["pooling_params"] = pooling_params.to_dict() + request_dict["metrics"] = {} + return request_dict @override def _request_to_batch_dicts(self, ctx: ServeContext): diff --git a/fastdeploy/envs.py b/fastdeploy/envs.py index 72cd6dc7c48..fef58eaf6cc 100644 --- a/fastdeploy/envs.py +++ b/fastdeploy/envs.py @@ -94,8 +94,6 @@ def _validate_split_kv_size(value: int) -> int: "EXPORTER_OTLP_HEADERS": lambda: os.getenv("EXPORTER_OTLP_HEADERS"), # enable kv cache block scheduler v1 (no need for kv_cache_ratio) "ENABLE_V1_KVCACHE_SCHEDULER": lambda: int(os.getenv("ENABLE_V1_KVCACHE_SCHEDULER", "1")), - # enable data processor v2 - "ENABLE_V1_DATA_PROCESSOR": lambda: int(os.getenv("ENABLE_V1_DATA_PROCESSOR", "0")), # set prealloc block num for decoder "FD_ENC_DEC_BLOCK_NUM": lambda: int(os.getenv("FD_ENC_DEC_BLOCK_NUM", "2")), # enbale max prefill of one execute step diff --git a/fastdeploy/input/preprocess.py b/fastdeploy/input/preprocess.py index 04c028d9060..56bbe1296f7 100644 --- a/fastdeploy/input/preprocess.py +++ b/fastdeploy/input/preprocess.py @@ -19,7 +19,6 @@ from fastdeploy.config import ErnieArchitectures, ModelConfig from fastdeploy.entrypoints.openai.tool_parsers import ToolParserManager from fastdeploy.reasoning import ReasoningParserManager -from fastdeploy.utils import envs from fastdeploy.utils import llm_logger as logger @@ -83,10 +82,7 @@ def create_processor(self): logger.info(f"Plugin input processor not available ({e}), using built-in processor") if not self.model_config.enable_mm: if not ErnieArchitectures.contains_ernie_arch(architecture): - if not envs.ENABLE_V1_DATA_PROCESSOR: - from fastdeploy.input.text_processor import DataProcessor - else: - from fastdeploy.input.v1.text_processor import DataProcessor + from fastdeploy.input.text_processor import DataProcessor self.processor = DataProcessor( model_name_or_path=self.model_name_or_path, @@ -94,14 +90,7 @@ def create_processor(self): tool_parser_obj=tool_parser_obj, ) else: - if not envs.ENABLE_V1_DATA_PROCESSOR: - from fastdeploy.input.ernie4_5_processor import ( - Ernie4_5Processor, - ) - else: - from fastdeploy.input.v1.ernie4_5_processor import ( - Ernie4_5Processor, - ) + from fastdeploy.input.ernie4_5_processor import Ernie4_5Processor self.processor = Ernie4_5Processor( model_name_or_path=self.model_name_or_path, @@ -110,14 +99,9 @@ def create_processor(self): ) else: if ErnieArchitectures.contains_ernie_arch(architecture): - if not envs.ENABLE_V1_DATA_PROCESSOR: - from fastdeploy.input.ernie4_5_vl_processor import ( - Ernie4_5_VLProcessor, - ) - else: - from fastdeploy.input.v1.ernie4_5_vl_processor import ( - Ernie4_5_VLProcessor, - ) + from fastdeploy.input.ernie4_5_vl_processor import ( + Ernie4_5_VLProcessor, + ) self.processor = Ernie4_5_VLProcessor( model_name_or_path=self.model_name_or_path, @@ -128,14 +112,9 @@ def create_processor(self): enable_processor_cache=self.enable_processor_cache, ) elif "PaddleOCRVL" in architecture: - if not envs.ENABLE_V1_DATA_PROCESSOR: - from fastdeploy.input.paddleocr_vl_processor import ( - PaddleOCRVLProcessor, - ) - else: - from fastdeploy.input.v1.paddleocr_vl_processor import ( - PaddleOCRVLProcessor, - ) + from fastdeploy.input.paddleocr_vl_processor import ( + PaddleOCRVLProcessor, + ) self.processor = PaddleOCRVLProcessor( config=self.model_config, @@ -145,12 +124,7 @@ def create_processor(self): reasoning_parser_obj=reasoning_parser_obj, ) elif "Qwen2_5_VL" in architecture: - if not envs.ENABLE_V1_DATA_PROCESSOR: - from fastdeploy.input.qwen_vl_processor import QwenVLProcessor - else: - from fastdeploy.input.v1.qwen_vl_processor import ( - QwenVLProcessor, - ) + from fastdeploy.input.qwen_vl_processor import QwenVLProcessor self.processor = QwenVLProcessor( config=self.model_config, @@ -161,12 +135,7 @@ def create_processor(self): enable_processor_cache=self.enable_processor_cache, ) elif "Qwen3VL" in architecture: - if not envs.ENABLE_V1_DATA_PROCESSOR: - from fastdeploy.input.qwen3_vl_processor import Qwen3VLProcessor - else: - from fastdeploy.input.v1.qwen3_vl_processor import ( - Qwen3VLProcessor, - ) + from fastdeploy.input.qwen3_vl_processor import Qwen3VLProcessor self.processor = Qwen3VLProcessor( config=self.model_config, diff --git a/fastdeploy/input/v1/__init__.py b/fastdeploy/input/v1/__init__.py deleted file mode 100644 index f4ede90624a..00000000000 --- a/fastdeploy/input/v1/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -""" -# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License" -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" diff --git a/fastdeploy/input/v1/ernie4_5_processor.py b/fastdeploy/input/v1/ernie4_5_processor.py deleted file mode 100644 index f6545dc068a..00000000000 --- a/fastdeploy/input/v1/ernie4_5_processor.py +++ /dev/null @@ -1,641 +0,0 @@ -""" -# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License" -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -import os - -import numpy as np -from paddleformers.generation import GenerationConfig - -from fastdeploy.input.ernie4_5_tokenizer import Ernie4_5Tokenizer -from fastdeploy.input.v1.text_processor import BaseDataProcessor -from fastdeploy.utils import data_processor_logger - -_SAMPLING_EPS = 1e-5 -from fastdeploy.input.utils import process_stop_token_ids - - -class Ernie4_5Processor(BaseDataProcessor): - """ - 初始化模型实例。 - - Args: - model_name_or_path (str): 模型名称或路径。 - - Attributes: - model_name_or_path (str): 存储模型名称或路径。 - decode_status (dict): 存储解码状态信息。 - tokenizer (object): 存储分词器实例。 - eos_token_ids (list): 存储结束符号的token ID列表。 - eos_token_id_len (int): 存储结束符号的token ID列表的长度。 - pad_token_id (int): 存储填充符号的token ID。 - """ - - def __init__(self, model_name_or_path, reasoning_parser_obj=None, tool_parser_obj=None): - - self.model_name_or_path = model_name_or_path - data_processor_logger.info(f"model_name_or_path: {model_name_or_path}") - - # Generation config - try: - self.generation_config = GenerationConfig.from_pretrained(self.model_name_or_path) - except Exception as e: - data_processor_logger.warning( - f"Can't find generation config, so it will not use " - f"generation_config field in the model config, details={e}" - ) - self.generation_config = None - - self.decode_status = dict() - self.tool_parser_dict = dict() - self.thinking_parser_dict = dict() - self.model_status_dict = dict() - self._load_tokenizer() - data_processor_logger.info( - f"tokenizer information: bos_token is {self.tokenizer.bos_token} \ - {self.tokenizer.bos_token_id}, \ - eos_token is {self.tokenizer.eos_token}, {self.tokenizer.eos_token_id} " - ) - try: - from paddleformers.trl.llm_utils import get_eos_token_id - except Exception: - from paddleformers.cli.utils.llm_utils import get_eos_token_id - - self.eos_token_ids = get_eos_token_id(self.tokenizer, self.generation_config) - self.eos_token_id_len = len(self.eos_token_ids) - self.pad_token_id = self.get_pad_id() - self.reasoning_parser = None - self.tool_parser_obj = tool_parser_obj - if reasoning_parser_obj: - self.reasoning_parser = reasoning_parser_obj(self.tokenizer) - - def process_request(self, request, max_model_len=None, **kwargs): - """ - Preprocess the request - - Args: - request (Dict): may contain text and messages fields - - Returns: - bool: Whether preprocessing is successful - str: error message - """ - data_processor_logger.info(f"Start processing request: {request}") - request = self._apply_default_parameters(request) - if request.get("eos_token_ids") is None or len(request.eos_token_ids) == 0: - request.eos_token_ids = self.eos_token_ids - - # processing stop_sequences and stop_token_ids - process_stop_token_ids(request, self.update_stop_seq) - - # processing bad_words - bad_words = request.get("bad_words") - bad_words_token_ids = request.get("bad_words_token_ids") - if bad_words: - bad_words_token_ids = self.update_bad_words(bad_words, bad_words_token_ids) - request["bad_words_token_ids"] = bad_words_token_ids - - logits_processors_args = self._prepare_think_stop_sentence( - request.get("logits_processors_args") or {}, max_model_len - ) - request["logits_processors_args"] = logits_processors_args - - # processing prompt_token_ids - if request.prompt_token_ids is None or len(request.prompt_token_ids) == 0: - if request.prompt is not None: - # prompt = request.prompt if request.prompt is not None else request.messages[0] - prompt = request.prompt - assert isinstance(prompt, str) or ( - isinstance(prompt, list) and all([isinstance(t, int) for t in prompt]) - ), f"prompt must be a string or a list of integers, but got {type(prompt)}" - - if isinstance(prompt, list): # if prompt is a token id list - request.prompt_token_ids = prompt - else: - tokens = self.tokenizer.tokenize(prompt) - token_ids = self.tokenizer.convert_tokens_to_ids(tokens) - request.prompt_token_ids = token_ids - data_processor_logger.debug( - f"request_ids: {request.request_id}, prompt: {prompt}, " - f"tokens: {tokens}, token_ids: {token_ids}" - ) - elif request.messages is not None: - task = request.to_dict() - chat_template_kwargs = kwargs.get("chat_template_kwargs", {}) - if chat_template_kwargs: - if isinstance(chat_template_kwargs, dict): - for k, v in chat_template_kwargs.items(): - if k not in task or task[k] is None: - task[k] = v - else: - raise ValueError("Invalid input: chat_template_kwargs must be a dict") - request.prompt_token_ids = self.messages2ids(task, **chat_template_kwargs) - else: - raise ValueError(f"The request should have `prompt_token_ids`, `prompt` or `messages`: {request}.") - - if len(request.prompt_token_ids) == 0: - raise ValueError("Invalid input: prompt_token_ids must be a non-empty sequence of token IDs") - - # truncate prompts that exceed the length limit - if max_model_len is not None and len(request.prompt_token_ids) > max_model_len: - request.prompt_token_ids = request.prompt_token_ids[: max_model_len - 1] - logits_processors_args = self._update_thinking_prompt_state( - request.prompt_token_ids, request.get("logits_processors_args") or {} - ) - request["logits_processors_args"] = logits_processors_args - max_tokens = max_model_len - len(request.prompt_token_ids) - if request.get("max_tokens") is None: - request.set("max_tokens", max(1, max_tokens)) - else: - request.set("max_tokens", min(max_tokens, request.get("max_tokens"))) - if request.get("temperature") < _SAMPLING_EPS: - # zero temperature is equivalent to greedy sampling - request.set("temperature", 1) - request.set("top_k", 1) - if request.get("top_p") < _SAMPLING_EPS: - request.set("top_p", _SAMPLING_EPS) - request.set("top_k", 1) - if self.reasoning_parser: - model_status = self.reasoning_parser.get_model_status(request.prompt_token_ids) - parts = request.request_id.split("_") - if len(parts) > 1: - real_req_id = parts[0] - index = int(parts[1]) - n = request.get("n", 1) - for idx in range(index * n, (index + 1) * n): - self.model_status_dict[f"{real_req_id}_{idx}"] = model_status - else: - self.model_status_dict[request.request_id] = model_status - request.enable_thinking = model_status == "think_start" - if request.get("response_max_tokens") is not None and request.enable_thinking is False: - request["max_tokens"] = min(request["response_max_tokens"], request["max_tokens"]) - - data_processor_logger.info(f"Processed request: {request}") - return request - - def process_request_dict(self, request, max_model_len=None, **kwargs): - """ - Preprocess the request - - Args: - request Request: may contain text and messages fields - - Returns: - bool: Whether preprocessing is successful - str: error message - """ - data_processor_logger.info(f"Start processing request: {request}") - request = self._apply_default_parameters(request) - if not request.eos_token_ids: - request.eos_token_ids = self.eos_token_ids - - # processing stop_sequences and stop_token_ids - process_stop_token_ids(request, self.update_stop_seq) - - # processing bad_words - bad_words = request.sampling_params.bad_words - bad_words_token_ids = request.sampling_params.bad_words_token_ids - if bad_words: - bad_words_token_ids = self.update_bad_words(bad_words, bad_words_token_ids) - request.sampling_params.bad_words_token_ids = bad_words_token_ids - - logits_processors_args = self._prepare_think_stop_sentence( - getattr(request.sampling_params, "logits_processors_args", None) or {}, max_model_len - ) - request.sampling_params.logits_processors_args = logits_processors_args - - # processing prompt_token_ids - if not request.prompt_token_ids: - if request.prompt: - prompt = request.prompt - assert isinstance(prompt, str) or ( - isinstance(prompt, list) and all([isinstance(t, int) for t in prompt]) - ), f"prompt must be a string or a list of integers, but got {type(prompt)}" - if isinstance(prompt, list): # if prompt is a token id list - request.prompt_token_ids = prompt - else: - request.prompt_tokens = prompt - tokens = self.tokenizer.tokenize(prompt) - token_ids = self.tokenizer.convert_tokens_to_ids(tokens) - request.prompt_token_ids = token_ids - req_id = request.request_id - data_processor_logger.info(f"req_id:{req_id}, tokens:{tokens}, token_ids: {token_ids}") - elif request.messages: - chat_template_kwargs = kwargs.get("chat_template_kwargs", {}) - if not chat_template_kwargs: - chat_template_kwargs = request.chat_template_kwargs if request.chat_template_kwargs else {} - if chat_template_kwargs: - if isinstance(chat_template_kwargs, dict): - for k, v in chat_template_kwargs.items(): - if not getattr(request, k, None): - setattr(request, k, v) - else: - raise ValueError("Invalid input: chat_template_kwargs must be a dict") - if getattr(request, "enable_thinking") is None: - setattr(request, "enable_thinking", True) - request.prompt_token_ids = self.messages2ids(request, **chat_template_kwargs) - delattr(request, "chat_template_kwargs") - else: - raise ValueError(f"Request must contain 'prompt_token_ids', 'prompt', or 'messages': {request}") - - if len(request.prompt_token_ids) == 0: - raise ValueError("Invalid input: prompt_token_ids must be a non-empty sequence of token IDs") - - # truncate prompts that exceed the length limit - if max_model_len is not None and len(request.prompt_token_ids) > max_model_len: - request.prompt_token_ids = request.prompt_token_ids[: max_model_len - 1] - logits_processors_args = self._update_thinking_prompt_state( - request.prompt_token_ids, getattr(request.sampling_params, "logits_processors_args", None) or {} - ) - request.sampling_params.logits_processors_args = logits_processors_args - max_tokens = max_model_len - len(request.prompt_token_ids) - if getattr(request.sampling_params, "max_tokens", None) is None: - request.sampling_params.max_tokens = max(1, max_tokens) - else: - request.sampling_params.max_tokens = min(max_tokens, request.sampling_params.max_tokens) - if request.sampling_params.temperature < _SAMPLING_EPS: - # zero temperature is equivalent to greedy sampling - request.sampling_params.temperature = 1 - request.sampling_params.top_k = 1 - if request.sampling_params.top_p < _SAMPLING_EPS: - request.sampling_params.top_p = _SAMPLING_EPS - request.sampling_params.top_k = 1 - - if self.reasoning_parser: - model_status = self.reasoning_parser.get_model_status(request.prompt_token_ids) - parts = request.request_id.split("_") - if len(parts) > 1: - real_req_id = parts[0] - index = int(parts[1]) - n = request.sampling_params.n or 1 - for idx in range(index * n, (index + 1) * n): - self.model_status_dict[f"{real_req_id}_{idx}"] = model_status - else: - self.model_status_dict[request.request_id] = model_status - request.enable_thinking = model_status == "think_start" - if request.sampling_params.response_max_tokens is not None and request.enable_thinking is False: - request.sampling_params.max_tokens = min( - request.sampling_params.response_max_tokens, request.sampling_params.max_tokens - ) - - data_processor_logger.info(f"Processed request: {request}") - return request - - def process_response(self, response_dict, **kwargs): - """ - Preprocess the response - - Args: - response_dict (Dict): response for engine, contain ids fields - - Returns: - Dict: response contain text fields - """ - req_id = response_dict.request_id - token_ids = response_dict.outputs.token_ids - - response_dict.usage = {"completion_tokens": response_dict.outputs.index + 1} - if token_ids[-1] == self.tokenizer.eos_token_id: - token_ids = token_ids[:-1] - full_text = self.tokenizer.decode(token_ids) - if self.reasoning_parser: - reasoning_content, text = self.reasoning_parser.extract_reasoning_content( - full_text, - response_dict, - self.model_status_dict[req_id], - ) - response_dict.outputs.text = text - response_dict.outputs.reasoning_content = reasoning_content - else: - response_dict.outputs.text = full_text - if self.tool_parser_obj: - tool_parser = self.tool_parser_obj(self.tokenizer) - tool_call_info = tool_parser.extract_tool_calls(full_text, response_dict) - if tool_call_info.tools_called: - response_dict.outputs.tool_calls = tool_call_info.tool_calls - response_dict.outputs.text = tool_call_info.content - if req_id in self.model_status_dict: - del self.model_status_dict[req_id] - data_processor_logger.info(f"req_id:{req_id}, token_ids: {token_ids}") - if response_dict.outputs.text == "" and response_dict.outputs.reasoning_content == "": - return None - return response_dict - - def process_response_dict(self, response_dict, stream, **kwargs): - """ - Preprocess the response - - Args: - response_dict (Dict): response for engine, contain ids fields - - Returns: - Dict: response contain text fields - """ - if stream: - return self.process_response_obj_streaming(response_dict, **kwargs) - else: - return self.process_response_obj_normal(response_dict, **kwargs) - - def process_response_obj_normal(self, response_obj, **kwargs): - """ - Preprocess the response - - Args: - response_obj : response for engine, contain ids fields - - Returns: - Dict: response contain text fields - """ - token_ids = response_obj.outputs.token_ids - is_end = response_obj.finished - req_id = response_obj.request_id - request = kwargs.get("request", None) - if is_end and len(token_ids) > 0 and not kwargs.get("include_stop_str_in_output"): - if token_ids[-1] == self.tokenizer.eos_token_id: - token_ids = token_ids[:-1] - delta_text, _, previous_texts = self.ids2tokens(token_ids, req_id) - if is_end: - full_text = previous_texts + delta_text - response_obj.outputs.text = full_text - if self.reasoning_parser: - reasoning_content, text = self.reasoning_parser.extract_reasoning_content( - full_text, - request, - self.model_status_dict[req_id], - ) - response_obj.outputs.text = text - response_obj.outputs.reasoning_content = reasoning_content - reasoning_tokens = self.tokenizer.tokenize(reasoning_content) - response_obj.outputs.reasoning_token_num = len(reasoning_tokens) - if self.tool_parser_obj: - tool_parser = self.tool_parser_obj(self.tokenizer) - tool_call_info = tool_parser.extract_tool_calls(full_text, request) - if tool_call_info.tools_called: - response_obj.outputs.tool_calls = tool_call_info.tool_calls - response_obj.outputs.text = tool_call_info.content - response_obj.outputs.completion_tokens = full_text - data_processor_logger.info(f"req_id:{req_id}, decode_status: {self.decode_status[req_id]}") - del self.decode_status[req_id] - if req_id in self.model_status_dict: - del self.model_status_dict[req_id] - return response_obj - - def process_response_obj_streaming(self, response_obj, **kwargs): - """ - Preprocess the response streaming - - Args: - response_obj : response for engine, contain ids fields - - Returns: - Dict: response contain text fields - """ - token_ids = response_obj.outputs.token_ids - is_end = response_obj.finished - req_id = response_obj.request_id - request = kwargs.get("request", None) - - if is_end and len(token_ids) > 0 and not kwargs.get("include_stop_str_in_output"): - if token_ids[-1] == self.tokenizer.eos_token_id: - token_ids = token_ids[:-1] - delta_text, previous_token_ids, previous_texts = self.ids2tokens(token_ids, req_id) - response_obj.outputs.completion_tokens = delta_text - if self.reasoning_parser: - reasoning_delta_message = self.reasoning_parser.extract_reasoning_content_streaming( - previous_texts, - previous_texts + delta_text, - delta_text, - previous_token_ids, - previous_token_ids + token_ids, - token_ids, - self.model_status_dict[req_id], - ) - response_obj.outputs.delta_message = reasoning_delta_message - reasoning_content = reasoning_delta_message.reasoning_content if reasoning_delta_message else None - reasoning_tokens = self.tokenizer.tokenize(reasoning_content) if reasoning_content else [] - response_obj.outputs.reasoning_token_num = len(reasoning_tokens) - response_obj.outputs.reasoning_token_num = len(reasoning_tokens) - response_obj.outputs.reasoning_content = reasoning_content - response_obj.outputs.text = ( - reasoning_delta_message.content or "" - if reasoning_delta_message and hasattr(reasoning_delta_message, "content") - else "" - ) - else: - response_obj.outputs.text = delta_text - if self.tool_parser_obj: - if req_id not in self.tool_parser_dict: - self.tool_parser_dict[req_id] = self.tool_parser_obj(self.tokenizer) - tool_parser = self.tool_parser_dict[req_id] - tool_call_delta_message = tool_parser.extract_tool_calls_streaming( - previous_texts, - previous_texts + delta_text, - delta_text, - previous_token_ids, - previous_token_ids + token_ids, - token_ids, - request, - ) - if tool_call_delta_message is None or tool_call_delta_message.tool_calls: - response_obj.outputs.delta_message = tool_call_delta_message - - if is_end: - data_processor_logger.info(f"req_id:{req_id}, decode_status: {self.decode_status[req_id]}") - del self.decode_status[req_id] - if req_id in self.tool_parser_dict: - del self.tool_parser_dict[req_id] - if req_id in self.model_status_dict: - del self.model_status_dict[req_id] - return response_obj - - def messages2ids(self, request_or_messages, **kwargs): - """ - Convert multi-turn messages into ID sequences. - - Args: - request_or_messages: Either a request dict containing 'messages' field, - or a list of message dicts directly - - Returns: - List of token IDs as strings (converted from token objects) - """ - if self.tokenizer.chat_template is None: - raise ValueError("This model does not support chat_template.") - message_dict = { - key: getattr(request_or_messages, key, None) - for key in ["messages", "tools", "documents", "enable_thinking", "system"] - if getattr(request_or_messages, key, None) is not None - } - spliced_message = self.tokenizer.apply_chat_template( - message_dict, - tokenize=False, - split_special_tokens=False, - add_special_tokens=False, - **kwargs, - ) - request_or_messages.prompt_tokens = spliced_message - req_id = getattr(request_or_messages, "request_id", None) - tokens = self.tokenizer.tokenize(spliced_message) - token_ids = self.tokenizer.convert_tokens_to_ids(tokens) - data_processor_logger.info(f"req_id:{req_id}, tokens:{tokens}, token_ids: {token_ids}") - return token_ids - - def ids2tokens(self, token_id, task_id): - """ - token ids to strings - - Args: - token_ids (List[int]): token ids - task_id (str): task id - - Returns: - List[str]: strings - """ - - if task_id not in self.decode_status: - # prefix offset & read offset & history token ids & history token strings - self.decode_status[task_id] = [0, 0, [], ""] - - prefix_offset = self.decode_status[task_id][0] - read_offset = self.decode_status[task_id][1] - previous_token_ids = self.decode_status[task_id][2] - previous_texts = self.decode_status[task_id][3] - decode_str, prefix_offset, read_offset = self.tokenizer.decode_token( - previous_token_ids + token_id, prefix_offset, read_offset - ) - self.decode_status[task_id][0] = prefix_offset - self.decode_status[task_id][1] = read_offset - self.decode_status[task_id][2] += token_id - self.decode_status[task_id][3] += decode_str - - return decode_str, previous_token_ids, previous_texts - - def _load_tokenizer(self): - """ - load tokenizer - - Returns: - tokenizer (AutoTokenizer) - """ - vocab_file_names = [ - "tokenizer.model", - "spm.model", - "ernie_token_100k.model", - ] - for i in range(len(vocab_file_names)): - if os.path.exists(os.path.join(self.model_name_or_path, vocab_file_names[i])): - Ernie4_5Tokenizer.resource_files_names["vocab_file"] = vocab_file_names[i] - break - self.tokenizer = Ernie4_5Tokenizer.from_pretrained(self.model_name_or_path) - - def get_pad_id(self): - """ - get pad_token_id, if not pad_token_id, use eos_token - - Returns: - int: pad_token_id - """ - # if isinstance(self.tokenizer, (LlamaTokenizer, Llama3Tokenizer)) and not self.tokenizer.pad_token_id: - # return self.tokenizer.eos_token - return self.tokenizer.pad_token_id - - def pad_batch_data( - self, - insts, - pad_id=0, - return_seq_len=False, - return_array=True, - pad_style="right", - ): - """Pad the instances to the max sequence length in batch.""" - if len(insts) == 0: - padded_insts = np.array([[]], dtype=np.int64) if return_array else [[]] - if return_seq_len: - seq_len = np.array([], dtype=np.int64) if return_array else [] - return padded_insts, seq_len - return padded_insts - - max_len = max(map(len, insts)) - if pad_style == "left": - padded_insts = [[pad_id] * (max_len - len(inst)) + list(inst) for inst in insts] - else: - padded_insts = [list(inst) + [pad_id] * (max_len - len(inst)) for inst in insts] - if return_array: - padded_insts = np.array(padded_insts, dtype=np.int64).reshape([-1, max_len]) - - if return_seq_len: - seq_len = [len(inst) for inst in insts] - if return_array: - seq_len = np.array(seq_len, dtype=np.int64).reshape(-1, 1) - return padded_insts, seq_len - return padded_insts - - def update_stop_seq(self, stop_sequences): - """ - Update stop sequences from request. - """ - stop_seqs = [] - if isinstance(stop_sequences, str): - stop_sequences = [stop_sequences] - for seq in stop_sequences: - if seq != self.tokenizer.eos_token_id: - stop_seqs.append(self.tokenizer.convert_tokens_to_ids(self.tokenizer.tokenize(seq))) - stop_seqs, stop_seqs_len = self.pad_batch_data(stop_seqs, pad_id=-1, return_seq_len=True, return_array=False) - data_processor_logger.debug(f"processed stop_seqs: {stop_seqs}, {stop_seqs_len}") - return stop_seqs, stop_seqs_len - - def process_logprob_response(self, token_ids, **kwargs): - full_text = self.tokenizer.decode(token_ids, **kwargs) - return full_text - - def update_bad_words(self, bad_words, bad_words_token_ids): - """Support bad words""" - - token_ids = bad_words_token_ids - - if token_ids is None: - token_ids = [] - for bad_word in bad_words: - # To prohibit words both at the beginning - # and in the middle of text - # (related to add_prefix_space tokenizer parameter) - for add_prefix_space in [False, True]: - prefix = " " if add_prefix_space else "" - prompt = prefix + bad_word.lstrip() - prompt_token_ids = self.tokenizer.convert_tokens_to_ids(self.tokenizer.tokenize(prompt)) - data_processor_logger.debug(f"processed bad_words: {prompt}, {prompt_token_ids}") - - if len(prompt_token_ids) != 1: - if not add_prefix_space: - data_processor_logger.warning( - f"Skip bad_words: <{prompt}>." - f"Bad words should be a single token." - f"Got tokens: {prompt_token_ids}." - ) - continue - - if prompt_token_ids[0] > self.tokenizer.vocab_size: - if not add_prefix_space: - data_processor_logger.warning( - f"Skip bad_words: <{prompt}>." - f"All token id values should be satisfying:" - f" 0 <= token_id < {self.tokenizer.vocab_size}." - f"Got token: {prompt_token_ids}." - ) - continue - - if prompt_token_ids not in token_ids: - token_ids.extend(prompt_token_ids) - return token_ids diff --git a/fastdeploy/input/v1/ernie4_5_vl_processor/__init__.py b/fastdeploy/input/v1/ernie4_5_vl_processor/__init__.py deleted file mode 100644 index f7d30a78d58..00000000000 --- a/fastdeploy/input/v1/ernie4_5_vl_processor/__init__.py +++ /dev/null @@ -1,28 +0,0 @@ -""" -# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -from .ernie4_5_vl_processor import Ernie4_5_VLProcessor -from .process import DataProcessor, fancy_print -from .process_video import read_video_decord -from .utils.video_utils import VideoReaderWrapper - -__all__ = [ - "DataProcessor", - "fancy_print", - "VideoReaderWrapper", - "read_video_decord", - "Ernie4_5_VLProcessor", -] diff --git a/fastdeploy/input/v1/ernie4_5_vl_processor/ernie4_5_vl_processor.py b/fastdeploy/input/v1/ernie4_5_vl_processor/ernie4_5_vl_processor.py deleted file mode 100644 index bae80b60a96..00000000000 --- a/fastdeploy/input/v1/ernie4_5_vl_processor/ernie4_5_vl_processor.py +++ /dev/null @@ -1,340 +0,0 @@ -""" -# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License" -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -import traceback - -import numpy as np -from paddleformers.generation import GenerationConfig - -from fastdeploy.engine.request import Request -from fastdeploy.input.utils import IDS_TYPE_FLAG, process_stop_token_ids -from fastdeploy.input.v1.ernie4_5_processor import Ernie4_5Processor -from fastdeploy.utils import data_processor_logger - -from .process import DataProcessor - -_SAMPLING_EPS = 1e-5 - - -class Ernie4_5_VLProcessor(Ernie4_5Processor): - """The processor class for ERNIE MoE VL models.""" - - def __init__( - self, - model_name_or_path, - limit_mm_per_prompt=None, - mm_processor_kwargs=None, - reasoning_parser_obj=None, - tool_parser_obj=None, - enable_processor_cache=False, - ): - data_processor_logger.info(f"model_name_or_path: {model_name_or_path}") - tokenizer_path = model_name_or_path - preprocessor_path = model_name_or_path - processor_kwargs = self._parse_processor_kwargs(mm_processor_kwargs) - - self.ernie4_5_processor = DataProcessor( - tokenizer_name=tokenizer_path, - image_preprocessor_name=preprocessor_path, - enable_processor_cache=enable_processor_cache, - **processor_kwargs, - ) - self.ernie4_5_processor.eval() - self.image_patch_id = self.ernie4_5_processor.image_patch_id - self.spatial_conv_size = self.ernie4_5_processor.spatial_conv_size - - self.tool_parser_dict = dict() - self.decode_status = dict() - self.model_status_dict = dict() - self._load_tokenizer() - - # Generation config - try: - self.generation_config = GenerationConfig.from_pretrained(model_name_or_path) - except Exception as e: - data_processor_logger.warning( - f"Can't find generation config: {e}, so it will not use generation_config field in the model config" - ) - self.generation_config = None - - # self.eos_token_ids = [self.tokenizer.eos_token_id] - try: - from paddleformers.trl.llm_utils import get_eos_token_id - except Exception: - from paddleformers.cli.utils.llm_utils import get_eos_token_id - - self.eos_token_ids = get_eos_token_id(self.tokenizer, self.generation_config) - self.eos_token_id_len = len(self.eos_token_ids) - self.pad_token_id = self.get_pad_id() - self.limit_mm_per_prompt = self._parse_limits(limit_mm_per_prompt) - self.reasoning_parser = None - if reasoning_parser_obj: - self.reasoning_parser = reasoning_parser_obj(self.tokenizer) - self.tool_parser_obj = tool_parser_obj - - def get_pad_id(self): - """get pad id""" - return self.tokenizer.pad_token_id - - def _load_tokenizer(self): - """ - load tokenizer - - Returns: - tokenizer (AutoTokenizer) - """ - self.tokenizer = self.ernie4_5_processor.tokenizer - - def _apply_default_parameters(self, request): - """ - Apply default value for parameters in request - """ - - def set_value(req, key, value): - value = getattr(self.generation_config, key, value) - if getattr(req.sampling_params, key) is None: - setattr(req.sampling_params, key, value) - - set_value(request, "top_p", 0.7) - set_value(request, "temperature", 1.0) - set_value(request, "repetition_penalty", 1.0) - set_value(request, "frequency_penalty", 0.0) - set_value(request, "presence_penalty", 0.0) - return request - - def _parse_processor_kwargs(self, kwargs): - """解析多模态处理器参数配置""" - if not kwargs: - return {} - - try: - if not isinstance(kwargs, dict): - raise ValueError("mm-processor-kwargs must be a dictionary") - - # 验证参数类型 - data_processor_logger.info(f"kwargs:{kwargs}") - expected_types = { - "spatial_conv_size": int, - "temporal_conv_size": int, - "image_min_pixels": int, - "image_max_pixels": int, - "video_min_pixels": int, - "video_max_pixels": int, - "video_target_frames": int, - "video_frames_sample": str, - "video_max_frames": int, - "video_min_frames": int, - "video_fps": int, - } - - for key, value in kwargs.items(): - if key in expected_types and not isinstance(value, expected_types[key]): - raise ValueError( - f"Invalid type for {key}: expected {expected_types[key].__name__}, got {type(value).__name__}" - ) - - return kwargs - - except Exception as e: - data_processor_logger.warning(f"Invalid mm-processor-kwargs format: {e}, {str(traceback.format_exc())}") - return {} - - def _parse_limits(self, limits): - """解析多模态限制配置""" - DEFAULT_LIMITS = {"image": 1, "video": 1, "audio": 1} - - if not limits: - return DEFAULT_LIMITS - - try: - if not isinstance(limits, dict): - raise ValueError("limit-mm-per-prompt must be a dictionary") - data_processor_logger.info(f"_parse_limits:{limits}") - return {**DEFAULT_LIMITS, **limits} - except Exception as e: - data_processor_logger.warning(f"Invalid limit-mm-per-prompt format: {e}, using default limits") - return DEFAULT_LIMITS - - def _check_mm_limits(self, item): - if isinstance(item, dict): - # 请求包含prompt和multi_modal_data - mm_data = item - else: - # 请求包含messages - mm_data = {"image": [], "video": []} - - for message in item: - if isinstance(message.get("content"), list): - for part in message["content"]: - if part.get("type") == "image": - mm_data["image"].append(part) - elif part.get("type") == "video": - mm_data["video"].append(part) - - for modality, data in mm_data.items(): - if modality in self.limit_mm_per_prompt: - limit = self.limit_mm_per_prompt[modality] - if len(data) > limit: - raise ValueError(f"Too many {modality} items in prompt, " f"got {len(data)} but limit is {limit}") - - def process_request(self, request, max_model_len=None, **kwargs): - """process the input data""" - task = request.to_dict() - task["chat_template_kwargs"] = kwargs.get("chat_template_kwargs") - self.process_request_dict(task, max_model_len) - request = Request.from_dict(task) - request = self._apply_default_parameters(request) - - return request - - def process_request_dict(self, request, max_model_len=None, **kwargs): - """process the input data""" - - request = self._apply_default_parameters(request) - if not request.eos_token_ids: - request.eos_token_ids = self.eos_token_ids - - # processing stop_sequences and stop_token_ids - process_stop_token_ids(request, self.update_stop_seq) - - bad_words = request.sampling_params.bad_words - bad_words_token_ids = request.sampling_params.bad_words_token_ids - if bad_words: - bad_words_token_ids = self.update_bad_words(bad_words, bad_words_token_ids) - request.sampling_params.bad_words_token_ids = bad_words_token_ids - - logits_processors_args = self._prepare_think_stop_sentence( - getattr(request.sampling_params, "logits_processors_args", None) or {}, max_model_len - ) - request.sampling_params.logits_processors_args = logits_processors_args - - if request.prompt_token_ids: - messages = request.messages - if messages: - self._check_mm_limits(messages) - if getattr(request, "enable_thinking") is None: - setattr(request, "enable_thinking", True) - outputs = self.ernie4_5_processor.prompt_token_ids2outputs(request) - elif request.prompt: - multimodal_data = request.multimodal_data - if multimodal_data is None: - multimodal_data = {} - self._check_mm_limits(multimodal_data) - images = multimodal_data.get("image", None) - videos = multimodal_data.get("video", None) - request.prompt_tokens = request.prompt - outputs = self.ernie4_5_processor.text2ids(request.prompt, images, videos) - elif request.messages: - messages = request.messages - self._check_mm_limits(messages) - chat_template_kwargs = kwargs.get("chat_template_kwargs", {}) - if not chat_template_kwargs: - chat_template_kwargs = request.chat_template_kwargs - if chat_template_kwargs: - if isinstance(chat_template_kwargs, dict): - for k, v in chat_template_kwargs.items(): - if getattr(request, k, None) is None: - setattr(request, k, v) - else: - raise ValueError("Invalid input: chat_template_kwargs must be a dict") - if getattr(request, "enable_thinking") is None: - setattr(request, "enable_thinking", True) - outputs = self.ernie4_5_processor.request2ids(request) - delattr(request, "chat_template_kwargs") - else: - raise ValueError(f"Request must contain 'prompt', or 'messages': {request}") - - if request.completion_token_ids: - self.append_completion_tokens(outputs, request.completion_token_ids) - - outputs = self.pack_outputs(outputs) - request.prompt_token_ids = ( - outputs["input_ids"].tolist() - if not getattr(request, "prompt_token_ids", None) - else request.prompt_token_ids - ) - request.prompt_token_ids_len = len(request.prompt_token_ids) - request.multimodal_inputs = outputs - - # 截断超过长度限制的prompt - if max_model_len is not None and len(request.prompt_token_ids) > max_model_len: - request.prompt_token_ids = request.prompt_token_ids[: max_model_len - 1] - logits_processors_args = self._update_thinking_prompt_state( - request.prompt_token_ids, getattr(request.sampling_params, "logits_processors_args", None) or {} - ) - request.sampling_params.logits_processors_args = logits_processors_args - - max_tokens = max_model_len - len(request.prompt_token_ids) - if getattr(request.sampling_params, "max_tokens", None) is None: - request.sampling_params.max_tokens = max(1, max_tokens) - else: - request.sampling_params.max_tokens = min(max_tokens, request.sampling_params.max_tokens) - if request.sampling_params.reasoning_max_tokens is None: - request.sampling_params.reasoning_max_tokens = max(int(request.sampling_params.max_tokens * 0.8), 1) - request.reasoning_max_tokens = request.sampling_params.reasoning_max_tokens - data_processor_logger.info(f"Processed request {request}") - - if self.reasoning_parser: - model_status = self.reasoning_parser.get_model_status(request.prompt_token_ids) - parts = request.request_id.split("_") - if len(parts) > 1: - real_req_id = parts[0] - index = int(parts[1]) - n = request.sampling_params.n or 1 - for idx in range(index * n, (index + 1) * n): - self.model_status_dict[f"{real_req_id}_{idx}"] = model_status - else: - self.model_status_dict[request.request_id] = model_status - request.enable_thinking = model_status == "think_start" - if request.sampling_params.top_p is not None and request.sampling_params.top_p < _SAMPLING_EPS: - request.sampling_params.top_p = _SAMPLING_EPS - request.sampling_params.top_k = 1 - if request.sampling_params.response_max_tokens is not None and request.enable_thinking is False: - request.sampling_params.max_tokens = min( - request.sampling_params.response_max_tokens, request.sampling_params.max_tokens - ) - return request - - def append_completion_tokens(self, multimodal_inputs, completion_token_ids): - "append already completion tokens" - - num_tokens = len(completion_token_ids) - multimodal_inputs["input_ids"].extend(completion_token_ids) - multimodal_inputs["token_type_ids"].extend([IDS_TYPE_FLAG["text"]] * num_tokens) - - start = multimodal_inputs["cur_position"] - for i in range(num_tokens): - multimodal_inputs["position_ids"].append([start + i] * 3) - multimodal_inputs["cur_position"] += num_tokens - - def pack_outputs(self, outs): - # Stack or nullify image-related fields - if not outs["images"]: - outs["images"] = None - outs["grid_thw"] = None - outs["image_type_ids"] = None - else: - outs["images"] = np.vstack(outs["images"]) - outs["grid_thw"] = np.vstack(outs["grid_thw"]) - outs["image_type_ids"] = np.array(outs["image_type_ids"]) - - outs["image_patch_id"] = self.image_patch_id - # Convert lists to arrays - outs["input_ids"] = np.array(outs["input_ids"], dtype=np.int64) - outs["token_type_ids"] = np.array(outs["token_type_ids"], dtype=np.int64) - outs["position_ids"] = np.array(outs["position_ids"], dtype=np.int64) - outs["mm_num_token_func"] = self.ernie4_5_processor.mm_num_tokens - return outs diff --git a/fastdeploy/input/v1/ernie4_5_vl_processor/image_preprocessor/__init__.py b/fastdeploy/input/v1/ernie4_5_vl_processor/image_preprocessor/__init__.py deleted file mode 100644 index c11444e6758..00000000000 --- a/fastdeploy/input/v1/ernie4_5_vl_processor/image_preprocessor/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -""" -# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -from .get_image_preprocessor import get_image_preprocessor -from .image_preprocessor_adaptive import AdaptiveImageProcessor - -__all__ = ["get_image_preprocessor", "AdaptiveImageProcessor"] diff --git a/fastdeploy/input/v1/ernie4_5_vl_processor/image_preprocessor/get_image_preprocessor.py b/fastdeploy/input/v1/ernie4_5_vl_processor/image_preprocessor/get_image_preprocessor.py deleted file mode 100644 index 0ff6f7d1ed5..00000000000 --- a/fastdeploy/input/v1/ernie4_5_vl_processor/image_preprocessor/get_image_preprocessor.py +++ /dev/null @@ -1,34 +0,0 @@ -""" -# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -"""get image preprocessor""" - -from fastdeploy.utils import data_processor_logger - -from .image_preprocessor_adaptive import AdaptiveImageProcessor - - -def get_image_preprocessor(args): - """ - get_image_preprocessor from args - """ - - if args.vision_model_name_or_path is None: - return None - - data_processor_logger.info("use AdaptiveImageProcessor") - image_preprocess = AdaptiveImageProcessor.from_pretrained(args.vision_model_name_or_path) - return image_preprocess diff --git a/fastdeploy/input/v1/ernie4_5_vl_processor/image_preprocessor/image_preprocessor_adaptive.py b/fastdeploy/input/v1/ernie4_5_vl_processor/image_preprocessor/image_preprocessor_adaptive.py deleted file mode 100644 index 6dcdf3a4e96..00000000000 --- a/fastdeploy/input/v1/ernie4_5_vl_processor/image_preprocessor/image_preprocessor_adaptive.py +++ /dev/null @@ -1,587 +0,0 @@ -""" -# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -"""image preprocessor adaptive""" - -import math -from typing import List, Optional, Union - -import numpy as np -import paddle -import PIL -from paddleformers.transformers.feature_extraction_utils import BatchFeature -from paddleformers.transformers.image_processing_utils import BaseImageProcessor -from paddleformers.transformers.image_transforms import ( - convert_to_rgb, - normalize, - rescale, - resize, - to_channel_dimension_format, -) -from paddleformers.transformers.image_utils import ( - ChannelDimension, - ImageInput, - PILImageResampling, - get_image_size, - infer_channel_dimension_format, - is_valid_image, - make_list_of_images, - to_numpy_array, - valid_images, -) -from paddleformers.transformers.legacy.tokenizer_utils_base import TensorType -from PIL import Image - -from fastdeploy.utils import data_processor_logger - -OPENAI_CLIP_MEAN = [0.48145466, 0.4578275, 0.40821073] -OPENAI_CLIP_STD = [0.26862954, 0.26130258, 0.27577711] - -IMAGE_FACTOR = 28 -MIN_PIXELS = 4 * 28 * 28 -MAX_PIXELS = 16384 * 28 * 28 -MAX_RATIO = 200 - - -VideoInput = Union[ - List["PIL.Image.Image"], - "np.ndarray", - "paddle.Tensor", - List["np.ndarray"], - List["paddle.Tensor"], - List[List["PIL.Image.Image"]], - List[List["np.ndarrray"]], - List[List["paddle.Tensor"]], -] - - -__all__ = [ - "AdaptiveImageProcessor", -] - - -def is_scaled_image(image: np.ndarray) -> bool: - """ - Checks to see whether the pixel values have already been rescaled to [0, 1]. - """ - if image.dtype == np.uint8: - return False - - # It's possible the image has pixel values in [0, 255] but is of floating type - return np.min(image) >= 0 and np.max(image) <= 1 - - -def make_batched_images(images) -> List[List[ImageInput]]: - """ - Accepts images in list or nested list format, and makes a list of images for preprocessing. - - Args: - images (`Union[List[List[ImageInput]], List[ImageInput], ImageInput]`): - The input image. - - Returns: - list: A list of images. - """ - if isinstance(images, (list, tuple)) and isinstance(images[0], (list, tuple)) and is_valid_image(images[0][0]): - return [img for img_list in images for img in img_list] - - elif isinstance(images, (list, tuple)) and is_valid_image(images[0]): - return images - - elif is_valid_image(images): - return [images] - - raise ValueError(f"Could not make batched images from {images}") - - -# Copied from transformers.models.llava_next_video.image_processing_llava_next_video.make_batched_videos -def make_batched_videos(videos) -> List[VideoInput]: - """dummy""" - if isinstance(videos, (list, tuple)) and isinstance(videos[0], (list, tuple)) and is_valid_image(videos[0][0]): - return videos - - elif isinstance(videos, (list, tuple)) and is_valid_image(videos[0]): - if isinstance(videos[0], Image.Image): - return [videos] - elif len(videos[0].shape) == 4: - return [list(video) for video in videos] - - elif is_valid_image(videos) and len(videos.shape) == 4: - return [list(videos)] - - raise ValueError(f"Could not make batched video from {videos}") - - -class AdaptiveImageProcessor(BaseImageProcessor): - r""" - Constructs a adaptive image processor that dynamically resizes images based on the original images. - - Args: - do_resize (`bool`, *optional*, defaults to `True`): - Whether to resize the image's (height, width) dimensions. - resample (`PILImageResampling`, *optional*, defaults to `Resampling.BICUBIC`): - Resampling filter to use when resizing the image. - do_rescale (`bool`, *optional*, defaults to `True`): - Whether to rescale the image by the specified scale `rescale_factor`. - rescale_factor (`int` or `float`, *optional*, defaults to `1/255`): - Scale factor to use if rescaling the image. - do_normalize (`bool`, *optional*, defaults to `True`): - Whether to normalize the image. - image_mean (`float` or `List[float]`, *optional*, defaults to `[0.48145466, 0.4578275, 0.40821073]`): - Mean to use if normalizing the image. This is a float or list of floats for each channel in the image. - image_std (`float` or `List[float]`, *optional*, defaults to `[0.26862954, 0.26130258, 0.27577711]`): - Standard deviation to use if normalizing the image. This is a float or list of floats for each channel - in the image. - do_convert_rgb (`bool`, *optional*, defaults to `True`): - Whether to convert the image to RGB. - min_pixels (`int`, *optional*, defaults to `56 * 56`): - The min pixels of the image to resize the image. - max_pixels (`int`, *optional*, defaults to `28 * 28 * 1280`): - The max pixels of the image to resize the image. - patch_size (`int`, *optional*, defaults to 14): - The spacial patch size of the vision encoder. - temporal_conv_size (`int`, *optional*, defaults to 2): - The temporal conv size in resampler. - merge_size (`int`, *optional*, defaults to 2): - The merge size of the vision encoder to llm encoder. - """ - - model_input_names = [ - "pixel_values", - "image_grid_thw", - "pixel_values_videos", - "video_grid_thw", - ] - - def __init__( - self, - do_resize: bool = True, - resample: PILImageResampling = PILImageResampling.BICUBIC, - do_rescale: bool = True, - rescale_factor: float = 1 / 255, - do_normalize: bool = True, - image_mean: Optional[Union[float, List[float]]] = None, - image_std: Optional[Union[float, List[float]]] = None, - do_convert_rgb: bool = True, - min_pixels: int = 56 * 56, - max_pixels: int = 28 * 28 * 1280, - patch_size: int = 14, - temporal_conv_size: int = 2, - merge_size: int = 2, - **kwargs, - ) -> None: - """init""" - super().__init__(**kwargs) - self.do_resize = do_resize - self.resample = resample - self.do_rescale = do_rescale - self.rescale_factor = rescale_factor - self.do_normalize = do_normalize - self.image_mean = image_mean if image_mean is not None else OPENAI_CLIP_MEAN - self.image_std = image_std if image_std is not None else OPENAI_CLIP_STD - self.min_pixels = min_pixels - self.max_pixels = max_pixels - self.patch_size = patch_size - self.temporal_conv_size = temporal_conv_size - self.merge_size = merge_size - self.size = {"min_pixels": min_pixels, "max_pixels": max_pixels} - self.do_convert_rgb = do_convert_rgb - - def set_pixels(self, min_pixels=None, max_pixels=None, msg=""): - """设定pixels""" - if min_pixels is not None: - assert isinstance(min_pixels, int) and min_pixels >= 0, "min_pixels must be positive int" - data_processor_logger.info(f"{msg} AdaptiveImageProcessor set min_pixels = {min_pixels}") - self.min_pixels = min_pixels - self.size["min_pixels"] = int(min_pixels) - if max_pixels is not None: - assert isinstance(max_pixels, int) and max_pixels > 0, "max_pixels must be positive int" - data_processor_logger.info(f"{msg} AdaptiveImageProcessor set max_pixels = {max_pixels}") - self.max_pixels = max_pixels - self.size["max_pixels"] = int(max_pixels) - - def get_smarted_resize(self, height, width, min_pixels=None, max_pixels=None): - """dummy""" - actual_min_pixels = min_pixels if min_pixels is not None else self.min_pixels - actual_max_pixels = max_pixels if max_pixels is not None else self.max_pixels - resized_height, resized_width = smart_resize( - height, - width, - factor=self.patch_size * self.merge_size, - min_pixels=actual_min_pixels, - max_pixels=actual_max_pixels, - ) - return (resized_height, resized_width), ( - resized_height // self.patch_size, - resized_width // self.patch_size, - ) - - def _preprocess( - self, - images: Union[ImageInput, VideoInput], - do_resize: bool = True, - resample: PILImageResampling = None, - do_rescale: bool = True, - rescale_factor: float = 1 / 255, - do_normalize: bool = True, - image_mean: Optional[Union[float, List[float]]] = None, - image_std: Optional[Union[float, List[float]]] = None, - do_convert_rgb: bool = False, - data_format: Optional[ChannelDimension] = ChannelDimension.FIRST, - input_data_format: Optional[Union[str, ChannelDimension]] = None, - predetermined_grid_thw=None, - ): - """ - Preprocess an image or batch of images. Copy of the `preprocess` method from `CLIPImageProcessor`. - - Args: - images (`ImageInput`): - Image or batch of images to preprocess. Expects pixel values ranging from 0 to 255. - If pixel values range from 0 to 1, set `do_rescale=False`. - vision_info (`List[Dict]`, *optional*): - Optional list of dictionaries containing additional information about vision inputs. - do_resize (`bool`, *optional*, defaults to `self.do_resize`): - Whether to resize the image. - resample (`PILImageResampling`, *optional*, defaults to `self.resample`): - Resampling filter to use if resizing the image. This can be one of the `PILImageResampling` enums. - do_rescale (`bool`, *optional*, defaults to `self.do_rescale`): - Whether to rescale the image. - rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`): - Scale factor to use if rescaling the image. - do_normalize (`bool`, *optional*, defaults to `self.do_normalize`): - Whether to normalize the image. - image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`): - Mean to use if normalizing the image. - Can be a float or a list of floats corresponding to the number of channels in the image. - image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`): - Standard deviation to use if normalizing the image. - Can be a float or a list of floats corresponding to the number of channels in the image. - do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`): - Whether to convert the image to RGB. - data_format (`ChannelDimension`, *optional*, defaults to `ChannelDimension.FIRST`): - The channel dimension format for the output image. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Unset: Use the channel dimension format of the input image. - input_data_format (`ChannelDimension` or `str`, *optional*): - The channel dimension format for the input image. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - """ - images = make_list_of_images(images) - - if do_convert_rgb: - images = [convert_to_rgb(image) for image in images] - - # All transformations expect numpy arrays. - images = [to_numpy_array(image) for image in images] - - if is_scaled_image(images[0]) and do_rescale: - data_processor_logger.warning( - "It looks like you are trying to rescale already rescaled images. If the input" - " images have pixel values between 0 and 1, set `do_rescale=False` to avoid rescaling them again." - ) - if input_data_format is None: - # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(images[0]) - - height, width = get_image_size(images[0], channel_dim=input_data_format) - resized_height, resized_width = height, width - processed_images = [] - - if predetermined_grid_thw is not None: - assert len(predetermined_grid_thw) == len( - images - ), f"len(predetermined_grid_thw) {len(predetermined_grid_thw)} == len(images) {len(images)}" - - for img_idx, image in enumerate(images): - if do_resize: - if predetermined_grid_thw is not None: - (resized_height, resized_width) = predetermined_grid_thw[img_idx] - resized_height *= self.patch_size - resized_width *= self.patch_size - else: - resized_height, resized_width = smart_resize( - height, - width, - factor=self.patch_size * self.merge_size, - min_pixels=self.min_pixels, - max_pixels=self.max_pixels, - ) - image = image.astype("uint8") # TODO : 需要手动加上，否则多除255 导致结果会出错 - # 直接fromarray，不要靠paddleformers里面的 - image = Image.fromarray(image) - image = resize( - image, - size=(resized_height, resized_width), - resample=resample, - data_format=input_data_format, - ) - if do_rescale: - image = rescale(image, scale=rescale_factor, data_format=input_data_format) - - if do_normalize: - image = normalize( - image=image, - mean=image_mean, - std=image_std, - data_format=input_data_format, - ) - - image = to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) # [C, H, W] - - processed_images.append(image) - patches = np.array(processed_images) - if data_format == ChannelDimension.LAST: - patches = patches.transpose([0, 3, 1, 2]) - - channel = patches.shape[1] # [time, C, H, W] - grid_t = patches.shape[0] - grid_h, grid_w = ( - resized_height // self.patch_size, - resized_width // self.patch_size, - ) - patches = patches.reshape( - [ - grid_t, - channel, - grid_h // self.merge_size, - self.merge_size, - self.patch_size, - grid_w // self.merge_size, - self.merge_size, - self.patch_size, - ] - ) - # [grid_t, grid_h/merge_size, grid_w/merge_size, merge_size, merge_size, C, psz, psz] - patches = patches.transpose([0, 2, 5, 3, 6, 1, 4, 7]) - - flatten_patches = patches.reshape( - [ - grid_t * grid_h * grid_w, - channel * self.patch_size * self.patch_size, - ] - ) # [grid_t * grid_h * grid_w, C * psz * psz] - - return flatten_patches, (grid_t, grid_h, grid_w) - - def preprocess( - self, - images: ImageInput, - videos: VideoInput = None, - do_resize: bool = True, - size: Optional[Union[int, List[int]]] = None, - resample: PILImageResampling = None, - do_rescale: bool = True, - rescale_factor: float = 1 / 255, - do_normalize: bool = True, - image_mean: Optional[Union[float, List[float]]] = None, - image_std: Optional[Union[float, List[float]]] = None, - do_convert_rgb: bool = False, - return_tensors: Optional[Union[str, TensorType]] = None, - data_format: Optional[ChannelDimension] = ChannelDimension.FIRST, - input_data_format: Optional[Union[str, ChannelDimension]] = None, - predetermined_grid_thw=None, - ): - """ - Args: - images (`ImageInput`): - Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If - passing in images with pixel values between 0 and 1, set `do_rescale=False`. - videos (`VideoInput`): - Video to preprocess. Expects a single or batch of videos with pixel values ranging from 0 to 255. If - passing in videos with pixel values between 0 and 1, set `do_rescale=False`. - do_resize (`bool`, *optional*, defaults to `self.do_resize`): - Whether to resize the image. - size (`Dict[str, int]`, *optional*, defaults to `self.size`): - Size of the image after resizing. Shortest edge of the image is resized to size["shortest_edge"], with - the longest edge resized to keep the input aspect ratio. - resample (`int`, *optional*, defaults to `self.resample`): - Resampling filter to use if resizing the image. This can be one of the enum `PILImageResampling`. Only - has an effect if `do_resize` is set to `True`. - do_rescale (`bool`, *optional*, defaults to `self.do_rescale`): - Whether to rescale the image. - rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`): - Rescale factor to rescale the image by if `do_rescale` is set to `True`. - do_normalize (`bool`, *optional*, defaults to `self.do_normalize`): - Whether to normalize the image. - image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`): - Image mean to use for normalization. Only has an effect if `do_normalize` is set to `True`. - image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`): - Image standard deviation to use for normalization. Only has an effect if `do_normalize` is set to - `True`. - do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`): - Whether to convert the image to RGB. - return_tensors (`str` or `TensorType`, *optional*): - The type of tensors to return. Can be one of: - - Unset: Return a list of `np.ndarray`. - - `TensorType.PADDLE` or `'pt'`: Return a batch of type `torch.Tensor`. - - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`. - data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`): - The channel dimension format for the output image. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Unset: Use the channel dimension format of the input image. - input_data_format (`ChannelDimension` or `str`, *optional*): - The channel dimension format for the input image. If unset, the channel dimension format is inferred - from the input image. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - - """ - do_resize = do_resize if do_resize is not None else self.do_resize - size = size if size is not None else self.size - resample = resample if resample is not None else self.resample - do_rescale = do_rescale if do_rescale is not None else self.do_rescale - rescale_factor = rescale_factor if rescale_factor is not None else self.rescale_factor - do_normalize = do_normalize if do_normalize is not None else self.do_normalize - image_mean = image_mean if image_mean is not None else self.image_mean - image_std = image_std if image_std is not None else self.image_std - do_convert_rgb = do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb - - if images is not None: - images = make_batched_images(images) - if videos is not None: - videos = make_batched_videos(videos) - - if images is not None and not valid_images(images): - raise ValueError("Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, " "paddle.Tensor.") - - if images is not None: - pixel_values, vision_grid_thws = [], [] - for img_idx, image in enumerate(images): - if predetermined_grid_thw is not None: - predetermined_grid_thw_one = [predetermined_grid_thw[img_idx]] - else: - predetermined_grid_thw_one = None - patches, image_grid_thw = self._preprocess( - image, - do_resize=do_resize, - resample=resample, - do_rescale=do_rescale, - rescale_factor=rescale_factor, - do_normalize=do_normalize, - image_mean=image_mean, - image_std=image_std, - data_format=data_format, - do_convert_rgb=do_convert_rgb, - input_data_format=input_data_format, - predetermined_grid_thw=predetermined_grid_thw_one, - ) - pixel_values.extend(patches) - vision_grid_thws.append(image_grid_thw) - pixel_values = np.array(pixel_values) - vision_grid_thws = np.array(vision_grid_thws) - data = { - "pixel_values": pixel_values, - "image_grid_thw": vision_grid_thws, - } - - if videos is not None: - pixel_values, vision_grid_thws = [], [] - for images in videos: - patches, video_grid_thw = self._preprocess( - images, - do_resize=do_resize, - resample=resample, - do_rescale=do_rescale, - rescale_factor=rescale_factor, - do_normalize=do_normalize, - image_mean=image_mean, - image_std=image_std, - data_format=data_format, - do_convert_rgb=do_convert_rgb, - input_data_format=input_data_format, - predetermined_grid_thw=predetermined_grid_thw, - ) - pixel_values.extend(patches) - vision_grid_thws.append(video_grid_thw) - pixel_values = np.array(pixel_values) - vision_grid_thws = np.array(vision_grid_thws) - - data = { - "pixel_values_videos": pixel_values, - "video_grid_thw": vision_grid_thws, - } - - return BatchFeature(data=data, tensor_type=return_tensors) - - -def round_by_factor(number: int, factor: int) -> int: - """Returns the closest integer to 'number' that is divisible by 'factor'.""" - return round(number / factor) * factor - - -def ceil_by_factor(number: int, factor: int) -> int: - """Returns the smallest integer greater than or equal to 'number' that is divisible by 'factor'.""" - return math.ceil(number / factor) * factor - - -def floor_by_factor(number: int, factor: int) -> int: - """Returns the largest integer less than or equal to 'number' that is divisible by 'factor'.""" - return math.floor(number / factor) * factor - - -def smart_resize( - height: int, - width: int, - factor: int = IMAGE_FACTOR, - min_pixels: int = MIN_PIXELS, - max_pixels: int = MAX_PIXELS, -): - """ - Rescales the image so that the following conditions are met: - - 1. Both dimensions (height and width) are divisible by 'factor'. - - 2. The total number of pixels is within the range ['min_pixels', 'max_pixels']. - - 3. The aspect ratio of the image is maintained as closely as possible. - """ - if max(height, width) / min(height, width) > MAX_RATIO: - if height > width: - new_width = max(factor, round_by_factor(width, factor)) - new_height = floor_by_factor(new_width * MAX_RATIO, factor) - else: - new_height = max(factor, round_by_factor(height, factor)) - new_width = floor_by_factor(new_height * MAX_RATIO, factor) - - data_processor_logger.info( - f"absolute aspect ratio must be smaller than {MAX_RATIO}, got {max(height, width) / min(height, width)},\ - resize to {max(new_height, new_width) / min(new_height, new_width)}" - ) - - height = new_height - width = new_width - - h_bar = max(factor, round_by_factor(height, factor)) - w_bar = max(factor, round_by_factor(width, factor)) - if h_bar * w_bar > max_pixels: - beta = math.sqrt((height * width) / max_pixels) - h_bar = floor_by_factor(height / beta, factor) - w_bar = floor_by_factor(width / beta, factor) - elif h_bar * w_bar < min_pixels: - beta = math.sqrt(min_pixels / (height * width)) - h_bar = ceil_by_factor(height * beta, factor) - w_bar = ceil_by_factor(width * beta, factor) - - if min_pixels > h_bar * w_bar or h_bar * w_bar > max_pixels: - raise ValueError(f"encounter invalid h_bar: {h_bar}, w_bar: {w_bar}") - - return h_bar, w_bar diff --git a/fastdeploy/input/v1/ernie4_5_vl_processor/process.py b/fastdeploy/input/v1/ernie4_5_vl_processor/process.py deleted file mode 100644 index d8b90e54d57..00000000000 --- a/fastdeploy/input/v1/ernie4_5_vl_processor/process.py +++ /dev/null @@ -1,751 +0,0 @@ -""" -# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -""" process.py """ -import copy -import os -import pickle -from collections import defaultdict -from typing import Any, Dict, List, Optional, Tuple, Union - -import numpy as np -import paddle -import zmq -from paddleformers.transformers.image_utils import ChannelDimension -from PIL import Image - -from fastdeploy.engine.request import ImagePosition, Request -from fastdeploy.entrypoints.chat_utils import parse_chat_messages -from fastdeploy.input.ernie4_5_tokenizer import Ernie4_5Tokenizer -from fastdeploy.input.mm_data_processor import MMBaseDataProcessor -from fastdeploy.input.utils import IDS_TYPE_FLAG -from fastdeploy.multimodal.hasher import MultimodalHasher -from fastdeploy.utils import data_processor_logger - -from .image_preprocessor.image_preprocessor_adaptive import AdaptiveImageProcessor -from .process_video import read_frames_decord, read_video_decord -from .utils.render_timestamp import render_frame_timestamp - - -def fancy_print(input_ids, tokenizer, image_patch_id=None): - """ - input_ids: input_ids - tokenizer: the tokenizer of models - """ - i = 0 - res = "" - text_ids = [] - real_image_token_len = 0 - while i < len(input_ids): - if input_ids[i] == image_patch_id: - if len(text_ids) > 0: - res += tokenizer.decode(text_ids) - text_ids = [] - - real_image_token_len += 1 - else: - if real_image_token_len != 0: - res += f"<|IMAGE@{real_image_token_len}|>" - real_image_token_len = 0 - - text_ids.append(input_ids[i]) - - i += 1 - if len(text_ids) > 0: - - res += tokenizer.decode(text_ids) - text_ids = [] - return res - - -class DataProcessor(MMBaseDataProcessor): - """ - Processes multimodal chat messages into model-ready inputs, - handling text, images, and videos with 3D positional embeddings. - """ - - CLS_TOKEN = "<|begin_of_sentence|>" - SEP_TOKEN = "<|end_of_sentence|>" - EOS_TOKEN = "" - IMG_START = "<|IMAGE_START|>" - IMG_END = "<|IMAGE_END|>" - VID_START = "<|VIDEO_START|>" - VID_END = "<|VIDEO_END|>" - - def __init__( - self, - tokenizer_name: str, - image_preprocessor_name: str, - enable_processor_cache: bool = False, - spatial_conv_size: int = 2, - temporal_conv_size: int = 2, - image_min_pixels: int = 4 * 28 * 28, - image_max_pixels: int = 6177 * 28 * 28, - video_min_pixels: int = 299 * 28 * 28, - video_max_pixels: int = 1196 * 28 * 28, - video_target_frames: int = -1, - video_frames_sample: str = "leading", - video_max_frames: int = 180, - video_min_frames: int = 16, - video_fps: int = 2, - **kwargs, - ) -> None: - super().__init__() - # Tokenizer and image preprocessor - self.model_name_or_path = tokenizer_name - self._load_tokenizer() - self.tokenizer.ignored_index = -100 - self.image_preprocessor = AdaptiveImageProcessor.from_pretrained(image_preprocessor_name) - self.enable_processor_cache = enable_processor_cache - - # Convolution sizes for patch aggregation - self.spatial_conv_size = spatial_conv_size - self.temporal_conv_size = temporal_conv_size - - # Pixel constraints - self.image_min_pixels = image_min_pixels - self.image_max_pixels = image_max_pixels - self.video_min_pixels = video_min_pixels - self.video_max_pixels = video_max_pixels - - # Video sampling parameters - self.target_frames = video_target_frames - self.frames_sample = video_frames_sample - self.max_frames = video_max_frames - self.min_frames = video_min_frames - self.fps = video_fps - - # Special tokens and IDs - self.cls_token = self.CLS_TOKEN - self.sep_token = self.SEP_TOKEN - self.eos_token = self.EOS_TOKEN - self.image_start = self.IMG_START - self.image_end = self.IMG_END - self.video_start = self.VID_START - self.video_end = self.VID_END - self.image_patch_id = self.tokenizer.convert_tokens_to_ids("<|IMAGE_PLACEHOLDER|>") - self.image_start_id = self.tokenizer.convert_tokens_to_ids(self.image_start) - self.image_end_id = self.tokenizer.convert_tokens_to_ids(self.image_end) - self.video_start_id = self.tokenizer.convert_tokens_to_ids(self.video_start) - self.video_end_id = self.tokenizer.convert_tokens_to_ids(self.video_end) - self.sep_token_id = self.tokenizer.convert_tokens_to_ids(self.sep_token) - self.eos_token_id = self.tokenizer.convert_tokens_to_ids(self.eos_token) - - self.token_type_mapping = self._build_token_type_mapping() - self.is_training = True - self.role_prefixes = { - "system": "", - "user": "User: ", - "bot": "Assistant: ", - "assistant": "Assistant: ", - "tool": "Tool: ", - } - - @staticmethod - def mm_num_tokens(grid_thw: list | list[list[int]] | np.ndarray | paddle.Tensor) -> int | list[int]: - """ - Calculate the number of tokens in the multimodal input. - """ - if isinstance(grid_thw, paddle.Tensor): - grid_thw = grid_thw.numpy() - - if len(grid_thw) == 0: - return 0 - - def calc_one(thw): - t, h, w = map(int, thw) - if t == 1: - return t * h * w // 4 - else: - return t * h * w // 4 // 2 - - if isinstance(grid_thw[0], (list, tuple, np.ndarray)): - return [calc_one(x) for x in grid_thw] - - return calc_one(grid_thw) - - def _build_token_type_mapping(self) -> Dict[Any, int]: - mapping = defaultdict(lambda: IDS_TYPE_FLAG["text"]) - for token in ( - self.IMG_START, - self.IMG_END, - self.VID_START, - self.VID_END, - ): - mapping[token] = IDS_TYPE_FLAG["image"] - mapping[self.image_patch_id] = IDS_TYPE_FLAG["image"] - return mapping - - def train(self) -> None: - """Enable training mode (produces labels).""" - self.is_training = True - - def eval(self) -> None: - """Enable evaluation mode (doesn't produce labels).""" - self.is_training = False - - def text2ids(self, text, images=None, videos=None, image_uuid=None, video_uuid=None): - """ - Convert chat text into model inputs. - - Args: - text (str): The chat text containing placeholders for images and videos. - images (list, optional): List of images to be processed and inserted at image placeholders. - videos (list, optional): List of videos to be processed and inserted at video placeholders. - image_uuid (list, optional): List of unique identifiers for each image, used for caching or hashing. - video_uuid (list, optional): List of unique identifiers for each video, used for caching or hashing. - Returns: - dict: A dictionary with keys input_ids, token_type_ids, position_ids, images, grid_thw, image_type_ids, labels, etc. - """ - - outputs = { - "input_ids": [], - "token_type_ids": [], - "position_ids": [], - "images": [], - "grid_thw": [], - "image_type_ids": [], - "labels": [], - "cur_position": 0, - "video_cnt": 0, - "num_input_image_tokens": 0, - "num_input_video_tokens": 0, - "mm_positions": [], - "mm_hashes": [], - } - - IMAGE_PLACEHOLDER = "<|image@placeholder|>" - VIDEO_PLACEHOLDER = "<|video@placeholder|>" - IMAGE_PLACEHOLDER_LEN = len(IMAGE_PLACEHOLDER) - VIDEO_PLACEHOLDER_LEN = len(VIDEO_PLACEHOLDER) - st, image_idx, video_idx = 0, 0, 0 - while st < len(text): - image_pos = text.find(IMAGE_PLACEHOLDER, st) - image_pos = len(text) if image_pos == -1 else image_pos - video_pos = text.find(VIDEO_PLACEHOLDER, st) - video_pos = len(text) if video_pos == -1 else video_pos - ed = min(image_pos, video_pos) - - self._add_text(text[st:ed], outputs) - if ed == len(text): - break - - if ed == image_pos: - image = images[image_idx] - uuid = image_uuid[image_idx] if image_uuid else None - if not isinstance(image, tuple): - self._add_image(image, outputs, uuid) - else: - # cached images are already processed - self._add_processed_image(image, outputs, uuid) - image_idx += 1 - st = ed + IMAGE_PLACEHOLDER_LEN - else: - item = videos[video_idx] - uuid = video_uuid[video_idx] if video_uuid else None - if not isinstance(item, tuple): - if isinstance(item, dict): - frames = self._load_and_process_video(item["video"], item) - else: - frames = self._load_and_process_video(item, {}) - self._add_video(frames, outputs, uuid) - else: - # cached frames are already processed - self._add_processed_video(item, outputs, uuid) - video_idx += 1 - st = ed + VIDEO_PLACEHOLDER_LEN - - return outputs - - def extract_mm_items(self, request: Request): - messages = parse_chat_messages(request.messages) - mm_items = [] - for msg in messages: - role = msg.get("role") - assert role in self.role_prefixes, f"Unsupported role: {role}" - content = msg.get("content") - if not isinstance(content, list): - content = [content] - for item in content: - if item.get("type") in ["image", "video"]: - mm_items.append(item) - - missing_hashes, missing_idx = [], [] - for idx, item in enumerate(mm_items): - if not item.get("data"): - # raw data not provided, should be retrieved from processor cache - missing_hashes.append(item.get("uuid")) - missing_idx.append(idx) - - if len(missing_hashes) > 0 and not self.enable_processor_cache: - raise ValueError("Missing items cannot be retrieved without processor cache.") - - dealer = None - if self.enable_processor_cache: - context = zmq.Context() - dealer = context.socket(zmq.DEALER) - dealer.connect("ipc:///dev/shm/processor_cache.ipc") - - missing_items = self.get_processor_cache(dealer, missing_hashes) - for idx in range(len(missing_items)): - if not missing_items[idx]: - raise ValueError(f"Missing item {idx} not found in processor cache") - mm_items[missing_idx[idx]]["data"] = missing_items[idx] - - images, videos = [], [] - image_uuid, video_uuid = [], [] - for item in mm_items: - if item.get("type") == "image": - images.append(item["data"]) - image_uuid.append(item["uuid"]) - elif item.get("type") == "video": - videos.append(item["data"]) - video_uuid.append(item["uuid"]) - else: - raise ValueError(f"Unsupported multimodal type: {item.get('type')}") - return images, videos, image_uuid, video_uuid, dealer, missing_idx, mm_items - - def request2ids( - self, request: Request, tgts: List[str] = None - ) -> Dict[str, Union[np.ndarray, List[np.ndarray], None]]: - """ - Convert chat messages into model inputs. - Returns a dict with input_ids, token_type_ids, position_ids, images, grid_thw, image_type_ids, labels. - """ - images, videos, image_uuid, video_uuid, dealer, missing_idx, mm_items = self.extract_mm_items(request) - - if self.tokenizer.chat_template is None: - raise ValueError("This model does not support chat template.") - - chat_template_kwargs = request.chat_template_kwargs if request.chat_template_kwargs else {} - message_dict = { - key: getattr(request, key, None) - for key in ["messages", "tools", "documents", "enable_thinking", "system"] - if getattr(request, key, None) is not None - } - prompt = self.tokenizer.apply_chat_template( - message_dict, - tokenize=False, - add_generation_prompt=request.add_generation_prompt if request.add_generation_prompt is not None else True, - **chat_template_kwargs, - ) - request.prompt_tokens = prompt - - outputs = self.text2ids(prompt, images, videos, image_uuid, video_uuid) - - if self.enable_processor_cache: - missing_idx = set(missing_idx) - hashes_to_cache, items_to_cache = [], [] - for idx in range(len(mm_items)): - if idx in missing_idx: - continue - meta = {} - t, h, w = outputs["grid_thw"][idx][0] - meta["thw"] = (t, h, w) - hashes_to_cache.append(outputs["mm_hashes"][idx]) - items_to_cache.append((outputs["images"][idx], meta)) - self.update_processor_cache(dealer, hashes_to_cache, items_to_cache) - - if self.is_training: - assert tgts, "Training must give tgt" - self._extract_labels(outputs, tgts) - - return outputs - - def prompt_token_ids2outputs( - self, request: Request, tgts: List[str] = None - ) -> Dict[str, Union[np.ndarray, List[np.ndarray], None]]: - outputs = { - "input_ids": [], - "token_type_ids": [], - "position_ids": [], - "images": [], - "grid_thw": [], - "image_type_ids": [], - "labels": [], - "cur_position": 0, - "video_cnt": 0, - "num_input_image_tokens": 0, - "num_input_video_tokens": 0, - "mm_positions": [], - "mm_hashes": [], - } - prompt_token_ids = request.prompt_token_ids if request.prompt_token_ids else [] - prompt_token_ids_len = len(prompt_token_ids) - if not request.messages: - outputs["input_ids"].extend(prompt_token_ids) - outputs["token_type_ids"].extend([IDS_TYPE_FLAG["text"]] * prompt_token_ids_len) - for i in range(prompt_token_ids_len): - outputs["position_ids"].append([i] * 3) - outputs["cur_position"] += prompt_token_ids_len - return outputs - images, videos, image_uuid, video_uuid, dealer, missing_idx, mm_items = self.extract_mm_items(request) - st, image_idx, video_idx = 0, 0, 0 - while st < prompt_token_ids_len: - cur_token_id = prompt_token_ids[st] - if cur_token_id == self.image_start_id: - if image_idx >= len(images): - raise ValueError("prompt token ids has more image placeholder than in messages") - # append image_start_id - outputs["input_ids"].extend([cur_token_id]) - outputs["token_type_ids"].extend([IDS_TYPE_FLAG["text"]]) - outputs["position_ids"].append([outputs["cur_position"]] * 3) - outputs["cur_position"] += 1 - st += 1 - # process placeholder token ids - cur_idx = st - while cur_idx < prompt_token_ids_len and prompt_token_ids[cur_idx] != self.image_end_id: - cur_idx += 1 - if cur_idx >= prompt_token_ids_len: - raise ValueError("image token ids not complete") - image = images[image_idx] - uuid = image_uuid[image_idx] if image_uuid else None - token_len = cur_idx - st - if not isinstance(image, tuple): - self._add_image(image, outputs, uuid, token_len) - else: - self._add_processed_image(image, outputs, uuid, token_len) - image_idx += 1 - st = cur_idx - elif cur_token_id == self.video_start_id: - if video_idx >= len(videos): - raise ValueError("prompt token ids has more video placeholder than in messages") - # append video_start_id - outputs["input_ids"].extend([cur_token_id]) - outputs["token_type_ids"].extend([IDS_TYPE_FLAG["text"]]) - outputs["position_ids"].append([outputs["cur_position"]] * 3) - outputs["cur_position"] += 1 - st += 1 - # process placeholder token ids - cur_idx = st - while cur_idx < prompt_token_ids_len and prompt_token_ids[cur_idx] != self.video_end_id: - cur_idx += 1 - if cur_idx >= prompt_token_ids_len: - raise ValueError("video token ids not complete") - video = videos[video_idx] - uuid = video_uuid[video_idx] if video_uuid else None - token_len = cur_idx - st - if not isinstance(video, tuple): - if isinstance(video, dict): - frames = self._load_and_process_video(video["video"], video) - else: - frames = self._load_and_process_video(video, {}) - self._add_video(frames, outputs, uuid, token_len) - else: - self._add_processed_video(video, outputs, uuid, token_len) - video_idx += 1 - st = cur_idx - else: - outputs["input_ids"].extend([cur_token_id]) - outputs["token_type_ids"].extend([IDS_TYPE_FLAG["text"]]) - outputs["position_ids"].append([outputs["cur_position"]] * 3) - outputs["cur_position"] += 1 - st += 1 - if image_idx != len(images): - raise ValueError("number of images does not match") - if video_idx != len(videos): - raise ValueError("number of videos does not match") - - if self.enable_processor_cache: - missing_idx = set(missing_idx) - hashes_to_cache, items_to_cache = [], [] - for idx in range(len(mm_items)): - if idx in missing_idx: - continue - meta = {} - t, h, w = outputs["grid_thw"][idx][0] - meta["thw"] = (t, h, w) - hashes_to_cache.append(outputs["mm_hashes"][idx]) - items_to_cache.append((outputs["images"][idx], meta)) - self.update_processor_cache(dealer, hashes_to_cache, items_to_cache) - - return outputs - - def _add_special_token(self, token: Union[str, int], outputs: Dict) -> None: - token_id = token if isinstance(token, int) else self.tokenizer.convert_tokens_to_ids(token) - outputs["input_ids"].append(token_id) - outputs["token_type_ids"].append(self.token_type_mapping[token]) - pos = outputs["cur_position"] - outputs["position_ids"].append([pos] * 3) - outputs["cur_position"] += 1 - - def _add_text(self, tokens, outputs: Dict) -> None: - if isinstance(tokens, str): - tokens = self.tokenizer.encode(tokens, add_special_tokens=False)["input_ids"] - outputs["input_ids"].extend(tokens) - outputs["token_type_ids"].extend([IDS_TYPE_FLAG["text"]] * len(tokens)) - - start = outputs["cur_position"] - for i in range(len(tokens)): - outputs["position_ids"].append([start + i] * 3) - outputs["cur_position"] += len(tokens) - - def _add_image(self, img, outputs: Dict, uuid: Optional[str], token_len=None) -> None: - patches_h, patches_w = self.image_preprocessor.get_smarted_resize( - img.height, - img.width, - min_pixels=self.image_min_pixels, - max_pixels=self.image_max_pixels, - )[1] - num_tokens = (patches_h * patches_w) // (self.spatial_conv_size**2) - if token_len and token_len != num_tokens: - raise ValueError("image tokens num not match the size") - - outputs["mm_positions"].append(ImagePosition(len(outputs["input_ids"]), num_tokens)) - outputs["input_ids"].extend([self.image_patch_id] * num_tokens) - outputs["token_type_ids"].extend([IDS_TYPE_FLAG["image"]] * num_tokens) - outputs["num_input_image_tokens"] += num_tokens - - pos_ids = self._compute_3d_positions(1, patches_h, patches_w, outputs["cur_position"]) - outputs["position_ids"].extend(pos_ids) - outputs["cur_position"] = np.max(pos_ids) + 1 - - # Preprocess pixels - ret = self.image_preprocessor.preprocess( - images=[img.convert("RGB")], - do_normalize=False, - do_rescale=False, - predetermined_grid_thw=np.array([[patches_h, patches_w]]), - do_convert_rgb=True, - input_data_format=ChannelDimension.LAST, - ) - outputs["images"].append(ret["pixel_values"]) - if not uuid: - outputs["mm_hashes"].append(MultimodalHasher.hash_features(ret["pixel_values"])) - else: - outputs["mm_hashes"].append(uuid) - outputs["grid_thw"].append(ret["image_grid_thw"]) - outputs["image_type_ids"].append(0) - - def _add_processed_image( - self, img_cache: Tuple[np.ndarray, dict], outputs: Dict, uuid: str, token_len=None - ) -> None: - img, meta = img_cache - num_tokens = img.shape[0] // (self.spatial_conv_size**2) - if token_len and num_tokens != token_len: - raise ValueError("image tokens num not match the size") - - outputs["mm_positions"].append(ImagePosition(len(outputs["input_ids"]), num_tokens)) - outputs["input_ids"].extend([self.image_patch_id] * num_tokens) - outputs["token_type_ids"].extend([IDS_TYPE_FLAG["image"]] * num_tokens) - - _, h, w = meta["thw"] - pos_ids = self._compute_3d_positions(1, h, w, outputs["cur_position"]) - outputs["position_ids"].extend(pos_ids) - outputs["cur_position"] = np.max(pos_ids) + 1 - - outputs["images"].append(img) - outputs["mm_hashes"].append(uuid) - outputs["grid_thw"].append(np.array([[1, h, w]])) - outputs["image_type_ids"].append(0) - - def _add_video(self, frames, outputs: Dict, uuid: Optional[str], token_len=None) -> None: - patches_h, patches_w = self.image_preprocessor.get_smarted_resize( - frames[0].height, - frames[0].width, - min_pixels=self.video_min_pixels, - max_pixels=self.video_max_pixels, - )[1] - num_frames = len(frames) - num_tokens = (num_frames * patches_h * patches_w) // (self.spatial_conv_size**2 * self.temporal_conv_size) - if token_len and num_tokens != token_len: - raise ValueError("video tokens num not match the size") - - pixel_stack = np.stack([np.array(f.convert("RGB")) for f in frames], axis=0) - ret = self.image_preprocessor.preprocess( - images=None, - videos=pixel_stack, - do_normalize=False, - do_rescale=False, - predetermined_grid_thw=np.array([[patches_h, patches_w]] * num_frames), - do_convert_rgb=True, - input_data_format=ChannelDimension.LAST, - ) - outputs["images"].append(ret["pixel_values_videos"]) - if not uuid: - outputs["mm_hashes"].append(MultimodalHasher.hash_features(ret["pixel_values_videos"])) - else: - outputs["mm_hashes"].append(uuid) - outputs["grid_thw"].append(ret["video_grid_thw"]) - outputs["image_type_ids"].extend([1] * num_frames) - - outputs["mm_positions"].append(ImagePosition(len(outputs["input_ids"]), num_tokens)) - outputs["input_ids"].extend([self.image_patch_id] * num_tokens) - outputs["token_type_ids"].extend([IDS_TYPE_FLAG["video"]] * num_tokens) - outputs["num_input_video_tokens"] += num_tokens - - pos_ids = self._compute_3d_positions(num_frames, patches_h, patches_w, outputs["cur_position"]) - outputs["position_ids"].extend(pos_ids) - outputs["cur_position"] = np.max(pos_ids) + 1 - - def _add_processed_video( - self, frames_cache: Tuple[np.ndarray, dict], outputs: Dict, uuid: str, token_len=None - ) -> None: - frames, meta = frames_cache - num_tokens = frames.shape[0] // (self.spatial_conv_size**2 * self.temporal_conv_size) - if token_len and num_tokens != token_len: - raise ValueError("video tokens num not match the size") - - t, h, w = meta["thw"] - outputs["images"].append(frames) - outputs["mm_hashes"].append(uuid) - outputs["grid_thw"].append(np.array([[t, h, w]])) - - outputs["mm_positions"].append(ImagePosition(len(outputs["input_ids"]), num_tokens)) - outputs["input_ids"].extend([self.image_patch_id] * num_tokens) - outputs["token_type_ids"].extend([IDS_TYPE_FLAG["video"]] * num_tokens) - outputs["image_type_ids"].extend([1] * t) - - pos_ids = self._compute_3d_positions(t, h, w, outputs["cur_position"]) - outputs["position_ids"].extend(pos_ids) - outputs["cur_position"] = np.max(pos_ids) + 1 - - def _extract_labels(self, outputs: Dict, tgts: List[str]) -> None: - input_ids = copy.deepcopy(outputs["input_ids"]) - labels = [self.tokenizer.ignored_index] * len(input_ids) - - tgt_count = input_ids.count(self.sep_token_id) - assert tgt_count == len(tgts), f"len(tgts) != len(src) {len(tgts)} vs {tgt_count}" - - tgt_index = 0 - for i, token_id in enumerate(input_ids): - if token_id == self.sep_token_id: - labels_token = self.tokenizer.tokenize(tgts[tgt_index]) - labels_token_id = self.tokenizer.convert_tokens_to_ids(labels_token) - labels[i - len(labels_token_id) : i] = labels_token_id - labels[i] = self.eos_token_id # - tgt_index += 1 - - outputs["labels"] = labels - - def _load_and_process_video(self, url: str, item: Dict) -> List[Image.Image]: - reader, meta, path = read_video_decord(url, save_to_disk=False) - - video_frame_args = dict() - video_frame_args["fps"] = item.get("fps", self.fps) - video_frame_args["min_frames"] = item.get("min_frames", self.min_frames) - video_frame_args["max_frames"] = item.get("max_frames", self.max_frames) - video_frame_args["target_frames"] = item.get("target_frames", self.target_frames) - video_frame_args["frames_sample"] = item.get("frames_sample", self.frames_sample) - - video_frame_args = self._set_video_frame_args(video_frame_args, meta) - - frames_data, _, timestamps = read_frames_decord( - path, - reader, - meta, - target_frames=video_frame_args["target_frames"], - target_fps=video_frame_args["fps"], - frames_sample=video_frame_args["frames_sample"], - save_to_disk=False, - ) - - frames: List[Image.Image] = [] - for img_array, ts in zip(frames_data, timestamps): - frames.append(render_frame_timestamp(img_array, ts)) - # Ensure even number of frames for temporal conv - if len(frames) % 2 != 0: - frames.append(copy.deepcopy(frames[-1])) - return frames - - def _set_video_frame_args(self, video_frame_args, video_meta): - """ - 根据已知参数和优先级，设定最终的抽帧参数 - """ - # 优先级：video_target_frames > (video_min_frames, video_max_frames) > video_fps - if video_frame_args["target_frames"] > 0: - if video_frame_args["fps"] >= 0: - raise ValueError("fps must be negative if target_frames is given") - if ( - video_frame_args["min_frames"] > 0 - and video_frame_args["target_frames"] < video_frame_args["min_frames"] - ): - raise ValueError("target_frames must be larger than min_frames") - if ( - video_frame_args["max_frames"] > 0 - and video_frame_args["target_frames"] > video_frame_args["max_frames"] - ): - raise ValueError("target_frames must be smaller than max_frames") - else: - if video_frame_args["fps"] < 0: - raise ValueError("Must provide either positive target_fps or positive target_frames.") - # 先计算在video_fps下抽到的帧数 - frames_to_extract = int(video_meta["duration"] * video_frame_args["fps"]) - # 判断是否在目标区间内，如果不是，则取target_frames为上界或下界 - if ( - video_frame_args["min_frames"] > 0 - and video_frame_args["max_frames"] > 0 - and video_frame_args["min_frames"] > video_frame_args["max_frames"] - ): - raise ValueError("min_frames must be smaller than max_frames") - if video_frame_args["min_frames"] > 0 and frames_to_extract < video_frame_args["min_frames"]: - video_frame_args["target_frames"] = video_frame_args["min_frames"] - video_frame_args["fps"] = -1 - if video_frame_args["max_frames"] > 0 and frames_to_extract > video_frame_args["max_frames"]: - video_frame_args["target_frames"] = video_frame_args["max_frames"] - video_frame_args["fps"] = -1 - - return video_frame_args - - def _compute_3d_positions(self, t: int, h: int, w: int, start_idx: int) -> List[List[int]]: - # Downsample time if needed - t_eff = t // self.temporal_conv_size if t != 1 else 1 - gh, gw = h // self.spatial_conv_size, w // self.spatial_conv_size - time_idx = np.repeat(np.arange(t_eff), gh * gw) - h_idx = np.tile(np.repeat(np.arange(gh), gw), t_eff) - w_idx = np.tile(np.arange(gw), t_eff * gh) - - coords = list(zip(time_idx, h_idx, w_idx)) - return [[start_idx + ti, start_idx + hi, start_idx + wi] for ti, hi, wi in coords] - - def _load_tokenizer(self): - """ - load tokenizer - - Returns: - tokenizer (AutoTokenizer) - """ - vocab_file_names = [ - "tokenizer.model", - "spm.model", - "ernie_token_100k.model", - ] - for i in range(len(vocab_file_names)): - if os.path.exists(os.path.join(self.model_name_or_path, vocab_file_names[i])): - Ernie4_5Tokenizer.resource_files_names["vocab_file"] = vocab_file_names[i] - break - self.tokenizer = Ernie4_5Tokenizer.from_pretrained(self.model_name_or_path) - - def get_processor_cache(self, socket, mm_hashes: list[str]) -> list: - """ - get cache correspond to given hash values - """ - req = pickle.dumps(mm_hashes) - socket.send_multipart([b"", req]) - _, resp = socket.recv_multipart() - mm_items = pickle.loads(resp) - data_processor_logger.info(f"Get cache of mm_hashes: {mm_hashes}") - - return mm_items - - def update_processor_cache(self, socket, mm_hashes: list[str], mm_items): - """ - update cache data - """ - req = pickle.dumps((mm_hashes, mm_items)) - socket.send_multipart([b"", req]) - data_processor_logger.info(f"Update cache of mm_hashes: {mm_hashes}") diff --git a/fastdeploy/input/v1/ernie4_5_vl_processor/process_video.py b/fastdeploy/input/v1/ernie4_5_vl_processor/process_video.py deleted file mode 100644 index 91120096c70..00000000000 --- a/fastdeploy/input/v1/ernie4_5_vl_processor/process_video.py +++ /dev/null @@ -1,205 +0,0 @@ -""" -# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -import io -import os -import random - -import numpy as np -from PIL import Image - -from fastdeploy.utils import data_processor_logger - -from .utils.io_utils import EXTRACTED_FRAME_DIR, get_filename -from .utils.video_utils import VideoReaderWrapper - - -def read_video_decord(video_path, save_to_disk): - """get reader and meta by decord""" - # video_path = get_downloadable(video_path, save_to_disk=save_to_disk) - if isinstance(video_path, VideoReaderWrapper): - video_reader = video_path - else: - if isinstance(video_path, bytes): - video_path = io.BytesIO(video_path) - video_reader = VideoReaderWrapper(video_path, num_threads=1) - vlen = len(video_reader) - fps = video_reader.get_avg_fps() - duration = vlen / float(fps) - - video_meta = {"fps": fps, "duration": duration, "num_of_frame": vlen} - - return video_reader, video_meta, video_path - - -def get_frame_indices( - vlen, - target_frames=-1, - target_fps=-1, - frames_sample="middle", - fix_start=None, - input_fps=-1, -): - """ - 取出对应的frame index - """ - assert frames_sample in ["rand", "middle", "leading"] - if target_frames > 0: - assert target_fps <= 0, "target_fps must be negative if target_frames is given." - if target_frames > vlen: - acc_samples = vlen - data_processor_logger.info( - f"target_frames={target_frames} is larger than video length {vlen}, " - f"will sample {acc_samples} frames." - ) - else: - acc_samples = target_frames - data_processor_logger.debug(f"sampling at target_frames={target_frames}, frames_sample={frames_sample}") - - # split the video into `acc_samples` intervals, and sample from each interval. - intervals = np.linspace(start=0, stop=vlen, num=acc_samples + 1).astype(int) - ranges = [] - for idx, interv in enumerate(intervals[:-1]): - ranges.append((interv, intervals[idx + 1] - 1)) - if frames_sample == "rand": - try: - frame_indices = [random.choice(range(x[0], x[1])) for x in ranges] - except Exception: - frame_indices = np.random.permutation(vlen)[:acc_samples] - frame_indices.sort() - frame_indices = list(frame_indices) - elif fix_start is not None: - frame_indices = [x[0] + fix_start for x in ranges] - elif frames_sample == "leading": - frame_indices = [x[0] for x in ranges] - elif frames_sample == "middle": - frame_indices = [(x[0] + x[1]) // 2 for x in ranges] - else: - raise NotImplementedError - - elif target_fps > 0: - assert target_frames <= 0, "target_frames must be negative if target_fps is given." - assert input_fps > 0, "input_fps must be provided if target_fps is given." - data_processor_logger.info(f"sampling at fps={target_fps}, frames_sample={frames_sample}") - duration = float(vlen) / input_fps - delta = 1 / target_fps # gap between frames, this is also the clip length each frame represents - if frames_sample == "middle": - frame_seconds = np.arange(0 + delta / 2, duration + delta / 2, delta) - elif frames_sample == "leading": - frame_seconds = np.arange(0, duration, delta) - if frames_sample == "rand": - frame_seconds = np.arange(0 + delta / 2, duration + delta / 2, delta) - rand_offset = np.random.rand(*(frame_seconds.shape)) - 0.5 - frame_seconds += rand_offset * delta - frame_indices = np.around(frame_seconds * input_fps).astype(int) - frame_indices = [e for e in frame_indices if e < vlen] - - else: - raise ValueError("Must provide either positive target_fps or positive target_frames.") - - return frame_indices - - -def read_frames_decord( - video_path, - video_reader, - video_meta, - target_frames=-1, - target_fps=-1, - frames_sample="middle", - fix_start=None, - save_to_disk=False, - cache_dir=EXTRACTED_FRAME_DIR, - frame_indices=None, - tol=10, -): - """get frames by decord""" - - if frame_indices is None: - frame_indices = get_frame_indices( - video_meta["num_of_frame"], - target_frames=target_frames, - target_fps=target_fps, - frames_sample=frames_sample, - fix_start=fix_start, - input_fps=video_meta["fps"], - ) - - frames = [] - for frame_indice_index in range(0, len(frame_indices)): - frame_indice = frame_indices[frame_indice_index] - try: - frames.append(video_reader[frame_indice].asnumpy()) # (T, H, W, C) - except Exception as e: - data_processor_logger.debug(f"encounter error when get frame: {frame_indice}, error: {e}") - previous_counter = 1 - later_counter = 1 - previous_after_flag = True - if frame_indice == 0 or frame_indice == len(video_reader) - 1: - cur_tol = tol * 2 - else: - cur_tol = tol - while previous_counter < cur_tol or later_counter < cur_tol: - if previous_after_flag: - if frame_indice - previous_counter < 0: - previous_counter += 1 - previous_after_flag = not previous_after_flag - continue - try: - frames.append(video_reader[frame_indice - previous_counter].asnumpy()) - data_processor_logger.info( - f"replace {frame_indice}-th frame with {frame_indice-previous_counter}-th frame" - ) - frame_indices[frame_indice_index] = frame_indice - previous_counter - break - except Exception as e: - previous_counter += 1 - data_processor_logger.info(f"error: {e}") - else: - if frame_indice + later_counter >= len(video_reader): - later_counter += 1 - previous_after_flag = not previous_after_flag - continue - try: - frames.append(video_reader[frame_indice + later_counter].asnumpy()) - data_processor_logger.info( - f"replace {frame_indice}-th frame with {frame_indice+later_counter}-th frame" - ) - frame_indices[frame_indice_index] = frame_indice + later_counter - break - except Exception: - later_counter += 1 - previous_after_flag = not previous_after_flag - - frames = np.stack(frames, axis=0) - assert len(frames) == len(frame_indices), f"len(frames): {len(frames)} != len(frame_indices): {len(frame_indices)}" - - ret = [] - - url_sha1 = get_filename() - for idx, frame in enumerate(frames): - tmp = Image.fromarray(frame, "RGB") - if save_to_disk: - save_path = os.path.join(cache_dir, f"{url_sha1}", f"{idx}.png") - if not os.path.exists(os.path.dirname(save_path)): - os.makedirs(os.path.dirname(save_path)) - tmp.save(save_path) - tmp = save_path - ret.append(tmp) - - time_stamps = [frame_idx * video_meta["duration"] / video_meta["num_of_frame"] for frame_idx in frame_indices] - - return ret, frame_indices, time_stamps diff --git a/fastdeploy/input/v1/ernie4_5_vl_processor/utils/Roboto-Regular.ttf b/fastdeploy/input/v1/ernie4_5_vl_processor/utils/Roboto-Regular.ttf deleted file mode 100644 index 7e3bb2f8ce7ae5b69e9f32c1481a06f16ebcfe71..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 146004 zcmb@v2VfL8(?2{ad+uON$H1LAY>MfWC3h031oSW{sOP-O*@UYa+3X zh(|PQ->#F#SC?)Obx$XXINhvM=Z4Rg-K;X~U-`jO6UuosEl$l6d1-Zm@3U3hJ z!)#QOh+V*akGR3H!_EZ9eof?goX9dgZp+e`D`8HtMx;G_fhYUa6b@s6(D@gNDY%Zd=Yu5EUFoLlX&~^;&DO`-s;9~S1orU{8E=C{YV)Q8~+5dvRg!?rv zMwh@ZM%QpLx`~U?ZTbc7U3v!h1ujM=NU1^Mad?6@pus6^3hL#o!i)ObJ$+ zg#xe4s=%$rs==+z>cDNl8pCbIn!|0y+Q99=I=~G_|75H?12m%v@l!2v-6Toq>oxGLTgpThlI zdxi1Bb$B(?I;_U0)}b{_>yYX_tb?n6Y8_PFv<|FZ z);gei0c-y%rnO%s)7rPvr`A4|O>6IprZv8TX^pFpWbIXGogS~~^zuyzbKtsR1s ztnEWgYrBxi*0#k>Ya4s0wY5FT+RAQPTiUx=Tj0KVG1J=2W?GxtMp&B!nbyWZBdm?A zrnO0;Y(^|iPFo<0~F@GXQ5e_O2E{=iUNW^xX!0uL*0QoQT9~MXLRtZX`CrT zLkmEvokNWpY1KH#v@b0f>s8kv50wCYz>nU6Zcm2JJ4NT|bGilIy<|CAE*8v6unMdO z3uB#FPu7>c!zQtXYzf=U9P9ym#!Mc_tMf>HoWIXMdnR9+<_p*&G{`Lq1p z;6|>ZRJ2~v$wfC6eN^-bI6yXUo4+l$Ex*lX3kC-&*=o8tFv_;ow!@YL4ji!^e}x0x zTpY-ijRW!E00RdYIPe@CfFPK^n>W~F^Idk^JOkOd1h@$J67V_TQ^3c7bAb2Fe5QwK zy!`X!UG9bZmM~L$ziiCvz4-3Mw=X_?ap1+S7du~Ud9nUQ0$5v+v=qMZMj3xSfBF0w z(etB#RKQNa=I5K>t_Cay%tfk3fH2@ah;F{PdE(}fo2zf~n=5awx;En4$ZJDk(+#>d z@LK!i>@kxGZeemX1;h=Y_S4#B29Dh`V@Xr!a! zm^dy@h?8_yoQC~z4wlGy^!yL#L;46d-6!H*Wz`8&d_Z4`59w?4-;c#7uu(41Mfyg3 zE=hbr-@>B*4w~(2`d(as&G!TSC@zZ6=(6}md@H^a-@}smLHr0?@QS!Bexf_D|1#)T zafR+d$375O=^;I$$Mi&8qu<2O;yV2 z%tP6Wa-9@Xviz(7E655-Lt3P#^pI;=8CI5+W93-|Wg|g%RZ^BCY$tD471m>QRs&jh zlk}0ka*13j{bUZgS#FVQSZmfsu9B-^U-`>ia+zGty0ES+LKc>RvWP4ytxa^VJrSn9ZN3&JrE(s_DNm~qvGdy=nl!G(KKh_Tm& z`(UoL#)bQ4!~Iw>InEWHgR04fF1%nCZBP)tvYIm-_PVz$%x7I&ruhihYE`gE<7)l z6k}X?J}N3ATzGyeCfqa^Ae+eN3NJ{-`6CxzhzfIeS%DP7UHr$~qbMwNCp{q)0KMeI zOOP-0loKyW-h6-yFGcxyxC<{$#V`+WrU|72ypRholht0VEEU2$!P<%oYx-5ON%rGBXA zztt^^8~^y!>a*{ZlQnsQ9N8*=sczTpmJgAs-{q zHnHIP-zDzvA+~H%_@>w_8r*e$Q>yq3hPfJsTNi} z)ogSaVuw&4txHC^o^@-Ka=PvNk~9C2WTU38bT;Y)tPk=UtNUVa#Pvt6w%2lqN62X8 zue8(%;EEoVwiC6W*6`c8%Fmv&s#lz*Y9!n;NYzC@Jrdzcrs-?0&VZ%D?2s||aG{u1f$xP zYq{44Ubnmpde`xu?S0HAk58men$I`B(l^w1u@A&)sxA1?@Va-9yy6VNR)=e(A@F?o~n`R41CZ*V>{f0_K-@;@zLDPS*fyuhagsbFxy=z>QJepV<) zp|XXV6&hA(R-to+zAW@}p~rzTFg$Q`;Hkm|3U?^Hw@B+Edy3pBTBGRvqQ6)RS<6{t ztaGfptT${GZR>)9gIWcx4oWZ9qS&TlSL~(i?d+@VY4&gIKNY9qfyG-DA5%QF_!q^W z2m1#X4=x)V5j;A0VeppV%fUB;pNE8oObba2`7-2QiDD&2mq;y{tK`IzmrMDS8eQs# z(q&5bDSf8&?a*4GiJ{laR4g;G%x7g@maSHHR@wKo|RTsx==Z|a@)$&EAOs+sY?DTU8>Bj@T5Q>XWOdR=-oDV2uVf`qr3MGk?wcHAmK5RrAAI9<`d*npo?@+9hiDt^IBt|2hNf z9H{f8Zo#@k>YlA9>P6Q(RPSN^()Hh}|8ZEkumufdg9#1(XgH$bca2InifOc}(Tm2> zO{mG%ranzaH9g$)&t@f?HElM$+016^npbH)qxt6MCz}7&qE3s=E#7N!v!$(Nx0XX% zPH8#6<))TDwF+$2rPZQVTU#A!b-Q(e)?Hg~Y~$HxXq%~RmbZDo&C|AiZEbDi+wN%l zW4k)-*0j6czI6M#?c2AX)&6w*uiM{h|5pd!4&6HJ=z zxL^4A@Ee`OI&bWJw@a-qGrN4zHCNY;U5|D>-}S4mPyhLf;1S*tfe~dQnnuJ#OpI6_ zu`l9S#QPDCBLgDqL`Fo$M~;eI5Y;fMPt?q)^-(*b_C=*dor(G=>g%Ws1?H1Cl zeYc6-9Nms|d$-%i-7a*y-0fDk``!MGX3^f!xuXk52S>M#9vOWu`g!;8?lIldy1(qv zzQ>v#cY2oU*}Lban4&TLVm^%Ju`Od`Vu#1hie2A}_bS}0U9Xs4bK?reb&5-kyAq!( zzES+x`04SR;*Z9kkAKm-X7A;_|LoJL&(l6n`#$Jbt>3_Y3;TW2zi9u3{df2OYe3xr zy$8HMkPR$2u;;+J1CI~9J*d>6HiO0wT0ZFE;EIE14o(~Va!Ao3ZHM$9vUGJMg4 z`fGIe(X&S%9sOubnK9kREF5!U%+s-z#?~7#QGCwPyFhw3UAGR>-)FsyglyiPu_m=PQ`cLdT04N&n7jU)OFILNv9_F zn4C5x$CL(BMo!r}<(sLwrgoZ|I`!wNFQ%287C&vyG{>}i(<@Jpo<3{(h3S{4-43-fEw zpFMxk{O1co7K~i5W5Kh9r5A#|3x_W}x=1W)w5Z>rZHo>s`e@PRMGqFs#RV3ZTijrA z_~O2cCoEpDc=O@|i{D@T!{U2O*pfU;N-U|hr0tT}C8L(iTC#S@-X&+2Tv&2z$)8Jo zm)e$ATiRl2^wObArz~B*bm!6|OFvn9b?KvJ#yZq&fTq}yNsJ5cTis%(XS4>&4Y{kwMM^}8Z;>wCgD~*)}Rt{eI z!^(TBXjSf2!K-SlYPqWWs$r|9uUfe(an*@cU#z;m>i5-Ns|&9#zq-NdPOJN@9#psvHh%5cwR6{QT${S~?AmYF-dX!%UCwpI)>U2CVqN!j!`4k*w_;u5x)bZZSa*Hh zAM3r>7hYd}eS`Jk>-(;sxPIaKt?LhO@Yv96!{`mOH>};TXTzBd7dG79@N}c!M%%_p z8=Gv5*f?FH*_&9==|HaFWGwRzCy$(xsM-o80)i`SOITgq)|uqAv;pDp9IBy3r~<^EQ- zHTTw#tu?o{*&4HTs-rII@+qG?vw|i_axIK7#_3bUTM{ggtefsvVx8K_S=MJA8MR!!((P&4P9RqfZ z*^#hg!;X|4=XPA$@$-&5J09(Lx|8qp-Wjm7(9U8zOYf|-v)0arJ6rDTv2(=E89P_) zOxk&3=a)Ob-+684FFPOZ{A-urF59lkyPEEb+%box1zn?oW4LOe~OSODvgKC$U{(*TkNQ zBNHbkE=b&zxIgiH;`fOeNtBc)sYFt(q=rc?le#4JPa2*yJ!xf9a?+Wk3rV+=o+kSx z=S?n>Trs&xa);#Z$-|PTC9g={ot%<7D8h2-1G_xJEUdH0msQ+H33J#F^H>>0Ud z)}DoXw(dE+=i@zB_B`5a?9IP7bZ`B=o%Z(KJ9O{Zy>s_&+`E78`Muxoy}Q@k7qBm6 zU#)#@_r>lTy>HIG#rwAJJF@SSeOLB9c32#R9AzC19N~_Bj){&{jsuPl96vf9q{x&4 zDP>Z^Qo>V)rz}X>l5!~J!;~LW?x(WUys0Hq>!h|zjY}Pqnvl9a^=RrhslTK?-|xS_ z`2HIETkr3=f5iTo`&aMZv;Xw|3;S>He|8|pfzSiZ4n!Rod|>i{r3ZE#IC9{V13w*j zbkK6J(7|#C8ypNj*!SRsgYyq=I=KJf`GemdynE0*l>1PLL$wdJJrsLr)S;P&wjMfk z=)*%l9=d;+AI^KY9HKg ziXE$VtmUyD$3`5Rd2G$GJ;zQTyKpT1*z@E5$BQ4YalG~MnB$|4&pN*5_}=619{=X} zo#QW01e^#tQS(Hr6FpCiJTdFU+7tUuym#W;6W34NKk@9O$I09$ttUfIRz6w#WTTU< zPIfxk?PT1^fhR|voOp8D$+;(&o?Lr!-^q7RUObt8^1;bJPx+q8bE?Ftnx`6^YH_N= zsmN1KHcW@>eHJ~?>T+?^jD`ZoxXMYuQR@93Y-Z& zQ}0a2Gd<1>KQsNz$}@>)4xM@T%r{{zSFc*RV%E$V)2B_HGI`QFZ@)Eh!uWAx$BZ5| za>VdqLx&6=G;l!wetrA&j*sgV8`HB#_vmg>kr7?HbPn&-p?$lyZCbZ#*`j%~rX{WZ z-d<%`ijQ|Ad!ydoWy?^Cw+~RCvSpaV(#YYV10C%`ZFq$k5!9+=? zLk2fd5F4Km=Z=U(7I+HJB8yg??5#RPMcA4o#OS9G68_4aGroq(QOCJ32X7P};bH~&(0SYZwdfY^XuHip?GqB>Q%D3Oo3OwXrV*pj+(^_U((dRLY7eqU^hQ=G zSY8bZk7)$V*Nrh-Gn8OU^QT^bxUTjz=E9;P9JZJ~kkXSZrWIkUhv28JT9DDXdc4gOC@ehADP7yF*GCo82iY z0);wkv%@1&Ipd9D@M5+&;v&*)&O}GT@SZ3xjXT5qb-I)gRS;r0Vx%)BOvfNhBFxhnHrbiFq${;2V*FLi5p1|1 zunsgk|G*_QWSF-R<{9P{=8GK|XbTmTijboW+m{&a_ho)8Fa>#{1_(}LlT*CH0zoZu zat0m6>n1Oz!X{^ifpV2BTSh3-S@F(UHRv1_vEP@Fzjl$Lq53OZrb!BK8*0xI>kbiU zz$PiIZD6MoE_~!rnX39+{FSPr|S!$i|)$ zYYB-7O^9$PAycAd_pb#(6~Qgy>`h}-3Un0ww?I%_7h8;@S4=2Sn}5>;MBCzGRcgwY z!bNZjlfkSW)dQz2z7B7D?}iQ^d&A6_`c&T;V^J|4_J$6Y&&lT|_9nIh{S)Hsy&!X8 z?IZdI_KA!|-i|PPtV7xx2Bt`Ai2hW7p%P6}sBI`}(+X18F0_3&bW2rzTS7u(TS}M= ziH(a@_l<+l(-T}#_Qs7>H+4VJ#FpR)i;asxiYAddaoI8;A)$#q))o(Sf$D-+o$PA! zL6j;bJSrl=H{Kp^2T#Mo5@J#9KwDg7U_xY^<}t_tX;ik1k)@AZ>WHge9~{>Q4mv`w z7<(^gfa;I0hxL6upbt`I53{#a#evwCifZ@?E$vO>kwQUiyhA|lf^6~F5ut>H+G|bm zkBN{)W+EHfRA-*xUt39vx;&p_g*v5+IF^?&xhs zaIS=a1Y6An=&9~du`(pCOAH1A>_BL0(@h-)nin-tv&d{GH98aupG9wiu^}~wW=$hIc6Y7cl z{{B>g=YLliL_xQv!%GvQv{8tsBJ7`w!X^mSKrK}P@VK)YEmc(vFdBV2At5dnHdXf= zs)zc9Qoa|tLds6c=HzO-CmUr8c<<<8y=ym?*k#w!sCMofAH~)Q?z{$>5qz^ zfkJc9oo1Sffv(g})`7psPs_P0T6YYUnR;$)Xe6+v3Sy9|sRCC|@^ST2-`BLhD|=^~ zdc7KH&&*5>^!Cgg)m;knh50K34U`yS^9Q?X>81-oO>m!3D}{N4xDr4j{ZY+Y2?;)S z=a6Do>i-B)7-@5oBB7mM4RE}TrcX%l^m}uR=j-YH^aHB%qm}(KeU*!=J9sy8_%u@X z3mVYjrSwoawC~&Jl(<1Ev^kM&IMhL?maDTm2gw48_h2+00mp}GWu_9k`PN5wX=iq? zhlWSIjY1Ts&Vd>JSfxJ)=?PL0s92|E&VimWG&Is_a=oo&W2){_Q$uaG{;;zeF<4s| zD1qCc4UpP1L@T)j7(D%BF*HLDDZLX}0Jc;|We>yVxBJ_$`?-#DB4l?;TqleHGB~16 zV9iL(B+|@Z3M+jD)^H5+fbayH%|92S5^MpOS~zB?t|(n`b{&RM+!EqSrD`xEG$Fy6 zPFdl;JfT%5FhR}HylVt{gQms9TzuR9H=w zvdW#3S?*Yrt0+it7o0|Y-&m{?Io3_N`d?QyjW8P9orK5`SJ?^C2~n6J2BlF^RlJKH zb)TcKQsbJSrHUXTF%NIoTz^dMu>){_+KfGKbL6iyR!*WJ#*frrKBYda5B1|GXeR%V zN{A6uSSCwSYS^n%82iWCP;R6j2}lC;0Q3Y@1*`$g0JH@R2gI`nG=rs3DVd9! z%XnHP%hN3E>uTjuo(3B=sEB+{yN#+e&IqI3avoq1?nlxD;~?$kW$CaSKvfL`;q7R* zMWsc!F@?q&zS4Mmgoj)RkLlsc1o^;Co}A zPMBCnl8>jFGKvPvcW9ohK;7ju)S(`A=O0jAgvH8!G@m`91?-M_Qs&V3IuA*6s%7U;HI#m2grLlJ{AN)vukiys#a=_1H45w+p=SsY@ zl{={$>eF9Vp`0v(mdSMLCL*b~r3FQcWuW;C^_D|ulBFfWlV~CK?N5`;>kEpT71oNS6LXnDp@+j?; z@6tTWyVOpm&^vMoHA7y}avd#~A5c$YE%paJqjho=tp)GxyfoR3xwHjomUwKY{vM4f z7UgX=W>F7*)Vyv?qCRpdMH`F2&&Sw{D9r=12E~c9lqeR^0!6plG(m0wG%_z)no}zaQK-?ICVDQRN}k)Xr*0b6GY*i$ z@S&|1NtHbsQ6S{8y`sxVqzZ;7wXu|?aN`o~GbYeVk6BoDI!z8s0OhjWMH+*)BEMxG zCFn3@H%bnsUf|;f5e>dypmLTV$|wCP0$*ILmz@BcsEl!+Hb{I~q{77aRMPki^*f3C zDC%b!PwOokQJ+7ksHGaxMF84Tyk$20c<`g2c^mhAE$G{p8+c}p`9kI6;YI5#WoU}* zYX0C+hSH3`(3iial90EY9=`OE#g|T4zM`eZ&$QCZpu-+tLNQ)926; z2WW|BMezFxEdk$#gQr8l=UTEfmG&r0RgCUrHICCtxq|u`!{863>G0=RDo}lS(!8wm zj-s{3OR6C6gZKMUCgcU}w_9eQ45gRq(FWBIy^}+th;zkhUx;L$~aFqR+ z%7`EuCiCEnMZBwb`y93p#~yFB;~Dyvcm?2F07}d$|qI{aNK4xQT+I<70^Lwda32fjy*fKzk8O6A-7KnFw&|u>awC34LklLRQOS+UYq4 zZF?7ZDC7g<5Bir?+9?$JD#&O~-Hmo=*L3K?�egE!tu*v~K}gWeK5h%eT<6pQ9}X zpua7m0meeK63uq{FLRa~t&-p_h{Go@P zuo`|G>e5_>QVto8ewr8Y{jj(7Z5l00KwsIYG2|yf9tZv1F_s*s79QM)pJ#-B6e@*z)tPCI#2a1@)mQbp8OooJ;0vcz7%WZ2aS(u z494OT@-2$>Xh;o)Cc~{|Ir+36f+64KU2K}59e6I(+(_5B9zu8XH*%IjSZKxlMb_ttZ@K9#G@18n@Xq^DjMqqaFs)IM^t)waro)a=VneL0@;18zDan z&}Qp^!}dXZtQ>~EJ)XMCZ>S$+iCJoq7rq5_%UA^^XWDjzEok(hUfQy;-owD(=gR>GX9wnjnNm#WT+o`3O`{j1vI zAAQ)s=(F>|uS2NETKJ#BPlcZW|4Z;QKYL_8g`WufG|wsA#bH~GXXj&dpT}g>Gf9t zx*p{&1tbDK1T6+&mcUq`$Ap74O^pw3(DW%9U#C-zEpAX_Ms`qqaYObxWWz0Lyi#>i zpvEy5=zjCSoM$O$uHlQ#O~!KbsZq%sWArhf^Ht_r9gjJl?jL9;C3}$J!%Ai?ug(3G zjHq@sFwawZL$|TA`<1N^`%K&Tu;caoMfGhpPr}?x={J;hL)rMc&%@U?zUq^hUnm(> zW$HZD+!N!2Dqq?0ne?f2NDul_U!hCc{JscRdI0n(+O&RBeO}q`iVoL&(=|6Nt@MFV zwm$j?`r!hlm-L(yw!TW^mgU!dw`?;tS66d*8|Ly#_rs=BGs&q#{)YKb`^<}DI)4YwZCJjEPJ@xuf28Z{Tu?eUr(boU{*UemTP=6(|3 zel9a_HD6OSxP3JjRl3AIhj9Ct6RCNN;=AI*-+auaz}tRm4yETgn7_O0`)1$#@Xg_G z(C4q!&;6Xce*f&7_hdaY?VsWm@6?zKe;e9*=Ii=1+db@W^WALsP9C{=<(_N*oBz7r zTm(Iexw(6e{!hMJw*KaCh7N~Z>p7>3|EkX@`E}2i)VO3A<^=_a6C&Dt0si^OlIGWL zc~bpCV0}dQ)j9LyhHo$KI|TYjQ3xT(|W&{xzVth$9S-LTAnwr$i8N0 z`NI4|$H9J6`d;f$t>0nO!@gIxex}}5a{#52UGoJ!_5j!8gtGIYztvnog}cWBgxNFm zS9Yu_12(+QSIrsJc=b7amA4v;RQ{NYXxm%Gt1y(IZE|J%E1O@<8`3bxg3YL(N4aV& zaDz_o>RXw81mm768~p-SBqmMXELmVG+qh}#a5gM%{D?tpSlzv4M@wwrSVeMoZc@=whRoj_vTlcDeA5tq_YZnoYdCQqy5h0((O& zIC};gEpUQKPV%Q*6hOHt59LLg{Pk~LvCbJ#}>hW*2 z$e!NgU(!qDt!j?dh0HSka|wZQl}5&~wsCQThq0#dgZlSnVevzT4raCbjEIe66%Y{1 zN^7^McJmDyGI}uc$GJy?n74MNa_OaZ|3u^ndNdRsWeij9t&xM{hSAlLBP&#-OK>aE z1-O;zGq_dg1GrV`UAWcgINa)VXyoW#Bgrvx^steXI7;QPP1nWAg(4a+s@(wX=F+ae zc5`Z1ood6tRpsKFcDPC&nTsd6HC|Y|7VUay*U+w~c9o=Y@*=FqK-QG+#PRJaLWGtv zNEq@A)UFR)buNnHzADR_xA2vTg0>XqqSq7tOtv8Nj^j&6Z?>E!5Wdr4E7(f5imhgA*xGC$gnq{G=c=uytP z6vQ)%cZ&1qdu$oQm-6h2`tBb&)JL7&c{8%GDY#?ki|#OvuUYcQWVwgWmAm9_oYIiQ zC(FIypgPweANZFCeYFTm%!`#94k`+%zKeU%z~gy;;(Rb4OrCrwA4gs|dt}n?uG$(jbR^$fGy%n1ru!2djKRKX-CUQ|tX|E__rrDo2CPv7oen7B!kS(1Sei{j=gX zD4UG-oXh88+}#DLa5j&tmGI>>PVJh_=WulflfXA>t@!k8u|@HvaZeu0V|Xtf$EWbA zd>Wt5XYiSP7E<|A2`WjURGF$$Q+ykR^>BRW7EYbBU0QW z6Enj&dV$3#X4sA5Mz9fLlt7GyU*zBL+dN$i5F;~FA{=>xGr}&4XO(WLMAa;0{BC3z zmyBJ;3S*uTWrQK8Jn|0YY@3`a`^he{kn|My#95IlCW|p*kch&m4MlO1gExPGvd-dk ziGKKUs}3*5{n=0K6PC(mvoUNC&ZUZAZCP=8N?+j2t+()97S19wCWxEZihH8tyJR5$+yXm9MA7hE7W6zr#(Gz2GMCGjMldEnB5T z-SGC$jWx@}nrC87HKyeO>!m0|G|(a0d9er~iC8T|?&3GNyW~8$d-&&YcSADrz^3pN z_jSzADn>liG0t3n0-h+Zs&KS;8JuiqL%+o{@)W|7MFkx{8Ez8)8163EmMZs8;O+)M zQt|dfS1-8B!&?p&7NWw0O{c}V4d@mAI4$7>?vtgG?>z#&5-AGl6soON-`M>c zCC5~Hyz^C*_JfB>lNdFuChY^-6yTDC^rK;#x z9M~m0!rjA9!gX`n(H;VaYrVZlbIMHwo*=D&Jag_h7v% zE8puXM1<*(?6R{EA&GK6+$8Y??k>DBP|qHKn^}9DN}*z|sTlD<$2fU-8F-?+qQY@* zR27VjC8?ORmGRaOw2AUMg&F~pxC8DktS+hiHJ9X%ke0;Ex+_h&N0icacejYD{~i&n zTf_={H#Cu3ijLzx8LPo6MW9ZhMt0q+@GebZuhVo?J%M+;it4^_lXw!`U9vaaJvb%=r#j4{)5=$3R8QI?xQdZV zu3wjU)uzc@_2xvpk5`oLfV)e^!QI1m!gbbD$^ISyDB$ST$W3wi_1u&sBx7!-a_a;) ziEn|s3$t96+g7;lTD+%1@TSQvf2xl6O5A3 zChnMi6i;=KpR$mZUFN>f!*m)of~tH3=tfN_nV3Emkm1e(3fsVKi%at7*dJiv6rNwO zl5khtB`)Kk4^%o;mP*@}KS%ss9qKBd{YZEDJl=-4=B;r4VM7eCCVR8U98=UKpo0 z7UfoM!}*QHaC&2L9?Wy#Ogw*{iwE%ByfZJsOY%~@w2>R9I+n#3*X4OdUYRcztwkHr zR_Wm?ox+8DgfGC1#5` zI7x0UPEwnXv(y%fMMe>^80X3@!?|k9#R{xzCNF->|;$2mMKZ!H#-H&*=rd#JP14WyW!`s6I0ar$tp@RdDiC4OWZQVGUVh z)|9nmZCDgviBq+*o#%zG>yPsj{3JicPxCYUUH%^Iw{uzNdu5&N#lPTR@e8o;zT=np zkNhWomH*6dV0M_!@9+$M54PVU{)GR||HPc}InMsVK~};L9ylA&2j_m}#F<}taMD)+ zQAiYqwP?favbYF=!diP&g~NK z;@r@);+!}y-WNZKE8?oSCVm#z#SL*&+!D9lr)P-_@vFEm9*W1}H|Z^XapqNFX_uvN z9^q=aMsAQ>U?<}=Q{AIIp%b6dQ+!c0kKJW=*e@)d-DbDgO?HD_XFva+`ZVVPrx&14 z^To~{t$H)JID0Yru(S7O?~QpfdofO4{fFNBrXE`XeYO&MZB<^4*WlGfOVR58+<*Uv z-Yeeuf9}8Q-_V0~pH^05+4?XqsC%)Y^q}tPO8@yn1ERk} z3+8&Ye`ogYtiJseW0p6@s_g4=fv`l1U~CJ~bI(#3t3oldl*I^D9wSploK{g8V_j8@ zb=9c`&ibf@F|Upu^VG>6Q!t`Ur+KuH7UNtyoK=Ujkq_Y1<3n_q(g2u}i zTl$W^Cw-C}T_&6xM_1_@{Y=;C2Hm7vbeq!Y7o6=@8E3gw$ElRHaV}*e)`Ydd`M<3( zql#oH>@YjdPT_>z6*x6-P1f0dd=uZyxA3ie8{f`%@SS`Y-<|!ezpQiq_?P@^%oM-n z-}4`Es^=AcjbGm~Z-{rsJ1l5Q9G5?MK!T;jV_zV6LC;M?Bg++J@Z=CCw184f> z#z}tpML`iLiU_L+!pt%l=lPYwd46R@c~MbR7F9)cQB%|wbwzyyG5e&Tt9WHpE}bIv&{YKG(T}jpX2udP8M~Y%ZC$2KNnx%j7^-zCobR= zzE@7*!|8kX!~^k2Jds|~2WQ?DlO<(2oOZWfZpIwBxbF8@_l2I`&S?%8qc~;b6~fcWRY4D~^*9Hw-7Y*BjV-#1N{c-3r<*sa+eAs&x~U ztMZ*%NadjJun?_7j4t^qJP-a6Cp#-!6gmpp-WzTWUDjEOfsb`Sb2-wJIUCSLVXywF zaQ&y&$SmZ8Ia&bDz)oe;*$g(5&BEM#4ohGQ*h2BU_ycFxaip-|Vo2))*B_U%^KyZ% z09+i>oJvxi`G@s2BHT2WE+SGBg18>fS(0<)X%fhWoa#HuEn4wwO$ z1wg$)4+*TsqkKZ?2%y}i4JCn>`CBVqdPS?8a~<+|i|m-)=s$gmx)XKY|EfImqBF=` zW8O4>apDNQ;|jfF{$~D~?MbyB`L{pwhC9f-?#8nDYHb+5JfIx&H{~iH*Aw$Tu!n%p z%-QA^9exaV*mtM@e9g1wZroos-*<)l2hV(Hp7;;Ze~U78?UDO^HwOIHE6LRAz5X*F zC@(9WGxW`_`Ph8)=9vFJ6m=Q*_tZ`*UJZA*jJv(uvF>p9{XhBU-PfOdHKpk~?e*2r z|3B{B<(l`v*^Jk7%;MqSM%@(94Ws&}L*7FpryOb*Mk2 z_#r}mMYzM&cOVd^z&JP8Uqhqdg`?Z*R5NxygByu zP}P%FYrxHtyZ}YN^AB|V%b$4*J^gaz{pr_x>ML=N z%?sv1^DFe08|GKe81q2(mNl&^)chg)Q<>=w{yp?xL(qc%%b)q7J9YM!Hq+gq*?sin zF;3diqxGM89&gDr|6U$PO|p$QW(^gk`UP~un9O7<>~8kE{|!fb=rZ8n&C18!`?B(K z^IP@5fBO4>)5(9YhjTo2_v?SRKirb=y6u3R{|DP4tAAu+|5W2QhI6cfC_7SFX3CCK z>x3M0s47^WtAVvaf2^l8!<*F>xC&u4r8QOvJ79&dFjfe=z|M=p6$Il6oV|G*?Uz$`D6=^GGNtJ0EW=d6QJ7!DOX$NLZHEAbi zO|@wkW=>6LH|F=9u=}Zp=tak|+BSgBV3lnsPBot_rsGue*&=~H!dlr<`V=#udpObj z0oL_D$4uxEeIcH}7XA{p_)EGdYhv~J5`L8+jtSXYj=)c@Opp^;fP7oN&G4n2oW%0r z_Zk+ly!fqxLo86H$uwq@N98dVBu~f_%r4K!_gHaxPM%{WSHW+rN$kyp!)W+0eYSj zz<$7Q^5n#N@UK(=>%#Z&CER^>p8~N`{1Ce!l$007%JE~In*AI54a=8G;ti}Zn*@1> z#9OiM+zPmocppf74E7}8eL7q<56cT#--K!VX1*CANgTTdF#q3=y$T2LMkohX1&?9n z`#AaoM}Ihn-3#Z@D?G3Q_!(B*Kj&ZI9sQU5OXTns{|YI;=6IWe6~PO@FY=4Pzv15i z|CWD?oWE0Vx3HS*=ddxPJAdyC(K zo6ghW-a&8VScABW^cg$@_^B}GZ#s&D7keLJ`A+qre$_QdLD1+wTK z?um8IO2Ab=_r!{472vA3dt&Xg8gSL;J+TT}1Gwt>o>&j91zh$2Tr8^`ymJ)EylrBdc?pA{9=4fs7dK|(>fmnNAkEfJ=$))v6 zF0Eg3LBBi!Rlng~wXdG{`{{YVpLi~wqZM9=7g!H?3GE}HeQJ`BweUVu%G!9tBxD`D z=ajOptP8vzc2`PSU)Bd6Cc}W^_)p*sWkcYNWFz2>Wn`WmDkIWHaE+@wK0n zEo2MeEwQ6q%2u)!@Yb?5@HVmy@V40HEAhKD?SQwJ*nuHCsyDVWT)o4^9)~V)BV+{J zDD3y;GFnE%?ScKjBhNN93qDRAByj{7=8_A81Uh8IPej21n`mAWx(VpISToY#_j_q$Ee*3axC^E zFgZ@`U%;>Ej0X-)jr0>Gc7MpX)LsUqwegz-@8Au-Md@zfljUUKQzhP*$?0-B+?jGF z+}UzA+yt2bcdnd^-2n6CJnXxeFXsbaAQu2%C>H`(d)0mQnoVB0U+%~L$^#Oo>&t`k zAUJkd;@vmKgQK9~m^=o0PGMhxMV`f1PF$W>Z@T3d@(VommHZ0s*Yazm{7!yH{_=bI zJ-$G~sDN*}f0RFh+RO4X6_h{8pHSizymu}ruga^yui@QvE{qx1u`}xi_DST2X3cRjfWdi9pU4ygiLLya8% zdgSo<+sIK=j~ss4Mh<~DKus||HNzEvF{B07-<0LzsYek{J&IWLD3VuOF}d}~;iE^6 zVtVB8ag7|xqRFc*8k-&=g7nznt49T2t?LVE{q6;e<|#D(GuSmnTv}e)HC|fFduzKU zhqh(>wSMPXzvsZJ?{?_o9eBI#)bbu$%lm09FQDa*<98lTK-U{u*L!MR@6_@>TFY~- z<%QPrTx)ruwLI5aUT7_!Lu>f}t>trQEgzt@d=9PU1GJXUp|yMftg36EM`?KjE6dk` zD_w73jrk^UrST1{Hs1!W^uB?0=U;#;?Qg&qdyHB<;ZHDD{RX>?X}gSRyNqeOjA^?J zKgt2U@1gCo0@^OisqM0y+Ahnf?XsM(%dmG#j}4w!x6T1vjS!w#!S)BPewoHoj}R6; zHdyqiV9|D5Zf(cq)^=QOt?T`@uJ_lv-e2o_53S`rw0`%{n%zU|bPuh~J+kR>Kdr}o zv>x~Suk<+H>p+LL7OgS1DQ)hp^|+VT-QGgkfL^cxCu7(56j*}!vuav{wb1&SYke)WzUEqA3$3rY*4IMoYp(UR(E6HdeJ!-U=2~A1*#SEooLbwEov`yk zYTa$f&e#hfwFWn2SJ@S~(&L82kIMm9+T4)cWH;bSryH_6_L@tr*$w>mTujV25YuQ@EdTI^JwT2b)kURu>V0R-{X;_Q0yP+}9;Ct6V{DRuM*oXSQd>^qN z$Pb`HKa?L*0cC^3g$=I0@xbvTlT8=e}`3?4x>*YzvCBUkGzu&QMh+=cU7CTjZfGg}j{E9!nDS_31g`)z1_jFBxdLblT*WG9S@5f~G@VJz&4(GTx% z)VPOHPrZeDgq<2wN7<2<; z&n>kx7GupF{v_M`Oy_Qefb73c=GwL3e4DA>VAjighxw+r?O8ixobS_zX6|uu?rvEM zeXQS2LjUTwZjg)@(6F_kP3uE*HiDLH294MX+OHinT}NoRaA>wJ&}b3RWKqyy(a>By zps`|LiS~g7+7FiJ09c%ZU}+A4g*gnCq~O*X)OOIizOr&?E0*-u?kJ z#>dbUUqCB-4V~~Uw7_N9U{_#!U4z9X*nRUi_Q1R+)?!tO0mGg&xQE$qSOHb{4{`M9 zujWmp!+s|7F24W$Mf5dqily+^nh(V~;Onsps8Zc?<@OM%9wF5|q3}Z21Xi@WnOA5Hxck8T6gH&=>yJKwD6FS; zHy^X<<|8%(FbgmnFvonvKQ|xqFU-fH9^f~??`VfV08h49HUkf1I_|aXY3>ak08(f;6@&5fcZr;te7_kv;edMbOywNQ_JA5Hq-I8+aDbB z$J=gyyzTbK+iri9(ovgq)CTr8;0_=I za1ZbR@W@OTzhhnF55QB{1s%k>h27c+nQn z4$vOZ0nic93D6JFA20v_85NLG0T~4qmqEp4P;nVlTm}^vLB$PFaYJ51+3Gu%9H`w! z@TZ%(4O%4}9D4}vKLmds>$W(9yIZ(Rr=956`OvTPQ80F&lzaIn=F`~qYyJf3y$k8R z3+es;*n97QD6ahvbk5A|E_JC3f&v1{Djk+ymkt6-u?q+&U;)9kw}`z)jlH+Dt0@{g zdSgk9i5fLAxkil|6HB}j6H_#ava|1V&h8=_bFcUJ{(KKTJF~Mhr+m-1f4}EUGqATA z^xX^$ZU!80bNw(+z#y&^WjU^`L0N}uy9~X-A6&p6T)-b(z#m+Anc*m(kFwU#n_q{r z9%Tc{MwCq`n^E=}u3^>fHGV(J0hA_`gD8hk{t2vkALTI05tO4SCr~~{`2^)8$|;o7 z*a_rQlrt!2QOi|O^h}47UOV?$1wrNL>!ZFOvX`*V+xL` zIHuv4j$;OnnK)+Qn2lo&j@@v~#W4@Zd>p&u*aOD`9E%Li;0n#)3eDgO&EN{n;0n#) z3eDgO&EVC|;ML9G)y?45&EVC|;ML9G)y?1-&EOc#;26!|+s)t_&EiXjyJ92Cbd;GW zvr*=v%tu)WZn6kv3CbR{k2wdIX$HS<7LVZf<2YhoVZ}H@Zry?0x8y*`-hyNC%HxJ4wkWiGh;TQd2ZehdhsMSk31q8k8Rsid|Ul&eSh=aj^?eHqseXWX`2bq_PY5T z9O$V(6V%5)b4HRdp3VF9;Jy=r6JvrYe+)QmV#b0 zXbd`hrkn3GBpJ^PM^PX0V9XI~%<4tdhYW+Ze=L@cD7!IkjyOC0L#!P!c;qF}@1|e; zke{*v87%JstLzOsaDAEK0CHN8(Sq|Gn5*~b8T~`N{2rqyj?O*4-vTBW$3efdYxH{C zzyE{>ZSOOlKMR$=-tjlB+<4BNt(kSt`|ZW9{TKS6kx6PDuWZfxeIHr=A3w2kVcjAN z{HeUx|CaT72)Xaakn_%LMe!C^JKRNv^snL{;3J48;opDG)^tMt@or$y9{iJI{lOt{ zocHn120c55zS4JKi`;4alf6)0Cyh2bU=(uB7WwQ7c%zM(o?C=1h*iCg-NY2g=6wSm z#BBoBD`7qP+t4F-3A=E8udo-__X+zDFFPO{z1%r!%`xyUJ%w>6gM8=xxQD{X2pt83ixq>51%rqcaN!oBEOh)bHhxPsav5;rSNQb)0EW<&6tTkjbPsc5{204E@2UN)QLgeU4cc^lk+6Y-I(Kk2#`F?(G@Nqvh^(=1IJErXUQx; zkLG%B16J&4lihs)7{IWcj)*%2nLP_SwFP$o*?5N`y*~h-ISy;?l;Jw}Dfs3YaJRFN z?zT)jH*4hwp!pw0{?QS`OWcQs8t#N)4)?L43QujsQ^WAoC_GgLE#NTd_&D^`k5Lbg zRzAW#bj{lVA0 zIqdhqAxE1-OWy<2=SjGTOh7TCbqiA6P$5pm(3(+DVjC&{IxBdjM9T zh0AE+GFrHd7A~WO%jnyEwDJ>Lxq?=HLMuO^mCI=5GFrI|uM@?zc4*@Y+W6Kqr?=p6^!X6;_z~JUfiZr9drqR9g64DDa20Ky#r%GW`TZL0ToDezoAN#^vm=HJXzyFJ zcNLJofHslwfT!+(&r-~zGx;&bfLX-#xeM@~cS;J^)H+GJK9-D3yHz8rh5A_G9G7(Nk#3z=LP+vEl0U zN4Uw-6#P-h`laIIBB$4#ee6qu$vOqz%v6jCL=FQvCojn`m@J8TSr^cRy}Os}fx@AQvm&l>I< zvrr{iGx)|V92^06^Pk}p!#n14dPHXB`&5TE{z-K>IvVETIx@lW`8vuWdWYc^b_c^K zc9qT}FpFLN+Yf3X{q2wR`5R2H1pfT>?@31KUw@-l3=_az{)1*~xb!#A-D6MRZvC~@ z`ZQkwzrr6lz;pfpQ&Z+YZJVw}fp^S=&w#xn)~!G?(6uSgvjz?08Ld2H@fe$VetS=M z0UQZ4hQe$dhU2C=Hs1%^7Pc<_J=F$g@VPKHKlP%==VhNi|5?~h8z>i|s{jnU4St3u zaQb%0`ag~D%#t#^ggYivXhB}W-eM~OuwKAiAH|IS%OCQ@skLXM056IKPjbTH&BPz& zLFSxhXT18jcA3Cz|#wc-qvdt(h3Y<{{DW3$Poe5$iX z?0sHFyU`AO>gH#@{J(kDSQWGo{5?&36nFm5f42YUC;#R_M)xLp^S{&4-&ESVGVRIl zG_(EhN8Tzg|HADJcb??)8*4L31UpJ~)Tdy&$~+#_g8F@lKU6;6?d7t!dA>ayN8E@z3C1eg zP^_bkXkAI0hL!O>NpGy8Eg_|(>?s@F&A2myKv$fe7CsfuJhR@^xZ3ox@U?ISYfb4& z^rzRM-!nya4I&X`q8w{bZLkVeDcXtlqJ!utI*HDrN^}w3u%A)~(L?kUy+l7zEe41| zSf?5yhKgZgxTq1AiYvucSh2nyD_1vR?W!P?VRbWIlero2kV7}3S#Tod9I%#X{_G4*bbh)E0Rgln=+QXACzk#r%7S5Yj)1=mS1gDH zV#&4yS>D8zxZ=DWX~#KXR@|^>v^#N!-=RHekLw=91D~G66E%1dFPwW5Z+!X?A6y9_ z$O6X~Vlyys#=d78%kB$SNl!i6r4Og&M`@DMJnAqylwT zlWIIQj0{67!^v=5sVDWg(m)ze4_zB;%`*B#mhEl9^7=%U?QOxby)Bqm!411{C}FeN z@yJoejvUzi3j27taCX?2Lyb>=y#K@oJ97l!*FZjy?t_b6Ti51bGL)*V7VCOJk=hyhWB3wbX*j3>wd=b}#Yn%$$ik*G1cfxge7`_$0 zMb`3n!gn~oA>6?An*wt4u%E&$_-k$p$i~LL3g6?`JHj1gmmz-z=l6trCSQ!mGRH(r zICe-8@ef%>GEv640_)}QODIGIu3O+e1_G$S8lN^IRy2beY>}s=6qUHgUc{!$(tjGt2sBrEgy5QVhbjMSm6?}T)Z547*j2G_r#V)0IE2xOw96&*Ss9!Cr z;r-|$cENd95$jGwe-S&kfwls0Js3OQD?wo)Xg5?0#d(+*hVyVS9OoKQgYyV60$Hb# zVkCU|(PA`ej)Av>2Q@B*M`^jZ9KWs*SK!xG;wt>QMcjh1(ACY>pidd+4jQ$=9^!T~ z%q(^xu;*;B6M+MEj&_zgbfQjyW54LBaP-YFg0at8df!E_+P3I1n=PfB}sjOy$e)wwXLdF3I@Rd z44`-{XBh0lFqmgp>%_2D#jw_uVJ*+FR$y2wGOV>@SnJ5J){bGVBlx=;>ZG_UXSge8 zxGON+6&b!dFnqOV*l7>$9|Z3V#ZfDUqddb=D~6*y!%-`SqefhGhICkod_IbW){qiw zvDf%|aXoyW8^jH`z6ozzkTL9Zh2#)81;b7^hMn#VFWZB^Ia3Rpo26Q=ny#9x8mk(j zN>$l7ZFHRLkZM2LzQ`^{*-hEbcBOrheUWV*{j@K#skFXnUu1pII>G9M)oYfIEZ18u zwfN2AM~mYY-4rnjANfsre*p2C!bdh&RwlC%FUYQ;XA_a1RB3Ts;UiqMT&iqm@gw!3 z3R~@q=sB%dSZ=x=E?i{a@vCi~xE?DkjNg|d7t4a5sBFjg<-3a$6+Wo%N4~q|Qn3=> zQ9FJO6f4a&itG8lwD8^OKi1-Mnx(1mL9LbM!cV0A;eD(R0@JpF(-LkLU}D21oAcBl zMXA*R! zzUd3_O<#a-`U3pW7vP7!06+8vL>y=g{729X^|{eri=Y*E!*3)4Jxxa`#P>2B8{oIS ziDwxk#c;mKpBowhieU3b@mM+7&lOELr!9MU}%?ApR|F5wGlg@c%UFE zg}J`gtj&dWe&<*lzE7;(wL4CxnOpP>p0@*j}n0MK>QYr z^9aK|;Tmw^8)Tb(3v9Z9<4weTZsT|l$A|Fs|BmtqF&=`fGuYAKLOa2Qc0&8M7M(Fp z7Zi7V?||ZgqBdLrN7@OFbOD;OH8f;vXvWsyPCI4jkIcq!PloiZb=eT?`%MtmP5y^oPJV+76Uc{6%_AH8lyukWMB_tC@q=-qwv zj`rw2dUPK>x{n?;qeu7Aqh|EtK6=oM9yFtMx;vj7bZZTIoQX0EWe&<*lzGT6hC~9D zJph%_EN04w)X<f>g|k4!Cv?dK{8qfl(*X&mTEgjLVqM`w+Bjq~ZtQ36vL6E=kAxm;7GP<^#|s9ru&G zUcd)*`T%tL0Cf5Qbou~!`#*r=P~4_m=RVGVM)?(m?m6@T##4heA2n_c)>-B9CI` zV7@r-j1rBq3hi2O=P}C{fKjn18Ne3g-HEoyPhN`S7Vt|C-0>avC3J@iI9`Hp>N3ue z*~ndkSN|T4zu-v`MFUwG3#v@RF$2c}91C&16vt|mdK6@-37;W{{R*OMG&|BB#Si+HTrb73 z8l{%CdJC=IHMM#Jt=_`;Xv??I@~vlDzJW2{1lHe1wCW+UU@2zA8O}m7eh;bm4W!}c zkcK~?=FNcj=QO4a53Dr`auqq`8a?%-8`C7tk}N#^H#Gpys0MfxbafPTbrf`U6n$+5 zjvWPU9R+P26=jA`(9dQ-s2LDy79;T1&*f+<4mPbFAaxr$v=WfIjUDftVB0F;eNw`@ zwMLI_p-0&54p@E<-lku0|3f_aCnDW+Y$EhDYQegJLu)uCpm`hP{T`kjB_MhmJ--F& zyMx}}0tMay3~s`&qXaj%gYQNOc-#g&Zo_M%1WaxNCf~zfqXd_?gWYKjkBJ@T`Zhcy zN{r$*Ab%Sk2_-xccJM&h!P}sOMP@C+LWY;X8Y8;}s=WgoxQ!9s0_EPpjNC@gFChbg z9-E=J-9v=wCrEZZW`savQ$U~VfLRKLYz+nG#o)UZr5m)md}y!(4evvD`Ubj_1J{vj z56V9eUiBp?^#YEU0h_BhM^-*y^)q1c8!+oPz~X0M)^C8r&%mq)z^dOc-d`}@Uw~Z? zFy3E)(+@CPzhaC(1G9d^7=ObU?*qGj19ts}S^E{U_A6$MQq<2F%Y8`eOBe?-1Thm| z0oq@opI-p77tmL#aehxKA)f{_$$Wun(D9gOd-VA__XXzdOPpW8In}DJ;{03CBs5yg zuM?BRe`4fR`dVPVm}d}2H(YCvqX&u?iZ@CuuBG6Zfn%28Tg>xy%=5SC&2{wTTgu)jF*8!hzxJ-u86q_$&rmlfgev464ET*#MN6gss zWy@zwwwUD##cs1qaRW@LT+yJ%5tvOXUyPW3wpGTY;kR`BX2g1m@pt|rZ%T1b8Sbe> zeS;y7j2I6O3v%ljmP0O><^KZ~on-KOKCHg^ zTUc3QT<(x49l^QHvN+ao7w?*2`OZPg8)68$!9r^ zN)+?hLvS93qCqi^`!JK&g{|E6JmSq4P%eXh+HzVWMx6W`j5znVqokbKh!OXJ5hslp zAyMhy79*&2aFS%;dRq?u?5Im2d&_X&GqT}tIr|mq0eHpzRtViPoNR?qTdq$v8Or%> z4etPJ?gML10&A!xa3b8_yu)R16nc!?nILH7c2DFLh2rz^8Gy&&HC{)IzD8bOEL-2bj zj$t^45cY;bCKX!bWzkTU@bi-Tpw!E)kY zIdQO@@B`}Rpc`o&+SB^_jc}Hp- zd>y<|oN!KvFL6LvN3hh7Z6C!Sk8;(I<>%{n#PNmfL$)(2>QRAgCo8j~qGTnH7Rp8& z3oOwspyNL%ouH(opn#+#)HhFb;yZynWazQYZtM<^*G6d0grg#r;Oh|S%|3~X zXFXZxSuf}2O6y3;EA(^Fe0XTbftJ2#Gh$z@12#dnRVq0vOH1qmrQ;lF&m3YyG_-^} zXhOp(FnDhi;vI<%og%#vf^NS=x{0WChs8Lod<-=>_c3fw6ATK3Sqt>@=L4!E456Z< zd_9A2A{V9YAS&YGECTF(Mdy&nNO_2@j__Vy&hGAlvkc9i(S$f?9AjhIiKA;QJ;%7{ zJfM&WKRuJT{&|FDUF&u z^ z;YXPv`v`W2FV-()ak(5In-iWZi%;(w?yBr4lj*{3DP+Tshua?y*Erx`XjrjvFfJ}C zWAFX^qUmU8?;|5ITsl1R0{1u(H5r;cyQFMZcGj%Y(mB}+e6ypXGJSlrq9U_=`M4)P z&CX#zW#Ol2{B%YiJB0sO%*`*JTU0c+IKPk{Hs%KxX*5N_`60y`O>s*>_ab_=$avV? zJs4MuLb`|0dxZSQi&8}$xG_i975^$b&ZTmlw9dYcoShGmyN8AbI5@;R=@jk)1|J>} z9ugiN;^0bWH5y}#U?XG`2qH7jEx>`fRLGQ=QwoeEI;Xh6{)0mNC`8EG%gGrfJ>v?x z1a=SS-@B$iojW#dz-uFhuj?7!FJAg4JJl&^YI)C1FXg`B*CWx=$};!s5wCoq>#;!S zvM!)cUVKGVzsR8SA#p7y_}BU_o8L{hp{Un_;>`Mzz^@|9ySZeI&B|Z=@q|G;S0!1t zvn(tec5r;j!ijX8Lqs1=V)SS6o&efAfoTNIIYmmUX++MAkd}~vQ*qyH$)8^-9}bRU z@t)j?3jwKPx*Z*(qFt3dj|q1K+A93~cz$-}>VIUZ<0n)lB~?v`S7-fWbtRu^b(vHU zs~zfzK^t#NpB+{nmd@RIHMBv!gIL0BMEz>iA8o4NNzUp|Q91LxT%k_l9f3YP$R*mv z#nF+EHa0N6Iw^JNxJY9I1y)~68>QcOs2hT|z@>u*i01B=z>(@5(nDzz>hHi;3cY3T z!N1_FwX!7&QB>NWxD zA1BWJSJsoh($@Ml1EilzNk<7v9gnwuEJc36KE@U7_bTwT9Iu!=p|L+f*&*e)pms5$ z#hou`5^Udnmu!1{MeIb!>>*u6L>;*pfT_rFA!kCvs(>qW#F2=#mlO}}80dOs#emc4 zIe`NzqE^#q{P6x3CBGl7VZP7~$d0zbE2Vb){?)5-#}aVpv4~Gvb5^?rOEBjEaMN*Z zSn`tin3#B4Dnn9ILV~qgTCbB~h5(L-dM$S$w8e!aqhesO;Go~Ki7%J!{PPQW5Jo!+ zbYTs;aK#=U;Qs+~FGrb9ZsY}&2Vl~a;A6o8rYs4J%To{>B7y~9AE3X3v!_7uU$oYa z-!`o0qLBsuDdV>d?X|d}hyOv>z_dP5snr>sT!T~lM5b3~_y~E|c0XL1=3lzcMzudrcnPm*p`uB^-Sj;P1(h1^@1j7zcu(3NcVV*edtL-`Xr z1nA2mJM)*Cqy@Z#?7*XJ@$B$9xo-aPp#iyrVx)|D(BV3 zW{=6to!w{BrxPl6jM2`R{!*gUkK)ZhtiFDP);nNjslV2xeFsl3Ac2R6qb4MztF6w~ z))J6KPr^eepP}LlG6dsc?DTov@%2?jGErij0m^1f%$MFx=FF^^njF*6yZgAbriRnA zXPy|L8NPko#*GX14GqYc@{d6S-)zioN}JNF*QB((1!aOJWn@-QU0h;AUj4q>(*1Lz zr_^S2sZSj-Bc*!dh?uCFRTX)<va_Tb+zI)@gzZ>Qp;)zFEI3u*;-^s?8%4 zVs-C~>APW4zST~f>@nR_t1~U$Q7d6BN z_nw%SF}815IK`b==$VIX7xrmrueDVMw)61`6gW{AXrctFa8PQ{s2Jv}m=JI63MFw+ z0+sTNSU~O(j!~V2WPAH%Cq`?63#y{EgEKmHOdXV-KdYC#Qkg$$aY5b2>ae&`|EM3k zb97wN8e%WowPE0<6V(?gR#ZoX^_!TJGGSnJ?u0DTu&FAfXx>}>Yxc~|tJr#?y3bnD z<^Cet=kAyzKR{DXDVN|4DWXnbRKoP46cxzbJ5u!1zy2ZLzvTzn&LNKu$l8JHh}=kw z)(*I84?komr{%o0_O^+Mor6>=F)~P|^APEIs1$sCNtn~-Pinga^%xwZtzuJ|mQ>eEBlb-hy}qP&V^vt3Zd?77o%InOg=>G` z>$I-_>LXR(3|c!RN>eg9CuL$~Y*??__~I=yd-PoRPQM}h7IYu{kMs3uBme#7n-zHf z$Sm;qTI>qs$2lTErQjS54~0><-W3w1C$6h;<(e=GGi`4}sWTh<0Z$aM_a)-LE?OHR zD3B>?A;SyAAVNYIUnwRk64{w3BFV@6?v}GcxW0R%5YBIw?i?qRMtv#WRE{Rl7_g@; z-VJgYIa<`pQ0|p090lHIlt2`1I*>vli3;guDk5~(#|ph$w(}=n#>aS~IwH*;bBVN9jh1H79^}G1Fb?z8 z5qmmf#t7-u$=_P%;9%>*)H2F29W+$<`X=$wPk6TiUyZDrsa|cZ6yEC~@y<&Q&JPV6 zm>D-jOPct)gQKSIF3Ztrqy0XJiHwSgiHeLNYs)89r-T$&q-P9{^6!&5a%=CBx5o5e zH8VleC`~XMHRxfT6fZv~TZPPqA)H$4=qQtA6*=iD1`oC_E%oq7=o$*aO*uq}1|MMY zj8_0{pzudKX#}|mx1yNf#AH#q2~Epr67bPcj!{vaDF1MjnQwJ9J)IFq5oV~U54O{$$7)61hw z8(p3sL4Gf3T3Z@cGQEKJ*I&pf33E)YU)FQQqYq|h<}W#=U%K+~NAt4$JL!S~i#OdM zK5xH9e13SVr02$)(se`Th&;7N(%{VOqRBl2l1FbDG<4U(N=!JYXPt zf^$*qvE{mIokfMh%GMTvYa1KhLf|10EQ}n8%E{PRK-CCSuryeDYM8rJNjffhe#8e% zC9g;x!c;!!o0hY(kVl_0J1=C~qXNGD#*Gxewt?l%MQd*0b!f}2s~@jc6F)ztjUBJ+ z(7wHrwM8e6wuZKhhMl+Cn_4=Q(RoAZ@MioSs3v$vKlvfr+hpAFJEg@u=}|m6BXLT( zbn_Uw+OTtyw&c~i?owVI$tsza9yhsCy2q38U575sA9s=8wk~F9PC-@3qaS5Eg9a|G z?*B@Ot-i|!?U;hXvD$SpmR0M^<{qk`e4g$nbP@ES2BlfKsq6(^Cr)Mw_UL$>mywvrd4NVyRX03F9WG@EVl)Uq+wCUJ} z8TU5zoATkD%#8zg>o@kLw^+W{X~ZOo-MG;{VhuHf-1+bD|oFe$j>vybeeru=+z<>&NI|&+%5)*1$!E zifnYi4nP~Dpmnk`?3m!IAI<-yH+dKIBR6j37o$ePO~kBS#jFKTSOxg|%ZxL}CN3QF z2HDDLqjLv4i>VAu75&TXMLS>wSKE(0_v)-~UhS8^^76}Xemm9im~H-|hN|V|A@L*D zl-4w6J4rkEf9adM49Y3pO~|VwUnSh`(ov_T^_bqH+x+8`M!j@mLAGz03&~theU*SG z-lk@TCwhoo90h8g@j3=f=3_C^9L7pvNohbVZ~4LVm^kpLkVGM8_A6nu`+=zAS$#Hp z;XWb^BTT>=f6O$ExR352_9^ajqB@Jcjg7)mN8LKWo)GFJ0s@&83A45<*c=^uZuacY z#vV(b&^>QLCfOz`9{;)IL`}&*7nh8UNttFEjf}NU`AG+@yEy}f;iTqRfjso%L&Kj!uJM`hycN8xuX6&T)nx@Rrfzi18tx zUTdkAcW@ANa>{Th6`&3n@>Fy^g|&`Ct14yIrDRq`YrmQ`?W+yFd#(R!+N`hE6(5PL zURK)wm718Cp|A8WTUr$%$V*<6TK1Kf?<3-CC9e@-e|h6Y|6wfw#YxobONo3Z@g()cuM@s)V#=E_K zbL`lgeEf#&#-iMrg_H&BPEK2Rq6RZH!eAwnWAp;#MF_PdJF2J}5+GOsRXOM)fB`kZ ztMDF9@`8{MmJ{7{X*v1#QvkWRmI7pHS-)3?03b{IQGmQDeLbFd^IyL(SWI|%3UFx$ zx$3KRZ0~L<*w|QGa~<5;S?I6~fua|>O&N|+HS}f%FGQK*X&L7YicoDM-@SHYT;Zyr zQBpildJmbOnfh|Q^vyA$SlW1!)W~)Y{&3OY*GF|wZOECN+w-N2M?dn*iGCdPQocXDdxjC~ES-5P&1b5UK*>=bqJwEmO#jET*jeP~qGhnof-jjNiL zS+y{SG-!Hg<2*a|sGixY*OK80Q%>rNrsgCKOApS~M!EU+9zLg6za`btP)F)=hi7`n z3;`Tr2#Tu!M=E1L-&k2g*`bk@nRGhJJirh64-fN8AFnfV3dZdhV&vQLtcuxU4svVo zQARP)Trk=vY>jd_lNm4EJ-YPgmzV#v;7HF^gR55c1U0K)yIfay`8DBEi(0olJ9{~N zmps*$L_O-J+7d9IlUoo=NIi?TbX~=G266Qz{8aMJ=lWYKq%CAyyoDNb1K0SmRj8$1k*FybkU_NJ-XBs_ZasVDc&vU-&_M zskSk(#ZNfcGJ2e_V0=+a3B-Un=h2lVvW^T>mPw!R*78PbZS8?;5tdts$+@5cUsicL z@YY9;NOR?ltN$FoT5$kl>Iz?8tg$b)N?l>9FKV!K9_hJE3u4Qmx9t%szb03e2{ivL}>S*I9+LU!IlYtz?+;C0Lu3PaNTWYVVV0>1mC=M; zc+h^kup*{*eO1l6VaYZ&$-~xGS8p61bI>!jwnt`7c96Aw&ZGgk!_zv-xku;E^M0za zCzlsx%|1M?Wc=YdSv{7XennNdxGp)guqOKF%xOi5wak-L3aH-$)YVkp!6)ae(22ka z=4>|lD}mz>tkM6`(%_R)8?t3PW#f*Hnz(mdeEhh*6Q>@o!-bUkZoNjOd2~n{H8W~p zV)^S;y;pu%=U?~X%HB2aHKf(1q>Y3rnKi#5jp7IOqyF3mVz%C*=wc0+DRhJZj0q9+ z2lVHQ@Dnw8ATv}ZgV}8IL`O1Xn-`)FZr8U<9J!&Qdd=`88|&oZYikB>7@2s`JF_;o zWL&nhO;A=%W**$m=Vif&X`|NJ@CaRq7pBFpjKCCcCV({dF@-?RkJjI&Cvvy~Ga0lt}Vt-{yA==C=G z6(Dq7(Jiv2sn_Na9aFrlRFUrSy#jCR_la+h*-~s_u}UU}^iS0-T7=oDka`KfVBAjd z;U+M>f%#FeTy?k6lqhZl?h4GM=vmu zJohqC?mFX3?o?tp@Ge#goxtgM1}>An3u)nG_VD7I@fw)3nN2U}6Yc0Nl?9ChLJx+Q zXJ(8n3fc|OA5gYpxIUi$xK~X`T=fhp%PF}HW!j}3XTM)zE67bvouwc{ z;6Z zS(b9Z%!&amJRrH%(6Z#!(MCHf>!8p>>LUGN_dY3zm0l_*Ek-~|agsv%@uTm^;KwI; z@pU~`Wpgl4NpIGQ&ED3jFhU0qF+NB8umGCO1mL0wX%P+|wnJA(KG?^{)fzwrC7Pzt zfqHcCVXQzS{BM7 z4LulJla*Me^%e%TY!Krl8{>qREUDIq@SpUmrW5Yr*YCB)+~9%BtM#$`$pKYe{d4e! zAtTP2`t7E*b>#ojV`gQcH_z)W&~As%U-3>TZh7ZPH~xZmbf4--v`K9``~4jptaWNR zlcY@eJYi2J0fa!mQQ&}c8M#$SdgH#4y8YAA)2Hp#>Gn-aZ}Q6@ou4-o~{SQnircd9eYxs*$l=PXCa?xSh&w*08aR2Fk8s(6v)S;`bvOC2gTJs{TEy{j- zOMP6!rt0c-by^#1ZQX{N!JF#i(a*Zvl1bg1twXvE&CDB-C%pGO76yP8 z%z!K!QXHnOhpx)-iC@k7=>qv}E4Q?iL4db%7L$(3pxN2vNmeL~dJ>Rn5B5TE?dqO|7)MKCmBKb^DiZvV=>%w8rSCTb_kp!Y9OSxof&k49);2YC zO&W?*w27o5spUs87UT9k_?~-y(BQ43d(0@)GYuuK54pXEPqVl@~(uv7m5PPq&($#iHg^BOAj6=hd_u*rPFopGE#W`WY z4yr@>V@#*fkEg6aIjlhQpjdp|39X?ym?4B%-MfED+DPgyJbLsMsg*W?;f*Ep^>_7G z$VzD#-wB&*09GziuCNpJI>93XdT@17!3Jz6*V$P>MPbmQ6;q3u83SsyQ9X&qkl@mJ zp9Gzdy4!{24e#cXt|S(|isZhDE=!18=qyRsAe?&a)ups&q>Z?e2;TW=;`){YAEx6) zbHFpC0SLR%u2DL6(AqMaQBG|}ITrRMQ$h4xgHhPsa*uzgj~K~+Jhn*hv3m7t!K)=_ zuCQwjOe$Du(o%ktJP~rfGsa`(VQFR6p@X9nfh8D@=m@+z%%g&E65Q-?%teON{H&-b zCk4|y;aG{J4j-DX#E(~x>tDEI(a1q5>vpK;u1g6l&0VykP(47NsOx-W(z(H}mQQS~ zJ2!Oyfx^#zQ9Z6B5&*X1I;vii*GuIU}F$yRR(={+v1@4TGo;;@MP0Kt;v&#j5iOb>|&>z~YbpFKFb zTS|CXR5@U=Kw3qr={pJGnXuVS6xd%XB;@K@nxUj|LeLz>D29s|-v(uyyh>@6sCwKi z%MVOS3Jip@Cm@5*8rdb-{RDo45U#V#G-z~R|2M->sFngBX z9XSSanT&Y3$aFTgwmi=gdRWZJ?7gj^*_*UDxzLOXJ`(DxF*(QV9TbN={#~?`mg&n8 z$UfjgVq)I)?dsq%`sCCZ2P%)I)h4A5Pu@VzZykLa$LOBb z!6{Rqnh!PHk_`}l26n2U3x!~<&Cm|uhGng*9OkIxqaZk?Kfx5y7^Xl`CB*nm(R{S`}o7NbI(ZI@gW4BoqHBs z+YWH;#Apy}qnVu-q;;_dN-G>yDhnm`P&qi;=qw#tGj%9h0Mut`W~Wi|(dboYDlZ6u z-+}tWghMUw3u!IylOd%3Us8`r(tR@0W1^JL*Bs@)oiatbI(hOH!KLMGp=ZmDXrZUR zcgrKZ6`ALjgCpdddSknX&=3-reDxZ2??8QXpu8c!si{f&1@GE>LvO`D#4`gV2XO-O zQA4m#fTz}}I2ns>14<*qLhZb5dZc?eJ1`-tVQd=Q3_;lc>MbT<7=Q5|N1Ft0o9FpM z#iNsai{@9?y)`Zg89@zeu8eKUnmcId>=duOc|#lC9iN;$=5^^_aczWG_XWf2-yVy* zcGs65T-C2BVbHkv@;O;P1#_$F-W{(^XxLagHlidhBr87J%QIu(tkU8+6KYJv$W6LvarCxZ%p8O)M-6p$`e8| zVxv4gv&!d`b{p3-7%-|5f98LZoq%+7rJh1dOFI=<7WgN4N1N=w&TY=C%r+d^VKy6O z15D;ySF;%>aCnD$ZlmWGsc#Y z2M%Ino_bR*aOb}e3sRpn;i8Kp4UUMuKArq{;q21gYlf}s)ob0bHG7s0EJ^Jj9Na&( zq)bm3eSGz5Ny)2UF?ty_;y5A8mJ#ydh!G!ROOR!f;rIx?_l6CX?~WQZ>fOrq8wT}G z9g{mcZ%k_M?`Pe9yS)7E+p}ihd~4vqw{FhfMC?zG9C=#0ziE?n|BOy|hS*d2xs2Z< z*aO>K@orG}yT^F(ii_>KM0%j_E;@YP!`#A>*ftTPaityF@>-^iMtZBU_3! zIeLs?Dj6>v7Q7k%cie|01f=6pH!Dq6Fy*DnC=nlSIVL1G3Eo?`=1NHWd zW41j(ZE5gpM{3)9wQtu^1P3N|U2Sb6Ba3b9>~e~objdkv@@Z-xn|ms%r;Hb`NuJ-QbE)h8y z-ChJ?2)9_~Dg?`Dp2|}>xk9kMLiRemU_3WNILc1}M%h94x7+JvbJR^_sPrkceh{Zo zXqf`p(F}0_6cEjf%+VPcy7Y8iM#kvO$@x)HIXO{L`QkC;bJPE0GBU9>=PK@qOn z_}r|G*|W>~H>QX5j^81)Z>%MD{;82Jp-FkM*<-WfhUP?;lteBCbdvcc{5rJe2L2ML zwdv4~Z~xaxcjAaR&L`V<6Nwrbb_O<1J%oEsGJcdoYd-QlXB^&7L4)C zB%Vh0g~+Bi9lbNU_MTmqH?CKQ&ZB)23xfNN(z@$24jiEVzDIgqcXUyAb+2^dT{At? zJ$mq5e)%Im+Rt=!A_Y>{hEwjNp)=#c81xm+T?QhGh2BES!9JU=j2wAo6LH`zCVV+( z_Ls={!`tlnS~5Y%fouVWTY`-|EmM}q88gb`_)9FMJQ?zt8!hFN?TGL=aSr>ets0$p z4kk@_IK!Y(8ig^1+E>k_HM{_;XW7IJROgM%RRre-`DDdJTIKZ#$&dCN?i}dr?dOE> z6gP=qM}EQ>x65FOo_G51aXZvD5IzY}~zDP=n^^8W#I~bkT~S zQ_^@m6;dyW85HJb(u*i?f<|Nff} z6egC(#FQr%CeQ=6PU32~HNpg8FSznDDDVoh3_C;8>(~WEwp8|xs3zA9m-*G4nk>i8 zmljCrh~qD?I|^{eNp{C_tb?||9oG&2!kr58jo``dXuusm$_}B9<>LmY1@#=9kuz8mI3Q!> z>wWugA3tE-tT^BNzGQ^iG{O2+?60Eei+L2$dmGLk`vJ7$!nEx;S62&#vy%e8)-EBgakGyUA|qeccleCJH|K z`TWQFPO|ooZUINickGbnu3jx`N5jnerK?x-BM@{1jYs;DDAspXS%X;a^FT9KkRNBKR0?h)1QX2_pk1cfjkJJVzrH#@tl3Y70F>zL{bcZDgEf{fh(!`@9R`r;ip7~M%D40e!hshn0?*TuH zZ~G_<_}ndsu03^vp*^Gi2mxgd%EA>!!C_ub)*6=ptf&i`F?6W$B>h0_kR^TVwpFUb z)tN(vW)6C{A!_^}cYohmncXV_=@tJd|M1??8I=)Y-^c$D5uvS*$zCzKV0dA7O?01v z{%%p^rur z?wXn5+?f-)x@Nd$*m=RbnU|Dg4meZS4Xh}ZUEPYaPvXvM^AH`7naV5?N{7f-HtLJU zym?{{1-i17NE!+>#gd>sG1nlA1>OrQ&*(NH;Xu;J4dr9s8?-=r+-Gb;(16USkptsN zdHXT@r|LEp#g6>P2>$w{e>SGl#U8b^XrU^JVWd@;&Q4kV*d zXioODOJ^1*jjM=DtXtN5@J?cJc#?;-bTFyI>}PRrV<*moIHMYz(|$K6 zta}xxkM8P;aQhQxov4jzg@`GfG(cPkS23u?oy;-INb^y!KX8D=g=)2-!O6)jQtMhQ zYHv-zin1m~m&S}TB#t4Hz;TE*LJu;gMHzWXYlO5FaDNTO2w&-kPU2VTIb%qsF+IKU zZ$dI+)1#v@t-kMz1ZD(o1>!PYnC?R}-w@jku$C8EHWd~NQC$fdTNXyX~6C5j9CJj z@HTk@Gzvu0=h#sL2dl}92pMNT%4acJX;ZsZjSd?Qkgt!$vf z*;<*>pD2@lqM1VkQZJr2ifrDa6=DK%aF2=ukBN&HFP3&ZlZQ)Vr@fdD8nIJzY#FFU zC#+YKF^J6cS9C|)>Z?XSY!nJ>q&!>VA#)s;B~!QZaBl(=&=}0^*(6NU|NEy7`^Z9f zsc;s!=xI;#Wvzivw!9Tb$BLn{kum|n)VCCsjEQY;U5OYKNbldyH-p^1VU z4an1Hwo19;&nDI8QEN+g_bI!3I zORK%2Je3Zqc7cfg-5#|&Fj-gE9iP5vuYUXh+k`%GnEqPJIK zpIBszbTWm4<@~5)i@JwaZLV+8H>@vESZ$EY{fiR?F1IGJLtOQ2coMNLTww{F+PBpg zs?oZK1PA+tchT^=u4+FEoyyVC#my4yGXy{o`o05J1_37kD;mmR+C&OgsHk9y(G>@# z5LdK{CI}<^Kt(?#EIPAhg^pjIzcMX#MUP(dDl`$5^Llk(k&?DNfAX2-1qI7K6@PA< zsxSM<)?%ZLT`bZxrL%Q!jEuFj*{D$V+4S{H5<2tiO?@6+G-F4bT>c7-ii2$T=d^eu zy;|#p)wx*-NYF_ravBh|2+s*S_-J_QlwBi7?U|C2GG(_;w|h$J!A?01 z`S}ewojP?JnV;8?+ma+FYfU>%4q?JlhkUwNh!NVDUf9 zJ4IdN2UsK0E}tQB2j57I$=lfq7ZqQAI)6!(R|M9)WGDj?z2ADPaP`nmS)E**G#>HY z{SYO7{Fss1a?pL>L{YX#hwLdE%cU||$beKGs0Vux$TrAhkUQfIjTsucOfK)*mBY^d zx=4EpJ|?dmV$EJx7RifxG3!En_-4;-Q+t$l6`r;)_ARcCG};$5%N+K_q%3M*$YZAM z8#qThQSbXdSQu23`9VA=zK0QbAqxOeZ$GH|*Z{}ARgVEY;V(A!{6%ybvGaqPDWn~(Mu9vYbyOlK9ZYu^Uez|<%G?QG`t zek@jRGNffwHx}<-RXs9L8vb;;ww!)`B+>_=;4Kbq1( zOfGe1=fDC`Gs}mu1EXC8OWDK@uoVi&_~rCB7{3(U^qZq}eG{s>byer-a_d%x=Y%(j z1&kYxJpt2%al_Sa*)MfZ81`~OV8P476B#!|pSFv)1f(#-58=EYd*O>jXtba%yl@)* z!c;q{eANoyNGIOzkF_gTtbTyBUj4Z}5M}40zgcACWgI=q7O?&Q9-)wtL_uhSB#DC-=Fp8C*%r#056^8 zF2Ry6%Ed&ru|pQM26_6ndSiHQg1DDg%L|d8hc^@KaTOaKtc^{xhfP)tHDlU3rYFQ! zBo3I&J_e;|G%2*mrp1-TOza(38acXeU{*$ONLCgdS-8}lECc^ip>_+qMwW%Zl+`Mj zy2dDpDVv6}P!iECpnJG)fOmwQ(k(x}us%{38{07>KhV32lf^_kkBYLWVU?&Aepew` zb_aciyhby`%28|Qp6w`Ww8i#6t`vII&xLS3c(>8v<;uXwu*ej$VJ0;H!IZd)L``Z$ ztiONkl-SCIfRwnTF8*<{JJvZP5~2 zk8Nztl_&ZUc-Od{>AqH9bqac|AMwOlZ7{UV$9>&pcSHsEdJAI59MPAeg=O;5@P63u zb>_K7*xBM*-bLT^=W1UM<*pxvM;!EQYy< zXZYu3SO6X^YVoE)F8k30ONG!#H$egQuv%Gel13uiq;uaAY30Ln;!R>DJtW}CL?0t< zm7EM?+WMn2+CWerW5)D{<|roN4v0UI_3~u2X0?k08k#-z!hxuFN8`5wiF(3x;?Kei z`8=FaOCQ-YPHt|t6j;c_peW^j#^j~KM9bPV**2_7-V6QbU@6wz2R_OiW|d7N7SbPn z&qP?k4NU%ZaD(xT8|;E56bj2j2EL1zrszcGk2AU%pcrEbVk*}3ZKUha(9@2v5=Z0A zxH=kk6MdmH%tIR+8Be1>M&*U=y#endvZJCkL`@x~y#D41zIp=LA@TBweERCg7i3)? zU1X+7Xyc;-EJQ)=HIU(SpAw^A6SarXm5fh$HcdKQ3hI=-kZ)5ss&5gJUP$v00#>c+ z{NhRnTa|@(o3yTeLdpRJM(YA`_~d}Jt#IH^Nzvzcrl z9pF(_>HBl;y)%>Ed+(Fpd+&`@QV1cW0U?znLqg~T2uSac01^oTBB(S46A+0A>RPs` zt7}Q<#?&oOUt5%82&gQvUSBv)p$RSNQ#~w~ygp7pvvLzM=&dMM+`S zs4!VQD~TIyEwkFmbxpWb18c#y81u-+JTkTOFq0uS#)L-OP>*gz(>G8UifyKpDSd?x z1A8IaPqfU4L`4Xh6NUWxiF$60=CvO3x#-$6JUlG77#`L{+iAjgax=J`ofrogA&i!! zY!{4UWo?a$B0aUMi;IDwFSUB#X7t3qVjO0*MIwSP;h87|Zz(7Th5+rIWmpir7a#uQ zx_1xG+(XFI&#d{?!SY?=gn0JE`qSm1bvx^xTXVW1q<#;v9zR#S`;BGC{aJH{%XH&G$Kppnrq}7-u-4$5nRGY9;Dvh1=)XoG2SCri0xt6pf zDCz>HLG4N@nUc+o&PS+!<%syVv(9Ti7_PdW(7I=q^NP`h;$^J5=DW*xH_X`c-e%53 z^WD5dtZXOXFZlP8o=B{ZKw?oDCx{-e_=y@$-XyNu6lc4 zY2Lua*7S}dfAK@|#hvryuiX1Ry%&}xB`v$q+tYt*dztZe<25(GU8COd)zdXKPk*&T zz53gmYfQGAlx@GIc^XHVp5^&K;a&>Qf(@*(G}O`=2P+~I5}Yjz)xNsyZp;|vVu+71 zvz5QEP6iHYZJi7Pk~$Z(&UBw2+cw-(w5mSSU*9mXY2Up0N7`e@+%no`C7- zwdNJHWx8>ZM~rNp%?!+ywyue8A^DMxcMn$t&t7w^p={~Pn{o@cz5JMU;l`%8 z#DxPn>3Lf^(=wMFCLd0G6CCeuEO1+Fj67#7tKfX8R2>ofg(B>T6Dq)5Qca>{J}Q-y zQ>Zg(K2_l~o zwF)krPimfCQoK5Y1n-?0R~~E?UY)b^>Rfm!n75*4u!^M34NdmSgOj2&&(puCExkaU z?=5^kXG5-|^AqO&?n!=D!J%PxVMl)Rm`(Ti&Jxrrb=&m4vd&uWP|~cKZBw|ZB{8vO zQ=v`j%GNyNq}I)NCt%t0e?^w0D>^d47b$a3PUd-=r~swf2&G@rTR|vEyq;n(NsNeX z>{P%Wxx~G+9vWTNAbZ6K&iLcE%evz|v*x9gF3otcXx&IzXf&+8SUPMNuDX8PSLZ%s@9hI29_@?$ZW~=&i0?t7*p6= z>=#(RBp;!akQx4-2?2I^mkzO?e-*sU7dvE~s&CB&BJ)9VFH+YCBU@`iRb_067CeMy zJ0cXMV#)J}*b;CLzN=guDQt_e@JaGtm*1O_Q0mp_8er>|5nVm2FW)_O#eDG)?>zB^ z{M=HpvgX*SrqJvN4}-=OdyDh0f7(ahMK%caM>c5v zkxqzf8rji$83gX&k~Bv=oyP4TQ9+qgy^%OIRMVFHkCsL8olnfC{>M@3b1d%4ai8XM z^sPP8d*_TWGk<=9LqhXo)c3jqR7<@^(C>k<jza9RBrVqmR8i+#C_nJp3+x{Pp1lVS~v_pX-i^sotk9 z^u067zohp-wMtceptr=I+B!Gx{*x=v+RF+$$*J}-<2I9p12@K|77b(~-~G2Sv6lM$ z0i{U{3I;ft4{(bUpdAayigrl_286&1oSR&xHbq)C%XnwOGnVrXb;g!J{kA2{q|ptc zB0Sv$_|U+?9K{dbCMVX3qp>p+d`-w16DJQleQTw{Fu*oxR=T_R&Jt7i02e(MePWu( zt0q1aij2rJcEPE>cTT&j%n2D1^qgnS;_Zm5PoO0sJLF1A8%^R8{xtT<1sV!sd#>iD zrta<$az`cxY1)x=29a4lc?juwghbZH_juj$1b!r?Ze%4egF;8#$sE7k!FxubZ_CCM z@1m8nw(nj%wlO!e|3YU-ZN*!=x34@~Q_`Q;IdgF`NhtMfa|v~F&4`Vkcf2pH{NOL; zNA?+*8AMfthn2>=_C!=K%|^XSuz`iCg`y5Kb`8tOPBBe^OrzA_zdrz$u{022I@o{Zr|UGvE%fIDMmq?mI7% zd1GAu{TqJpAmh{1wjDII2T?UQ$mkGy)?EA)s6(N$Sqi5?qX4g#;^R};v&w}Fo5Gfv zXPw`Zy>etu&$GSp(XEH)H>1?a5Uj;22c=EzS?(_Btp(ZhlAQ_ilD=4bV<4mA@Voui z>X)|_WUjrmn0R}ktSN56#zN~^hnHk#b*)dDaK+pfh+FvIV{SpnfOgcEGw01)j73rD zDe+Hw@~{N=?Io)$NancYW!E~+MR%0aBFy69CB>`qNPMYhqpQD-TV~9W54uv8ofC>` z&aTSnNM%*N)jONrtb;0MMyeZ}%RQrGRW3dbMxL90v5l1j(;cGnZ6$WdP8KW+r+T%& zA{<^^E3F)$>&0koxlRCX11<$5t0)}8CcH)?L7 zcCCASR|)SOWD>CT`8J#RFYYRzUcWYDQ)^;k>n8G=RFeUu3;wr^j%bywf!x5H(hv4l zn;N2}pa-aj|LsH{sTp++g-T=Y7hDxm&=D`hA2=}aZgjl0nZbH@%fx|p?949oUimkk z??P9=Olc3!W6vHgQH)kugV+X^6{@KwqwKpzU1IVB@;hhvaM=+zTq51#mRwjqXYaxo zAzsr}(w*%#WA(wpO+>yT`SF*!7XSLmx~PuRy%?`b^pI0d$Kulga=AU~WjstVBF(U3 z7AKVnELJdOH2h987L=0a7|m2ks90P_O81O;XEzpPAN>C8nZNDJj8~V>PssP=gPWdS z@$~DniSLX>d2SvUGAesPY{~AMJ)LhpQB!yP*NeNZABh`??B9|Jy{ur>wZ^hHI32 z#=d`^+x~}Rm7Fk&z^G`q=6pwZLI2^Rjf7jCbofeV@4F}Jau0kxLjLsQHyD38GDaNO z_$lBUM#R>@z%SZWp^nyW3$+fgMpHLKYot)^%7^ogfElmN1!Crl30NrP8`eDpB^0$3viw@5WtKT|X)41=gmei%ET1X)$p0C}Qle(da8;&iFviB~_ zvS>KDz)Z&9MGB_SDSrvv!UlEduoBGl^<8~N>dt<;s&nI`ICEAS)^xztGD24X6@r&k?D{iH}Q1QRCC<#HQ)Y<<4(GGn#hGgQ)8D>%y0(i#u0SG z1I<5+TKYb6-pMosF~~$y$}>z^4r82T=u^#$N*;-dr~(AVJVL9Z^Y>E`{mVTnQaIiY z6_RgwCHdG#RxBy5+*m_NsU*}lF~HGSD4j^VkvNk#b__`L4K2~qQc+)dl)X=i5I;0T zsj23fy^E44IsJHt5>sGFdw=!=)O6>^>9oYl0`C4xegPV#~Vhm&0v_jNEpTGU|CQK1q~8I@r~k=q6gN6ANz7u%E~J%xymIu zZuz_0Q&umtxz4?L=K#0mDQgd_wzu~-pIDqU=wH;HR<|zG!kuR}J=`)lt5n;h*lRp= za712sj?;Ixx8l`)fuPZ>Sa&sWF;gqB+Cee#La{*Uzo}6~)l|aMOt| z`c21yq1Qz*^qk|XmlZAFGlLI4azq@_0_p8KczY5^Z}YXDJ(Y*o)kPNWp)xKUA&La7 z7{zD^W)W(4cl}@+8^GM_so|mV)TSd)|Btk4lK+D@f%_vWCRqSZ6Xq>t@PPu{!D;RR zUy;7+?n?K>`mGI-zOkioUe*fkZ<;P%k8D5skvett<)xa>#z|euGYs*1`K&UH#aP< z!Wxb*8%s+aE{?=MSRMtXW)}N@#9W0r6F<6+3+@k}8y^Q(TOZ5ne4_2@%DIm(PQ`?N z$X}L~wImPoNrvC(U96)u^BiHP?TMYj+}zH|nNtJ%ipgT;%mgQ+s25U_0`=?_ST?xFE5+y(J>Ne&Ler!=EgfxG~QC^3EW6y?afhfrY)<6Q=eS2D5J-=f9-0 zk*MsPcu+3G=+u*l{7`)b;napass^Y?vcQRkOj=BL2LHS<)VWuzFD2Zf7^z*$AXu&OF8N;V|}qS7H1C&(daZKCt(KJ6y?o zv6>%ar(e8(rsZz4yqEh3qZk$c>^;QBI=4{|Rb z*R12cu4}5tT@*gCxie=6RV6!QWJKu0p{3Dqyha4fwB_(H+w)5Uzk6|aONPl2r`D(c zbrS322)}(aTPcb^LHl7~hBoHxqTym*VViIcM_j%lr+^58{{$8c3|S7pqrrj@&`sR&Q@leYL3}gO3Y;OAGlZ*-!U5 zr6klMNwI>zb&7_iLqTLNM^A3yvnLmAO3vKTRlccYylB(K`MuXRd%Sd4{i1`V0c9&nPjXjQBrnR%TAY>AnLBrXOKep0p{5xF^W!SQ>$lIL z*xz9bk9L6uvX&KO%uA(K0bYsGHLEn2%M=_Cv;^sFQ@UpUcI>t}q3{f9(Db+_;`((` zBw%eMAVJD2LP6F4*3uKTw2a`3*HhCH7Ceq_5X->O$~$@xBkplGp;Xhvr;ck94_ z6J4~p!Y(+g6q=J11%;HeT ztLx_-RVR1cT;GUqWWkRgD40+6tT-YcV&{f>BZ0b!0E!?0@{<)^NLztZjMM8GCD(E+ z`H;o4J0})Uo^?!fm((3QrsG?5wc$t--oO(n=LPSxrCb+6OJcfId!%-n*7|{ynV~bG z8i6X}gX8HN=QCy-*08WQ=g=oh`0jBy>czP3e(+ic(_vEw@Y-9)$!iGNfF93*Jt2w( zu7tVE^-=j|h@n`FL`or)#$rN{PX630*2CP`0F@d_rS2+-|1nbd!&v3xOuBb8kbPhU(8y!*L2F z)!D~KDA1e{k6bsKv1diYj+!91)VfuLUE61vUN`Ok;>2Sg94sw8@XP+QU$5c4hlj<} z3-=V3?zq~vXYLbgszXcmHtqUoW&MTQ2X`I(@qGOnYL>_@BkSlRHa{hzSrL#2mLbS` zHSw}X*b?P0_9$DDG-9nvOsy=s@>CJgMqB?EvMO0R&Y6lQ<(<>;L>~sQiW7hx(kvJn zGX2SrqAVUKEWpOvpt=qIc%UH2Wco%%_7JZ$P)a=l0N$g{K)S2IOkma_llrJ2qz*H8 zIYcf0OL1Y{@aa{59wxsU7yo{ABxhSQ`WRjohusgYwIB+Y7fcQW|WMPsq0^9f|!;ktGhv+6Hoc;q%(vC zIg&ZXW3@Jv6zAYGh7uR9&8pnn5@tA*v+iPhQCGr8ttWeubWS1*M?>$mFBCVePo3f| zQkXX`PIfc3DBSkhiJF>GVs)*{+SD^qbj=^Q*m5t{VILaqRDGy!u=a4ZO@S2bD6q*~ zUJL&tCxdw+_`w>aHASe}=ZIr~pPw(mf2PJ}3bl)~vnlfhGmTHGkAlQy+P@B~6*599 z!IebGL-EMb!)LR2gd-l6i{stX8`8cRB_|i2=uP%5Qs?K2)ior)uqh&=aCt%N>Eu&)CloNM=c1e$_!9wzB!=Ei-^cm?TbbFM|wd|p$g~yFa%#N z-b2c0P%t<{o#Pb5%;$8;9tq8nIgWrGoeQAS)KLMfz|AFkFK}1YbCADes?}<&nBX%D+z98QEnw%xaNBK~(N&J)Z zhs4f7yxB*dBfsJ1iK$OoiZ)yq&6I~v?#Pqmx-Mm&lU2d1_$=aJ&gdaS^+;EaOGvN^ z4@N1HsUa|(aPoBQ?GbvU)=7hdF-4EW`Mjq`!Upo>++TaKu-Ye}wm7vpX|(&LuG;<0 zF_Dd1#oxcyuy1}uQrF?ymRtR!$(?yc%~7HA3SZ>pYE^cK|BR5J!q9m;i~Ndv3u78* z_-A|PwqzD8D-X;a>T2&kOHD&Nx}K_M(ppT;Ne(;zFzKgY&IV^q(0 z^VZld>xsYMqu>MdDgW0R1$SMILggsGMsruxIK4~S97J^=hxxD-pIW3q{R1+@VW@XT ziU&|F)j-lx83Ft^EtRj=e9t{Ms`+%xN$DC@yeK!`Hz3L1(G>J0#R4=xEE^ptQ$5M^ z9vQ8B8t6OW{L9NQ9%^2Z8;JZ9@KpuHTr%XfkW18Vp?q}#jSB-%mWl=t9(RXyT@YRR zMYjvlqx_w*ANWlAI`{tRR*Y!hhGnXdM;0Hl^Ump#ULQ^+X#p#uk~Au=CmXe~IGHb84hM?QZwY(C_5EAm{- z&1Xg=8>%Ce>MEIfmbN}gO4HEJO7f0zDsu+#rz^^Y5}<{S^F|~At^kA*%vrP%imKL? z9b*nQOT(R5dU;Fuygl{BOUuIa^~1`R6c??kOK}?oBIK;t#Nt2$qu`>1?0G59cb=!Z zLs8-4l28Nv(30Mw`aSa^#vBvsvlEMgjSPZ{lCm2U90``$fCgp?17?jVl_<{kSl{WMTKHS!SY$blUWolp$=SwW0(t)DDK=< z9{o=}l`KcZmxXm$UnMMKnlnatB#g-nXpS!`yuq8bd$kyEM3veYrtKdYCEtzpTb$UA zfMqRz?9QzrQAN^*@VsX1*Gobo_MYLbn_B7X6L{Mt98s-?pN2iRM2OGR11tv5)MjAt zFO4pbUA!j0|7z1{bn}edHNDb?il#%>?v}G(*g_J;SF5|CEZrntA*Z<#Q?+{>2Z>z4 za4C7fl9<%PG2?AxWb5edm!v66)21u1M|@e)fhWn~SDV!C4XAdHX;Uw&6v%j}cCYBb z8r*4_e73X<{4x6Udn34jTU=)}tkdqD1VQrRviWN3Nhm>sm>=n%`%&~JXz0BD`MHs> z?_XCU0^$w&{kAzzESRyr5iy&I+W4CAu-X*79pxNrF!mJv-cE*wDU)mU z)AW1qd*eoqV%&Ov|9U!Z^(c4ifnhl#c6hN%7@Zf{=uE$V$jEr-N7)sR81o?m;ysbm z#D9*MBt>V!E~0Y=_1yxzPo_Uj0Af69JA3Nl5;}ceE9&4?ralv@RdaetojK>1UT;^Q zTb7)%>}>b4mpjg)2MH35`AgAt$qkdh#DkU1=Q|Oy%n1Seu=LgXB%RIhx=6lwt_1YY2AmHAcJ6bX|#s zg{390FHm?org2K1W~R|o!Nc?|=A59~JZhSy5I<>tU3~3z@-BIGRQ$epez1Ra4k;6# zox5v(WK7FJPS8xyqs||XDh~=QjT{P}y(}kZX>}Oz!T*dZxX-}Hsn0tQ-XY|GgogM6 zz|2~vwlV-lu>{TsM#bVzU?w!q#F|7h-w2_+=0WgQy1~597%f(ab}=>0HVsH}6Wtz$ z%uDh$>&Y$6=TpiU6tf=**2}$ul1+|g%OVa-cRYY{RRLAPoUyx|Qq59UBuBSS0un?D z{s%(#WzGBKD#cklEAN}cSzAU?vd;n~dj>_xcFtJLHG>H_{SSaO9w$Vm*oYmKq6o-c zvXlxvkzu63WPLwf(hoMLpS zM5Z%lZOtMFJPRaDY8G_1Oy7n8khm#3LyzDvS`8+QWIu$bj&UPUYglr|!=}UoUr9EJ z0Zp-7j9r|yDA49oVvdd`A5DP|$ATvHn)Uw$51M;TLa8_+KF+jd9x@QVG?_vH=|@i{ zYd1pACA2}PS2}ML`5yZ6DJr-p4iAZfd{`@>IN9TfH7;R3pioe&NuaB~+6ol-) z0VPO4mfeEWMWr^*G?m)aMS{FZJbyB_d+#dm^LS$Wo>{Z@btJvw8dn&c)0x8U9A9*8 z82}gKANNQq4@+N&Sk|3STZWq>Bb$d?1dltPrZ=X$#rIUvAGckpx$i2K)oX!D^&FEY zVbV+uF;hb}Q;BN-uQLU|=f)01HLNcxTHg>o<`9zNlUN(WC5$$nY0qC(6LG*PGTYxT zJJK1n>RY?LHaHkxxDPbGvE?yNVe@9tA6K6&n|jqLraTsNz55a9V?tpAJ;;&TuEqvH zHZgE;bTlv|YJ*9&no=y|1QLycc`=nmmwW2YPMaq?KQ+$}u;Zf>zNAN!$$edavaRLJ z0)@@(bt&^>q84UxYd{{n?8Up^C`M4DOKp`{5XP|QyPOwLCx-Z7Sx=*ybzti?mRutr-32ud_;pV7S}A=BVn3(`zrUY_E?@JK z#6t5c)vVEg=F;d-;iCK{rr-$FIuQUR0~t^K0_K=C0Cv!f4u_GJ8<@&baKO?ESm1(j zC#*F!&{+8e4SG@hHSU6#c#AV$K{~|@`EpLUt4StTUy@thM*B@HkJCN`jjEq~GD7nq zr@~ki$JvY$xX8Yvnu0 z)sHpLl65a~8Vzh(0IF+95}XU;Lqk5|0s@zfLiRI!`2p{VC?$JO>c5XnbWiKtMtI^rmhyDZGwihdiD754oMNWq=*3aJF!NJSX)z#C}(idqAmc+>m^*SmQ z(a{W@8l9Q$zsbsL#w;)`8kz=fj5XsD%<|Xaa+31#o)3q}?+9TEaqbZI-Lk_vqxcBH2$8Q2Zod;Qm) z5x$-1v>)ewf&CdIUmW~!@pZaC&U>cpkGSmK{*3d1gM-`vrSjV)KYg#1v>$>p0eeN? zm1fxZvTBv&R2Od#PtTwrrEg%MW0>|`sawT5G*n+-A@Pp;-&NW`?fGRY)lc3<;dc`* zRioVTpSqV^4Q!N8>8ZJM>nH9jFDntV`P0gcpvzQQo-82M%Ffs*HB}Z6sGpUQk)E5@ zpJNN{Atn4%ZBcIuGqV`pkkp_PYR#13I_fv?MO;~A!WOB^4B(SRg=kW{r=_m6cYuU9 zpY6{kI*xLfUd7mN-_I;WxU*?l)ZftLqmL?0z$ji=5@6WXZ^OjyaJe+oAhCskBmOb#& zc%lE>ORptCwNLb>rS_d{+P%HLFWLo7EyQ1M+ueO`SxVZHqqSkBnUimQpvf;`ZkmTr zetXV#)bXJV`i>CqU$QtizA@i#v2SjuTa2HbIO3JvlwPnDtzW0TwtY~Nw|}y~l~8y3 zv>^(oEzx0ZvKrAAR870aSs3}XhP$e!9_7?lfbW=gzfK0EB6UiKUwspj$S_TN6${alz)Y$}RH}^az>whH%6n zwdpbWt6I1<+Lv$&^4jru^-~>Yo)+dFW{aL$9Dk=Xq32Aep|zozi%Hw@t~k64MMsE) z-eV2BhV})rFm!KKIku`)hBBL0dW`QohUp) zX6uX~J+4+6T+o)D)m9Ls&>U3+6|@UQQ=4v}`v$N;h9au_GBf+CBPQO7oVy~2+F?}h zM*IVraW~+5wxpJhiG`lr63GC(+Jc3rSv;L8HptMkRkC;~WIXNq#yv(j_mZ~!U}EGE z;b$eT+%LXDY8v;83bKUvYhIf`PCF*Y+*t}s;5CtA95IiBV1)Isz-dM&B3m>TurW{@ z$+Zvx6ztKGWdiD|lvslSI+oE9r^316&QbQ(a$8d?KMzCk{pG*5Yoe3j`?3-2x#ZY!d+d@=?)oEgWdRJ;SPRw^SR6fEreHW>cOkN`XS zeHAUn0gGpR@TV2J+5pO$J2H)+7>PX$V6mTv<`48b)(+%kjq&HbP^-X??qv;p`-^HH)4Y_ei+(o)3(#74Rli2?x zPm{o`&`#w{b`H{H3uF0LM zM_>P0n>uQPB9i;pEJ|!`TO|$^xO)zVW{b}z)x~IS+)1Oi^)l}Q<2Ny}W8bXGJ?)8) zd!^5g3NDOubBilfyt~FH%qJ?w!fR2xPq>d*72}(us%!EoN>vZ1t)H*CHKQ}%3wAJh zMl7f1acuW#H5Whh#)9GDIQ*!&r9RyqORZyi1GSmGkh#yr(@%j4R>MXl>o>yTuj+`J{$8 zIE19i7q7wkMaAkCvN*;!*M}`+*C5px;ZR6*qxSgMUsKyXV&UP1#AD0l>cPQ?rbEr* zr}#aBn(kkJP0e|?)W&q;K|g{M#C#%3wAlq$j)g{H9jVes`8aUzl|k-$dH{rRGSw!C z(WG%{5hWT#BBQOlwWo3lb=~ueJi=QNCcef85b=dmhLaTZXt#-CsJ(qv+yBLN!b_eL z`FZv}ank!3PJJH*-iM}1z$ftk`##jCGEU!z7cZVt{uJkmJgdEZ&9?uOgN2hV5eLw- z^QO!lb_k*iHWtRb+FK^`B6M3Qmt$Sd7g%en0cj#XaM_{gK=LEm1MsTUt~)mG%;tv3 zlKpS5JpT6qa-Eb6byjSu4R=ebQeVEl=}2PNp=<58AN%6zxl2AefZpc0Yfd#x6m9FR z2+>S%fPX`raUF|0Pz@Wiur@bRs)5AdLFi&mn+1(5Fu66YL<&wcI*lZ%H=*jO8--D4 z(>k+gVU&C3q6?$zhbjlBbUoD8L*;Dk%0MSgLUX8kAe(o~lOdV)&YL33mhyC7uJq>ofXl)UfHiF=m#$)!`Di@(bx-fxSaW)sh=C;mjd^Y{R9-!m&ui!YNMXI7jc z9cNaZAv465Gb`}?i6-IXU6p(Slw*Bafh-6D2zcc3G6-HOKr-B|RWa@=+YdE=JV`hw zazgI4ntatx=QhzLx4kTAN0_B4^<+~w0DwzV6?FQq3FS(iV*i4D~KmA$UnZB*0T*};cW-;RdPvwmc~J0 z*T$ggq6}E-oibS(f<<(iQU?Pv?KjJ5zwy(4n||l?-=^O={VAHI-Q5k1-CaS^QGr~E z8b7PM17jis$E3^nG3e&KA2ru?cSimZ$DLL-=P|OG}w=Qj&N>`-A3l z-4DOi{pqUq2jM&2pB}ge>lGl*BW?0qkT?{pissKcqs9u+g{%l$(4MCXkTOv*a%*pE zrFRtiop{@#QIcGR4k&yKiLtd*YEEo6TwcF#lhN{e@_{iayzO8tUX;&j;7D5Lo%RM! zyq-noHk$8C<8L;Y?@v!O5ua7?CS>M>&sL+Ajf3XMQkCFlppScGjJW zd7OA9hoUQ8K&JikN_)Xa_A&5ORI(8gM$(B=W+tq;dlkAR2o6{?LzTpo5K>zy3y2Je zq@Oz?9uzLiU%}_Tl|CmMx%(ITIey*b=ZNletF)h+^!Tcww>rukQjHu89ige%+8P@h zNVP@KQQo0+|NrUgRp0iJT)q5UDH4GG?E=korPwUU{~?@Q#w*Y@$HELsx}>4sgDj*I zAn0%mDK+V21_#p3iEk!di^Sb=Q#zkNbse40zgW`y2mW2Lt%>Xt-+8bBI;ep5=ksL# zTRr$zB4ysGRx&=6aG~A~kSuU8pxvipGTlOg#*pSh-G34P8yJ@RkmPrWFNh!guf~0o z0I_6)xMd+(!WI0427Z$`v7GD}*3%1h&9oyIY6f99^2lFgjkB?VjCEng03F9y!E5ZY zCVgBf19a$@q85w?a$yfXo+0m>_6&uZHC}t*8GOdrB_zJd`$T{8zIa1Qf5S5_h@T28 zv9EHREXWohy;xdI!2x1Tys(079+tqXkTUzE+9<$?Gkl;Y-rxSo;gZC@^SxYYXQm5@ zY&wKy?5v_=;@gp;{qJvD_vW^|y>Tsjs}~$@jixgf|G>Wt%g72@oHWyj5JOuB)RDpl zqmW=UB?N8+I1xZV@Zllj4wc#}>c@i@5-W0<=S&`keEtjj5-W6xgQ9P(zCqIWXhjdt?rZ zdNOUqgNh^}9(Vmsx=Zf#kh?C32joD>5m3yEzv5Yf=Ut8VT(4!H2KGE1gRf_w=FK6| zq%lCp$Fci&wLU`X(+^3t1tz+ok$}EzhJd>@1Ru~->q((8Dg4c3VF+?tDQzI@gvQ*c z&mFm!${6B#9Ayr^`}cERUm)es#+h8y?JG9-_q5^%+~ys zJA7S8*E}Ykb|%mOf;6ABb5F|84GmEbCf<}=ODkZWo;ZtF1#X8-zG}+*P8}7&{VCP; z4sK$Jt4tcAU<2@S0S;l{gPnp_Y=oh`O?Jq0c z_p@dH+<*64YwI=5-e)%c@l5^k?A1@tdrtkzuF{!%Zm9oLxou%gT=UkMU#|Ye{!-Wn zi;x$81c;mW-UyyF`bNy$V~!KN7IFn(DNTDJs7I4m{>%FaD8ai>7dUfCUfz^lX3bk&ud>%!HcZC21*L2ddf=TZ)5J=qB4%d!##=N`QEr%m1*`o2WgP zzhUQ+P)Nf`IpL4H6zufYNOy66y#Fpp=16fwv5%czeehcBx4yBl9x%q`W=50jj^_XA zcpaHm>BN6Ck)x>BRy)0e@=UO}@9ryk;$^Pl%W_*5g+Z@W${)ZRj1A;a1(ZrNfu&F} zqSBr+kTK02y_adpeW6Jt7GvbQ(X}Wf5)#1DZr+!4iB%Z{PlH&8T>M-% zPg#{_tV*4bh`WB&u3fTpmm?rcwepXjtkGiqK&U>+^9BaiKt;9|fUf2!Fgd`g(Mi z(0Dg^BcTOrwkQs?Wo~a>?#9DvgrU(2SH{yfNcF)eSwnsB$4e%5j*pYq)U-JGUcs6e zhY=8+y8D&<8Y+PFfpmoYCb=^q&dv%aCj&2(f`%CAoJ}-@11>0O8g$8N7u1Kf2)XC! zpm{nJa3f_Xa#7{_i`D$;{eK!RE#H4rz46V>SrgH`zW2KpLbtgpIx-^^qk&P9g!2i;W~Mjgod1y)&lS?a>aq)eD%0Q~|yVGnP17uU- zgxcEZOXXLzCY<4PB0 zI+KW|!TBt!h=R^M%7 ztxxmk4WNJ$`)1-dn@PtI zGW*a1st+O{Le(VAzLCb)4n%ESH_vj9ZG}3cX>~m1x}lRg( zA)R&}2>yZ&WZ*B~Nq6zj!7JWwzqt8{n8Jp^XfksH zQ#A^dB?dwl@xP9^XO#S73>_0(qMS`lvy9QTK{TbvF9E1|m zDcL#hwiM3PouGlF1P4O(lj%bV8TQ($PR;^+$tGknCWugaga5^hH3#CLp zA+%>QW5_h9uALQiG*1>2-ao4aoanHUEY<)ZW%tM*m*uU$*jD+}W2K`x{S^(5&S{#wlbdS*NOD>MV?uX0oNS?EvnI{y^A?vH08$q~9mtsTn1c%p&0{L)wv$r?{%$WX^XU=1L`lAMYf4OZL|qk8>t)DeD=gA6jy>@JY2LU zFKF(e2Ag`6?b zNYjXLp)KvWT(VtU+%}MuG|)!suWLS&ur0WMY~P+gFOnmArnaVI7OrUb(EHNECN{DC zaY8bdm8!}r%T$q_-O)R))qxw&zx$PQ>;V4*)_EAOH*q|=eQ{m(DNDS4QzFaG#pqz!-3_{g-jQAHuWw+(4 z5M_6H-qD}q3OAgtYkRy(hoxGz`e=Pk-?hhbq$;2d^J0tpXZ#~(VQFakl7r%ZgqQE6 zrPfE2*ycTR=D#_=;mw6xf4(V`%YPEP5)FNXqNKP1(+?8SSEWMq%OpHQUmb63!jwR~ zBui;z5>b~v0}cWCE!l{?5~Tnzh%Lc6;2&bp;H=K~{`V!X+}Newn(C3axKuny7A_rK zmF5y((S^pYh+EiwmDF8qvEoFdzh_2pVr?oS{T?n?90E8lZ{^Y13-*_B%ZB6B6>&8g zp2Jbi2N%RPmZgQ685P?`+QE(sxt(3%mOV z;VZ}pXH<$o^GZ%ia>T0|`}=!D7^3UAR#1e8u$PtOKbjg%dQVl7@01o5)07f}-W(R*Rw3S25Tng6&-c!4%i`p6-ouR45Z{>CiUc=osN3@nfjCON z`(|b|Wqb3XwOO89kED^Y1HSoT5hWq}3wA9CpP3qJj=&1Qt-mI!aN<|co>_RJu<%$( z*Qw4J;t)SKtspeht1Y8v257fTe4nRToUZWBQp8k~__&lnBOGXVH#b6H?c?33fs5b{ z)p2w?N;C9y7Pqvccu&T98N#16Vrp!E;lWifgFK3C`}kPjj@0Ch9pX2{@`>MeJI7UY zX7YMU-of77#QfydB4Zc(Yc7t8)FqsKO<9Xupe?s-D6yxkq$hQ#_Qg%H4Kvfj%!~{P zJg-7W-Kq9Xensi__D(f(90rQ}(#4wnbKcC%3!KvsJw$b-W#TPZ3KZil0@yythK+S| zb;S-Q2N-B~kD>qK-KdcON5SOQ#pn{Duy1tt9`aTwuGhWQTjCGzRswvqwNm^7$}L4f z!Omq1`htz6v9I@*BWVT>wl6!{$U|$%=X1(h@mgdQyA>W;5_Etnrx7#LLd}SZgKf&S z<*qJmuuF(Cx3tL4GOLR&^#9lNx5`34qHgRKtWZMQ zANo@F*Jn`WNfO#q7z;HNw%m`6!u?LOj+;Gh8w?S12PX8aPweajg2GI8Q)n zt+;b0q$t)UcHyq-dyFIh7;^IvJ>Nck#dEso&#ki>uMt&T3(`YaA~DmJl*>O(4PhWVeyu3Ps?vFR%REvo?Cs6m4b`v-c$%uJP|K0I$t(ih%wR@Q zHzRc?P*RYlrd>}(Fvri;-1HseE{@p-rl?|P(e?Ad`hnwQ_8urNSyL0i&4;V-eP$YH z7cJJ6giT*H;#;*HTEY@&Z^z&@3!w253qu6ry`5QWCoB?`gsiWVwx6ezwuKnHG%d(% zt>ojl(Y5a#Dl0qm-rB{_r6;8Bo;7FZ{K&}pJLlA{%j7s`+bfnTj@$IH_1vdE-#`=_ zK7VR%V8B*)&)n4~YsLT6J+&qmOv#;01BTwa};y!32${r14X;APnb z?OAS~dA;k2|BT4k6>;%>2^~ALXARzH_iw*(V3vpLE=QNl?k!pV*;~3ZA#J=MS95W$ zX~zcC?O0hEIn&G%t$$rhR@l7h8+T7P5RTvJCm)U5;J{|Jbi@Lh*M;5C)f_zny&KZ?KJ@MzFOODlnHw6>xN}bXvFhu!Puy%; z{j0+j`8$42GGAWy>h^-7J#Vev{oU^Ca}L$5I$EGAAE+7S&6c^Q&dZ2v$n^!lRoBSE zu(*yVn&xb;53h)uKQN=|P)k(f3#)QFQ-F>YJv-eKT6L&6L;ft&s$m~f?O8xP1=je9GTHud+rMFemBO;wxfBac*-^e9%se6Yc!IiFOfnKkTUuLKs=9umWm<-g!AdVQE)%2Co8qP;2W3zoM9y;BX;qS<;3T zL3!z0OIKnhp@I#`giC^vmuX)8WcjFA>FY#C0eAy3}m$l63 zqM01cyh6fhZ<{Hz11$OE>uD!SK9?r^!&9l-)`sTg?5jL|?RqZj6zg-$@3KQ@KiHOduEpic%BG2#{g5%aQHh}B1GKW#{L3q$$@!W&pXoG{-~ z07L50`MT(H*t3R5x#9^={*wt$`MLj`Q0YU>tt)YUHHKK85Z>ZnZY*`yJ zeEd=^>nbb5n+ppw{rnQbjcrZPCSHyrD(J!N3%*MTGmLndeNVT{6ABfov8mgek|s5A zVDF$}o5HDp7d`qO`A@iqpto@*dVB%QHfO`!xf}Dgdgew)=XiML#6;zIa?SWt)vAn) zm9y^u6PcO0y0UgdE@?>kfIQJ&#M3OH@@)Bj&wmysk2>nkaOo>KzvVM zW`VqUIT9%J(ANmrX%bSsDvIz(5s%{B#XIMSs*lKbh#&GVh>M6Z84&x#4zj@*&#A_9 zOtsGe&(YJPOhgKm^buQYf+{U*Wi6WbNj}F*9LAMs?l5;%eomp}04>9= z)Zm^|>`H?6N+qs5BOk*(@TjnR67Z>$?4Bg;m15lU1bs4xhzq+XpWDanXIH={fL{eP zNNy+hrM!gkSDI1iCEJEg(Zb?fN!`Dom6f<52pA|9^5vnlsZ!(%D;ruTAHnp6CX>(+gPkKn20WG5L> z6iU3yTtP8BrPL3QIE^SU|2%4A`0~1emd&7b=Ian~O2RaXZr zTOU`Kx-7D79DIY$6wf^J;4`&6Uh8F4#kla~2z{tuk@Qf%hN@Wp2lJx9`e! z%Xx=;%AI;_=O1_j3n*L8ZRLKHkAdRs(V5!@DHDdAIW=ihq*w{%h7p!hpIjVG{H8mu zUZ>Yu1pDMIOfBwBJ?)him{ja7mua^0ZsJx_zM~~RWmCH-kdN~!ywes@2~a9-;D!lk zf>MdN3s+Vk&;9S@B(8jhE4y(8neW^gavoQHr@fLQZsZQrD_orRN;K}dN^2IIOes%@4#jccyPjg?A zR?O#dnQX!VGc39L7I#WkgeylTuOKf&_A;&v(<^lMHGd`}ToYpRklto2v$Ri=-fZWQ zPCI4iBa406#=Dpgf3?@S$D{Y`O3Q)gI(<5AYIj_tv{a(`iX72C%}mNu)yhD&=jik> zb_7WoOjxJ=Qhbu3(qvK>jr|ygZ8R?j!R^HS=)yhJov7 zYav(bLpB&mRj`tWmeP3xycopN28sj*9V6@JYat(1k1n~EY?g`o**GK|el1hs)Nogs^e#aYBnonbJQ6k>P6p%}or0w{o{M8H2|BRqufOi;^Y`!T86PFfPrOxI z`xYOodGY4xWCxS5Om!v~HB`Paf5u}Kx4)F1gO9iWXFZ%8yjgkY&6rpVeST1B9F4{% zpunf^{#DPM>CDlP=StZup#t9(bk*cMu%a&bcMwHKctGN2Yz-~Zn~x`!C=9|E1nTe;&&%JsT_%6_b{W~4aejTq^Wv|s zM6D3tiesVr_<8vweyMu$`$-zO*M)CpwfvB4BL*i1P2yF`mn4j>!rh3#w zfauMCMBW4ELGrNt3FhzcfWN>LuoUK2dRBVo1_1DsEUFX4AL=JH_jG4& zbE-a2Gu?#S#)Xvh&j}rKiYbW_Um4?H)ck6cBSP+$wf?av$2L7VWyZG^IDmDy%EMWbK^Lu$r}MjWPF6jqxT6v!f@qGKy=1ATar{d1amC~t@L#E;l05r--totTo-IZrMQM@oKe9Q|9HgLBYgacnASR&GvIB?z1T%9%EohGq z=EWp3r8lM@E8SzDt4(n#N&z4&EwQ%5ANe1HC+dSanda~0U#(lS)3>X||Ac(_+!tMQ z-+P|?^`&5qRj^P1qy2fVb!E=-f{L|y!!57xSU$c;^QsX84HsnYcfS!F+5DtDDRhVu z1m2Vw8uLh^!zsWK0O?~j2n-m+?x32gVra+?1&hE3B~j$P>s-#uWrsf45HdIPYW?2% zk=tl90!^u87ih|=o)jl8*Bk2X&f4_G0Jrplcya^8G9qi({NXz*}qcsqO6$6S|L+F~WERK`22iyq?CE z`>JDGM#@&Tc=9^uaN_s9y}v)N@#GSYWcSa?SyxV5D|WxOcH3($n<+lKfGz-kk*|l1 z;12I)7OII65lB~s=H$eM8mnF9CV6=_COA#hOM+Gx+HY8U*1sAC0qg*+`Y+WBOm?^U zDRzM+)Kbo6Yx;^=vsPz8i=VYJV{2wkNQ$dlYDi8xF{9=HQ|#~8`QMUMS(mJyteKli zsdk^;lO0tSQ5HgV`&k;_pQ5)D8inQC6qe{NV0GEv&W?B)L2y%@kFr=@irCb?L%e6y zy}&TGp8E8-x0;6Lhx;NFkuTw!ax}IHFW)?z+V@lw_tore zjgbLmJ^6kMKm6H9-MQY;UsMP9CHh!8#?Fk@*k-jC`eHwhK{(u!UkAUS9ooaNALap} z!TucYXyFB9ywpifGo5Vgt^njz2fgDtHG%Vy&1 zq4FnHrAdj!D&OLy#1fTw^)1cWvA4J~E&$T(xAPVruB<$~aNfd0m6eBnHZQ3rBBCZ~ zUSdsTWDPNbWL|V<5B#}1g>I^wHX|Mz3pzG}CIy5LeE|s^21w>c4_)|_0a+^rUYIbG z(8kSq2Iw6D!37(h%4W|f{?o0I+p4>E?d;NA*f(b{KS;KsMs$ma5Q?ZKJ2Zdh>`TR1 z@_H#o9D*F!3%{o`G$Y$o0}oqU!No-B2(z3nly)<9%%4>y(cEz>76DTF=5TT$tMUI9K9 z%JL@`-2GlrCVyT~K~whO_R)LE33?CieV%y+=Hre$`ElHlAk*V^vOA~~h~9CUJq@K< z+NV`xWDUDF3Am`zy>8ljYp0HkRcnP;G6IO~e(*{n+sj`jvxstqkKF?OK3QQLI|OO) z4hJLEle%n@kr&~@(R>LhNP)wj4B%j`VI<{D1oK={&B=JZ(409faZ$B%Ld?7?)g_D0 z4O9jC&s;K8qrSK*EoIrW{e|6mUS7G~1x4NYUUHx84qhzGsM^FR)0Hj$Q+S5S804AI=-`yzvIf9Tr_cd3Iw(oK zEUk7xtZ6C+vY>=#?tH=jV#0(IHA#(Jz2<1+lEFcR&*0$1Y5R!_JGi&GU+=`sPH|iL zzdYz?SZJ67hpDcDtB2)qMGH-t-_mUIO~g|3�PJ}HG?Q&#wR5zLv9qnRi zs4mLL2$16TED;l5rbq5sR<90pk4NMl7fd+0Hp7EgMI{TA2$WKC49X#_U2q|po0rvp zp`-oDzLGFwezAfP*ugo{To#NstpG=R#Y9jxyi9~=c444 z#`anVfBU)Z^HNe4?JUvWcYxiuUwdE4&P6FH^V;Xy`#aROH>M;n+6ij8dsdvm?Ld{D ztIX`OqZL6(^moV=VKwQLCoUj%f)W2ctn~<)BdYv`egpH= zFqeL>01%qPC*cmCN-e<4;Ps7&wgHwn6fCVRLRDH11^>vuEF>Wt(n{u}Bvj<5tqdU{ zE7S5T5>jeP(gs2w`4{f>mm(6l05=!k%*2->;#Dqg0bHi&i67}jRc7Kfb}^7c#tknU zl0DA6bxVV&GE4l!W1M`oAcPNVQx zRBvO0YWd)uZRFK$Najl+Xt6`aDes(nx0cM3{T-f5g9}1GKyYczDD_lm0}!)v@>{&U z|6W{tDJd^*X?7{KW#Nkv{;jZv4?CaA6j?rp_s7VPH2m8*S!S%h!7>#phR6L@&Bs|kQQfA>~y3xWiq&odhx%`9MzC!$t9f)CJTtx)N zr4fH0nLo5g)c$!o5p}S?m9_M$LAJKi1gHeT9^l`USxmAM`qJnR^N5ug8Lc?F$Suv) zz&gqyp(5~G%`;evx(mzmw&b}5Ezg-;96D1yy+OHviK^M^JY>1X#d8wzXGHbAEK zqEolB3ewKqPdjzGcoN+rUvzQ>A3pbouIsef|3h;Ivw!qbPk(_&Ek4e05_G-8Sbd7P zOg%qXdk-MLTcS|R$;sFl{hjUZg?1@RVib@@R~n2QwwjiegVwO_5BL3$b>bDGnf*!4 zs7rTHLaG1g%t7v^=JKs6zMrh_OiB*qqv+sYE;F+Vp9>mcG$VRH=>8F?r$SMouwXW2 zm~5deEY&d~JavBk978q9%1c$MeS!?tE@rSZtqimss@eL($VF`ZZi`i>eiR^4VJl$`c~yu z)wuY$a7Q%bS4TZ!Tx=taiA_#89A09KXIoqK?gi0tE&JxId$IBVG56+iRbAH}_}%B6 zdoPm=GRdSO0y2ZlAoDzffFn2pDxeI4Gft?9voX#_j2g{sCXIrbY^Ek@o8+Zw``V^y z`b&Cm=TqTIvpyY@c!0BTy_@ALcP_xh4>VcoUYUVH7e*R`E?Ok{}H(eC$+N+Omr9qON<=bC;&h%BKz(U+&vCYw&1#o{x=! z_*llCmG)EelHFb7iW_n=J7-O;J6<1An-ot+_Ls*P$oJEwGZ;FNp= zcHLaH)BL6JB4nCwnUEY%KLUyYMd5r+#Jd0)xc^gtYR;92SDi3CVlY(05Ofdl4Ocfmh!J>p&Q5B-jQw*F23J$`3~Nb&R@72tw(fxI*h(HZ{S zgf$d`^V)23!`lpi;D5spZk}!)c&`?kl`6-;eR7@g1z1v)NP|&L_Vp8EEiZEBVJ8S@zYz4|$Gqkoil+uGVJ(QrB32mLyf8v219Aeb9_J32bd z_qY9}uBCF;^7^FckOa$*j<8j!_`mx3FJ#Y;os<>D6v@G{KYC|eY;4?WcGS_q9F{VE znYwLL(~>Qlo0e>@E6&O+F3HL)Qto4ZU0v$`eE$cQ`gTXh+bXW&cbH~&-R%1Qd+D1S zH*U=9kTSmiUK!Wb)iwC#;2>Lb$<94vSjDPCn7vz@*vsSJ?qJMdjP$$Mbf3U2(>EXO*_#-h85dXP?VnPvUVH2@ zOQE9Jolq2FbGJ?G1^udlr;GA>$ohelE`2tbLoL}LV6S{`0Ts`_!uV{BqX zoM&cgNkL!Xj(Kx7=SIy)eQNXOR^}9%9q$pHUYcCEsvu=qQG8up{6UPg9!K+6vOD6# z0h#U!Y9)~~-ei!RV9Gyb7=ju^oZ_eT9h!HN1!ZQ2RrJ*ZzhaB%X$Wa{@|G^LJFABN zF7<(8$^bh&=ZQow464^~81eiQR{_?j0m28GYP9BfrM{wVvu1BEF4{h8_O{}_iD}`u z4KOhyJbc;&_TKK!!$n1hJH>z0e_Y6^4GF2uxsWp>G;{{p6*Gpumrp9EP|F98+->~$ zAQofi5VUcmM?SL1UOb2cKZ5dXoG#8ne$^1a2tX(9O;0pM1ZCGo7cEVk+EkiQmlq<( znW_)9Y<^_=f`RSJPcNOCy7%t`uJ@-l@2h^hptUG4uQNBZGe01$G^8PAZPoJUmbPDQ zE86k$0DZ^hgWp)%Ig$Q&n!nrV{**K2`FB$h7UyKlXp)N zCEwfyvhqtyiMv|6D`|^wzUA-o#jKUbXRm&xJ-y>ZUEPV583WT=3JRN3Qd)`%no}Q0 zY2IHu?H?EN)D_*@nu*=VnStjc%_h7`4wd^XC(lc6Oj%vE{Fx=~mzoQE9;&a|@z>{?K;IV7_d4bjK$7KeYb$xV z*c+uDeX%gPU{`+>Dh}^K5-2Krqq9yEbC!H47pa852l=lkT=(pvIsN;audKFvcFdMV zki2h+bHn z-Iy}4tbfhi!>uWi3y&^6b!zR|`5~EIC+F54Tb0|N+EQ53k{p=7C{21GYI=NPWq3$Q zZ2f`hp*8EPVw-AW!&`P!I0m<9LD4RguUBF zc|5{JSQvUCh`fpsI?_rv$qP!6nf%URAA6#-HlTg+toHbUC9myS|LpwWyt;Vx>ak{R~JnkkhniHowE9&M;Ox;Sx3LsOFazoco7gL!?*0m!)BbEY1NBpf0Dm1v9)l==qL z%j~bpft&5}Q;Yf#Sd$PN`J4PWXmp3Y;CR{BWn(aWSla1aE0Xr%72YD)1+56ijSO+g zkEbp_xnTauwrQztC+07BU`6V{_|#b`s41N=VcM*ul$oi1(&>x6*Egl7Z@S)lvGkn9dy7xB5;G5}yt_!%8+hazgl+yW} zb6O0DxJS{a*Z}+_|6%ZgoHh8bHL~5RqQU<@4cAZ8>yCuv;6sN>e^FnJA+K*VyI3$C(HuuVXqrMepb zI%xTW1j%HuwhYQtj6kw)XSP$AVr!HQz>9i%!doOE2?@mbe`e48*o2P|zCHet`U~|} zzhy^M$h=y{p0m7Uxy-Jqc~Xgl!&RX@W&9dg(d!7cg6I*M={Qz#W{!#jvr$Z}2R)5M zA7ee(;0kv-Ko)sV%!L0G%-;IfmPqxxjc48r-GsZ2Ij>(;f3fM0-=%L<2cwvWy!xMm z&C1@v6Y78C-w((Qv;zlss9(rO{t5X&gjT81c;|H4EQ4F9OQB&QAfoFqwO=e}nQy3x z`_zwFdeuG^eJ|u@j_0WQTYMQD>K^)=Vvir)4NjLGTug=@L?tssr+%^_$pjqv9!$Jk zHPG|^#@=_=T*=!oy?T8vX!>s3b93fAw@r2!{BA~De0&>DRtfQ1gKPm_$|$-#PlarT z9+(3eTF(UtK9pa)wBo8%Fc_}vC87`$WT*!FoBTV-eKN9Ic6LEd2{B>g+?=3VeiF+s zc9D|-VgF&bK^}3mohzeTnNGol3^FtBM9^Lf>r_LK8V;jc#recWDQ3Wwf@(7ydp|%Mt2UYnnxl5 z*XEe}F7{aSh0fHxEwdAmKwzP@?a2u%=lpFz`cVD(t1MdC8#QO!j2YYKj`KMdKQ}pJ zUi_c~RFVo^0xr)PN>FBjZ*)UEiuNetUSVN&cG5)mjo70?*^6c;4uUVvUuG z`XRDBX{w!r3nd3k1ft!e0*6#!ZIj9W(4V9}we4{ib64hK(kt~zU%RL93Qf9rO)adM zwZ-LVi7PF+n+c2g&c=Mv&)|94#>mW8nIKWtGhNE;e#=8>NLojGL2ey9gpGAbZpC_n zU&b}~z6n|kkK1M1N{)6$2ed&XHHM65Y%EC@^cbngSX?ilMmH`$xybHeJ1(kD_gH>* z%IB(5(bo@*9rAwKX%?fpg2rJ7!7JdGcLP7|RHw@(SyFmPV0hsM9fovaLEdfI6VXrl z`|tF9L6tI-|3qhkCv`%yH#*pu?3f@-=vO-6{E>)HXl;!)nQTk5jsTpz*2p zRo7Yvl9uF7n;Xlj<&?oU+D_M&oLH4Nb4tuy;6sp=RS9~Z6`F{1--T6yFC*qeD@B~Z zi>v4|CCfN4Y?K~2BqTc`Ne6>TZx#d7bbesJ{Ih;(_wnPBM2I*tjMxAfOn?lA(wc>Y zh8n$`H%=zcMi+ILQCo0yCq#;PB*y#4D+yl9F7N5OzPBv5_r*1bUSIBX#XhsUZu+{a z;OM!VDmHYE!^4kgwLP&ssbv4V+iH4$cA#Ybg_g8M$q7pj&z}LiDIwnD%L8J?;Okgt z=bed;`*^MnH`j+1^RRMem?Qruw_mz$`H9qP>Cu*I=-4rdK~)1|?vSheJ0p3$WT+Di5!ooNtpeuPKaqEIr$vy6|FGZBxYL!ey{MU_HqPwe@u1abDzXc-@YRo1Lu- zHcV{B*kmfM9a*UpNNPg1T2gWheq2mP(;mxM+m#Tj7`3|lBCh9^Ti1WuS=FBJDh%D$N^bh zWbAnmXU!81chhrq#O$HMOB92jk}IbLSA$e;2x`nLw#Ez^{7m(n1Y z=(`xR#4eWX?0aN#DRkAsofvHk%O!)~N^e_MOFtR>78+b~JARz~0R}ifSriy>eE-_x zG`2J1nGl0aRN?>leN*6|(}Yf9`J$x_Evxzml)Y$pV^wsrArslLw=^uyh;)w~V`D4Z znVg*+%{_LKgDfbc)g3OT7^Y9lRLgG-UYExVz9-qL(^^@j`gH4YtnqhfzV)V;<&FL9 z3cIo&bS}hMXu?=jbwc;YY(4Ncd(U`>P)z=(aQ!*W6!zdpJok}!@*g)_DE^^)q@;bM z9kbD0y{6{yTivs}e|)&Q`tXmtyU*3up6j05eQw5#bKTJc%D8E(FLri5y(T?v_0yf5 zPpwWnnYDYxialA`dsnR3nZ=xiu9t`|bdDhF&(DA?J<2v<;*IfjH2C?AA$DT!WP9Pi zkU(*Oi5frr3?r4(#fZn2{@#CHbl>#muEPHCS-rE|H=M0f|B4ez{qghawvzjPelJT> zf821gC2`@k?JS88FB?{2^o`I_lr4!f1ZTQN#Mwp0+D#cdHpInc>eLuJlb`U-X%YB3 za5r+4)9C3QNnQAbKS{L{NS>!l3_B4@&0YRcaMgwxwHvB}C*yt0<&Sht%k6z-{rXq7 z<)(ICXq~$G>Rc4N+@q_*0^&0WKCEBPLG_a8gwxqF)r z&JI2{E`C+5n8PXv%fcvE5*`~7nZ}&9yynUdo z@lto$iq!Ze2kV#2y01AoX*p@PDahyD1eaXkSw&Dy>y(fX-*7(`(KEr<4mWuD=BItD z*(l=jF`ivk5(9aR7o0*y0rs?3rO7+eu_*&?icf6yn(72OBrKi3c$>KMdSi0igGY2ecY;k{9 zQU@2P{-pNW!cCAQyobv*ppc~5mv*ZsS>v3JI0%tq*sY#WaBEAn1Zo*iB+C)jqO0N&Fb6nA0?cb()u+QAAMz>YhkpE<>aTl1$g|c_F1AncW z#yTr(T@l^aN^*JNKggA8b7rgktYBtqY}Zz37xuFy54NQ=U4PEfEB)$tN_}GOHZt=Q z7VW5NJv*08z~jBLAyR!p`3pvc2LcZ_Ak*2~&(GG=&d%4()(#JDb7zEWc+5-O%i{On zDZ7MQ&bPe$J^hNaF*u`QZd9Fd%EXNL0OK_2H+Xy*Pk;9h3|_kibyvLWgHs(9e#e4l zY4+*H_qBW>9fuw%g#Yp(^vHOe*c2zl<%jfi zcf?8U-EWPPj_B!tvLf7;)s!%FFe;d}vy6vHaa3^MI1(XL$@$>EIqk7YU3J;5S^Zfn zPt2KrZe3o^>b}npc0b;hP~7{-{K6C4r}rE4J96i2%nrzF%-$!3cZF3Xpijl*;*^rr zH4%YjYpe4*s>1W4s#j#p++00nV%gS(*_D&StCM3(V#bl)A!%F0IZZwhBGUs;?;SIe2B>rd6VTX%R}m~qvU6} zrg=j724fmIAE{sB7?&|UC>V0Kw~-`%Dv+wtMsMW35@Sbb8|9m^7Y-dpd+SmEQaMvVo-bnj*X#Ef(q1z8g03?>eT zCxmJSLhxm``h*E%>l}jRm^s#E%oz6`1y9O2J3BbIy4rF^@kQcY{Aj%}md-7*%&a4t zy2qYBAd684uhFQtaZTf#EoA(cd{*8h59s~vG}g`q-Zq78v@BIMojkYdX_5W0WtZ79 zuBMmAb_+F)W3cHuNuXLXtfuE(Xn~eSy%TY9iLw_RmM2lY6SF8VjzmCzT@amJI1h^Fqz7hCnY!EH)~1#!6osjE6TbrHI^OV7S#T-re{&(4Y?Zn zo5_PyCf;~P&D&(JjZu#F@C;36p$3F$X*Dj%+_;<2sI*;etL!)KzSTcOWOOY>Mk6C< zQe?C`Ov1Bw>>pe`{rYa)MDP>gFD-H&KszD8peSBG9z9+>+MroY;3$H9Mt=%~)BkhB zor@@HG|1EOmrB&PUuRkD6SddL(Zx-6bd>8?Cnv4wYhv#Vs9$YqL&qGJkN5Znr2Hmu zLt&`q^_;z~sA|*9NIGs?XC>7{kRG+izv$!_Q8AnV)PNe_Fk9VzC^yaMqCMz*D(^qoDl{iO}&$w~!W5+r>IoR5|noM4H zV^EW?AK?7_CT%dhSHTH>Qvf3~>iszC95)3#@`xqlKb__36YQ_DSfdd~$f)z2I_W>5 zl#ieqT152XHjOd&XtdM0r6~~2(ilmd_54!GAAWf7pSPZ2cP4_Qn9fQo|A}9SvteXA z11hko3oMQA$e#!lvZ3FD!q_;CWm*$(2DdDcV_=64AFk+E(k!1*=kc{{yEs7QZp$T+ zU>bFZDtHsqV00fbYuNH))))~hBT{Lo@38iAq7;X_e;^;zI^W6YCtD)fjht}Zayqm{ z6k*he3{jlF8taq?QRhIm24hj<>?a2Vq>LeL0_Ngh)=Mw9xU0hn(SOqy*;sdY@bz7V zaSiv?)g5Szn}6}E`*#2KcvaTghnq4Q@&n_S9ADi2_WmCtMpZwoleMxO+ zKG+h}ePwe_=EnXtDffT%c-{2VpWZ)V$GiKA`l1&fYV=?Eg^TFC%d~YG3&RB|1gd(V zP(pOfN+>NoihzMNcW?J-Pmf{-Fz6Z4z1f37i;pzK9*L^|}Gb(J({)VNyqm#YrcR82s`_Zc2)vxU@E8g|m`W^9$_sqy^O7dwsT$RwUXGU+; zJ&O}!JPtaAOn?P24<~dpXyp+z^ceNBx1T(!R)(C6;iMRLuqdt?KCG!~Mk?I&_P&y` zJuj@-0_x;_mhhs(9mQ*{R7E9wFF2r_D%vn7{+)O7gMv`d*e)kAwRkBDvAn>1w%^J1Nnbl_i?Vl> zn%u==)K}7$^kmJucV_6MvTX~qtAnHJ^D<{g(>Z62AG<^A(W*My2R29eHAh77Hy?3< zfH(Km-mVeQ&Kj3J#=QRCQ2&@7#Gi ztFZ4atMJh#zNQUKJj9czBeJO{%MQ2NxZq5*X*pCjcyIZ!N z@S+uRYnz8|aTxKBG>|$s&L+Y$wh6~Ew0iR7T0^6 z91lA=XKs37{R`_~+MMm;c*xm4V)nL~FU{49p}>j>syX(mJhOi_?F{kEi=qYP_+GY^`fIQW*l8q zy^#KMsXA$PZ0zi$>cm-bakH4ssMIm`8KQ;?d;ILy&?y zW!)(RCtb00Nc#~iqPQDHi$pYTt)-WpKbF~$kldI#hzgR0kF-?X+mJZ>=uwQDfr`$P zsOY4dL|&P8J_ZAx)wXdCpA3%uWGcn|xL>cuan8Wls$ISUuFgAL6T6(!JMxKF@pO7aYHthPmtFNzYaL{;-8~b#=qE{QW_p&xd;=|FN zhfg+w(R5Tvt{(P_cH{(QhlZw4D##7X50ye?RNK{dBo*||nYXhfx#NO7p*MNeoUWwO zprFht=_USgzENSb)AQF9m#r@-+q)of>A6lcM`5UOH%J$Fg|NeOsO;pG$h@u>mJB58 zf_nA>o2RDA?sL$>Z&*|J;^w+)+YTx z9V_nhh}xymf7ciwoi5u^Q?sL@Y)5VFjG6HrMY^bX0DJbZvtn4Z5 z^NSBm%=AkLOw9OINn>nmV@d9!goH&&0Wm&4F#(gN`uI#`Thr@fW9!ok7RAOcs+tht zoe(`f(mRP}Up(}^bQp7q#F~ut@v#eF5n={N$|7FIrC%%b)GCW{Zj7od#?pvP*87eP z8&U$(r-Y{YPe_l5$PQ#t(zK?Wnyyr=N&U9`xYma{oYus(PCuAfI5{*gWJ+exq&V-0 z$eBqwtINwamElHU?6Na$$)IXKZhJc1;kKuPrB}P{d0NSo+HfyRrr0(1udVBtF{pi% zy{Ney6yCU>YH_*5ShIOVd#^<6HifLQi3eZ&?RSm&d5ah4S9p5xbC|f}RpSCXblI%QQJ5g$P%Z%wf{7basgtknq!mrzt;W zSsKwV{nYZZbkMRwk6)=?c(eS^{GRCqwODb%J=3u`+2^7-JvP(3e!i(QFlgL3r_8LZ z$m(h*Cu4DVZ6L1Pp`h;$vrjFahJ)W|MlJc4(JE@Q5qdY(}fjEV%LSX5DnEMo8h z*4lCA=^tFb{QAbt*IUy&PR-x+qvic+D~gMjPfJ@-RJbB7c%Dz@q?CXswq99OuMnX+H~+BorDdzfjt}Cqe@h|xq@1sXuAlMRE_AZO~tI%&f1bE-XEixTkq#e z*3Q~SSJZ{Xy7ETaO~;zk9qiLvjxT9A)tWRgF{h;@V_r_UqkU-3f=u2>JHGsx=FHl| z9qjM!n}2??wrJ<|wIyq=?&8d$u$BRhB1$vkPi<&Ty>Nxqgpk$w3r z2>+9j-ZO|apmTkY4bmgx=#x4fZLBSKpx&tncSI>3520uZ(Mh^P=V3H5X_DB|u00v) z+gjA;uS;Vd{7pyAgPr4QJRW|b++n=;8{@rA8LOo3Q>mTP%eykDlg0e2_tqb5^!9OW zL)YZ(Qz~-2ynO2Gyth=W$|AcHv>L~QRuK!Ih<7e*Y$b|^BOFfc1BqOce4>`f#W6I( zpYhdHOOxRS;y7xG<=P4MI6|S)dCP2VCdy5HCf=EjCVR7u zgAqY}31eU$)LDu9v)UZA5io~|mMZm2`!!NSm?i#PrE%~%!jCz~ z-=NRC&MA#cGz&I*7X0AGC~%oa%p>0N+Y=tM*{%dNxkbvN9!{ol>Lkm+arPvpVYuJ2 z9I^vlq)db^qFg>Iw|u;vBKq67wrunv}G> z@}qusWN}}|w9z?3}glT1yslGF)e9iQN z?jp10#be>qlT&BK>_Iv5rlRVeno!6D)fZ7rMO2fc0(OkSBs(}T#3_x`FonAmRDuSE zQl%vI2VzvNh0@ZMFS3u2lA{xwaTG`Kzn5LoT z+wNK8X7oIyJIEtCr=AYan$R-IN7=t6wR?eY^|YnwYamTqm5-(0n|9+Qbv2B=Jl56D zFotRXH~e<$FUH~f!!2rrjT;;s>nRl+FS@sW-o3@ga%(E{EVO5RaYuAx`G*ZuG*dJ}xc}gd9Rj-U$AO0X#KBh>YxD%@~5^5s0%} zMS!|ECLt>&d{T5xXh}>&Zg!~uRP}GYk3VlbJT7{AOi)m4kb87>Y*1iq_8y8F&6OpV~??S_x|`T<&Vsy z-stOlhDoZ5=t;fuu3TX1<(Oc3n=@@}WYcE!2+?ryM}Y}d01HHuFkB0n{KjLCzRo78 zC-&`q%lHfR`_reH`AJn}66ji{{7L@8^d7KzL*I{a8Ecd2Z95txc;b!bKz#8Dt1$q^ zFSow<69_2Sk>c6d$6tT#G3Ly^SjoZ@yn`}>O_>XGCRN88|EhjJFu=?`sB-u6@bVi| z?-`aFn3almQ25vdrk`W%@u+^n9R{=WIEPF(S65#L2h(Ou$or>=kYX{s(MXtIbkD-r z=77M&A6Od6i$!ueBEByRD~y;rEk0yITv=FAcxZZB@Ps(yufFT+`)<%?S5i~&1B5qo27(5_FMVl4kFK_Qx*YIjR<7;X(dry6f z!fI5rcdPWexZ-YxCcWc+aVP18P>q(}_d1S0bgT4YPyAkb2L;Sd=ngO?)5XWv*ClZ5 zSQi&tAh)$eSHleQs!4v)Pl5g;To|iGuk94Een^FD>xTtQy1!uS+&S9{4(3*r=M-&b zF=^wy!c+atg)54J!g7O%kob7MjWT*92TwO=Pe;>q_n`QR!O`Q4ypN=`7u`m$qTk{l zF#ThQ;3i1N&~5ZdAg6Al^qV@sJi>oN-VOa`27dE|@d{w5+bI1e72mqRVGsx9zo`Js zX~N9AjS@_;bRW8n(r+>d1{!7~V&CJ@~q*PV8J{#eUP4|Jr@x$r;tp8saQ(~q2HpXgY0sy#KdtaV#y_n|8G?ylc# z%IFxC~L!WpYHT|g1Nlh(Dn9~ir$KSU)U#~m#9tcUGcr0KxVE(ArIYPxFNK!3 zY%iH}YIRO{xPcQ=MNUKBq;&K|^KzJ+t!MEPl1 zL4CD$KaQVaLRZlxxwrb&0}{&0a4J8%`{B3oP4t(7rS_nKdiL8PmG8b$o;Xpz%17~c zUN4}jc~M(K`$mM4%}kj%cn|xmB-?pvLs`YT{Qgxx+eS~l`NWsTsQZr{+s3YD%`Fa^ zIx`XRmX3k;xVm-u(e-m?g{!benZZdthnq@!A$_Q&hw>jUdpKDu7DaP5ZfQXD+>!bg z;dQ<4gC9Hrtc|U*tdv)(o7%4|uQ~S9bMe{wVs|7AZIVEpbaa~nR10DP@?)Eo&o8PKS$+_52v1x8}NO@bqjK%S6v5P$J zakbkUhGi}`--?$HR3)bFii=KaxVJKQ)$FMOnLK);J}*;kiwU<<7zxGj{Wk4d@Hdt- z*QA*iTqas}e!jke?(POUV4`Ai3gdX; z2Mf*@{1=ncI|Yiv#r21d=CmV)+v?_SD?FTCS(%+&bBpIu!Dt*DO;9mf9sCoDq}gVm2bglV~Qir0C@U>_%=568HqUDqkI!O zzM*Y_xAl&b&%6~46C$dPY>u{GgNojAM0#svumxrUghs#MB^#9^Ha7YVFdLgedu#Ec z9*tp!t?Hvnrg8?cLANUg_X!i??L*>`T;b6jsHQwp+PoDj-DjohX!=DWKm4^q2Q6I% zoyzwU`+fb1j`Wa<&Z5-D(y-vtmYnIU%Yv45y}G*fiFKI;n=UM9dwzRiO7s2CHfA5n z44T^dO2PgQ_cEJB_b;9rS+{d;-lj##F_rb<(!JLf6z{mYq6cNod3!$G|INNnE_SE- zPT1#4cS(nyS4-GZ+~akD{XYhtG(yA_#3Ns4L_t^l#R2NR73b&Zl;?<-5RZS;{y?SK z?v%+CTAfCgfqG7X5>rf4Q#e^1E3xm~ls)*o)R8_AR^DsUOqul#@`aBQZ_T_(H{#*LofxkN$Uqjz_ zU*5OxJJ^hV9A85X{yuaGvuwia7$yU{QV)LyJpWApX0=KRKBK$}fB#;c#pd915PfDQ z!$WVPGU0i&?l2f~tZz_``VJ(2H#%D0fP~AaNs7TPb#n>|8WRv;>w_nW6l`Kw+l~B! zG#81H@0nqvcxs6P$_u)}vxHQ;S{tEpPgA>D&KHH{T)ldA<}>&7zEYu{SoY=knvCjs zL8b8%Y7%qmr%qctr>1-1skYR{=em;GrLQec($~`dmlw2@s+BD)*fY3nHIKPyVAbUXOF!=0M_V zmz-?QaB#?IKGD?l(8~0Ipwf=g{3V4E_RdA?=azJo1R;3zU;iZqyRUld@brQmFRiXy zgO~6s4!(V#d(HmVY0*_n62H#hGCjR@AH)fLzdlD^vmCk{y&wI&yzpu*dYp)NYw5M1 zD6(Z>_9p(@2#J)L(_UzN%DDQKmFu2glR9n9<#n50T8>|2wUyMa%=7chUD=yBKW)MJ z#u-OmT^_pp)gv>OJky@jnw8VBvmmr!cY6+TY%Vwk0oOS;#$Qo9?7=fL9_Jv2fnS^+ zsAIgMpjGkS5)1{Y@xmCC3kJ>%-ppBhZei2$7KB1GT23sPe{OBgKyZ0y$?%hcE6aob z%NTjXAi212c;{e6!96dn!P|j53Z@@=YqxvVzSgvI?d`y6tp_-jYAtTLmm}H`g34p` z8;cIK?@YlfOO7Tq5JXU)(@85!_fQ;hNDSK~E_Xq7!>RPOPjyuMxFRY2!SaTUX%@Gv zHRl#%3VhK|EIFUKtd$u$N^=UfUv2N~-(FC#t-qrK|LqU0TwPk)T@?~qxvH$RyE0UA z@x(ey>?FtGJwx7g z?7!JFE@IEH{DIT5P5MggnZ6ZS4i4GNAHbgJN*kC`(IxhbTj83yB`YhY$jvwYVf?k_ zb8MPJ({r~z-`OrU%{$mM2gIgPE&1z9L9md41&U!bokC6($4=gFPutsBF6ha17i*sH zb!KpG=IRSgO(&P5{4aC)sU;2PS7(9Fl_hoSi{0#_ikIb=c2!JK7Z~l{yAdQmjkUdh zgsk3AvWhmBRlRuUa%pu;`ilFkyZc+v8A`j`b?jJ!7xvO*gFQHjy(BK9@>j~No5>v} z%N;6b5XkOa``X?TdVsp?(bl-ZUrBATo%8AW>FBxJYU}nyCxt)aR&n1OtHL+`cvoT3 z_Djuck9rqvnI607Kz(@C-j-JWc$u!FEd&L-eLEQ zKKSp3lQSMz@&3{L%LPe^d-7=WOi69)qs^o3|AlbU`~CA;OX}7XxYrv{6PrzPlh=}(p=g^RMnU09!6=+p_$Y0lAhL1R*C zGZX#P6|OPakq%*Fm`jF|awEw2Yp3|?y5Pamu-srtK5aG!9iAJ;p=zn?WsE7#@3b+ zn zH^vExQ_0g)D6!+IdbHPSoe>`%e{pSDR&P5`A!Q+J_ZxnyL!YT72kKD=Gne`ZPUL$`C zVH3AASK@RG9)9hU?!IXC2fO>H{zde6`1DgWcUbb~e)9D{KOR?4{T+^@xkLNe8JP8g zp*PUm0j>kjN201FDhll#A|lWrhZF~VyK(S@Hc;<OPEPCgO{*jIg@60gVhvMcV`$qdQ+G}?316&Lw>4BK*C$L1l5e1)w`(Z9f zxB5-;_lF*}8TRy}>4{l?OJs`rc2vwL2>C%vJoJ(Gr zW5)Y=*g+i|I$b1K-8l5Zrt-FT;D9UI2k=N0e&)rgU<~cghW@=}EVJv(BK{=F)CKvt zzZJpqExjTsJD4xGy|||=Y~JA(ORDr{?UJaJre5jSBr!6&+!>UkSR3^Fod2~*gf+nK=R3R{ymHq&3O;rOSIYFCQRojL3*?h8=M($R0zuum{7&JDzO zjpNOU@HVl!3~$G=D3%C+Q+7r?bD`DWycMdOy)2*{<$nX!XLw6Mxd2z|Xj}pMi02Kj zu~Zt-7r0tRL76N%RI}lG;2Lv9kpqBi4S~fC(|^AXcaBxEY!2rIxPn{Y4q4%(EH;}> zqE{&u*$17Jt`0r>G>44?Pu8e8%CF$j2f<4so~&gb4E=+A@F_+-*l4BjQLKHwIU7*8 z6-%5m+`!28&nsg`&h}O)yk#Juh`%~dZiVVI91>8t%EnfY9??c0u^F7Bxf+!~fc0AE zI`lE<*~}?J5{oDsy1=n96F97o`3zLv^bt_qh6DknjDQ;Y3{V9lp!$X~1r*BnbjpT4 z1k`*pQ6@%g9%|>5Nz4miI%NjT(`aj^wa3l@W#_>&T6;FmG-B;nT^f4Ap1&TtGzuYR+g}f%=H&4S!~7G-3*%W{-k04e3zL z%on)ET#=`hZZ;sGJW$H~gHU}~bK0NG z3%K;P+^VqpLq-Yrg$*%!WU+8lw^KD*<+nz_>mTwAbly=kO}2J%`H&T<0xt^;S433(DlL&{Do2u*aFUvwJcN92eu>h0Yz56Rh`Os+-jasPG?#0@v_} z2UaNxFgn*bCL?h~jMr>HfCn>`SQfx^jtqec)@W1#C*p~@yH&zPx}eX>-6Pn)i6daT zeK-th0xQhWyMPJTVa$fUp}-*yMVjDI4b#v7pmMqHq|wZHZKM`ZL=|a*%R2w@SNZpR z&0*Ff2y}iSU-M7^J!Yknig}KQUIgNp@1Q-3!0j#4;GiNEP&};H|&Hoe8i;zgkMy&>Si+plprSYzNbyzzbnJv>V=J8Q{F^j8(i~rHt=LWbTDF zF&WyI+tBKeE_vSy!!em1bxdar-*PP{)A?6zg%X&4sbMl3&QK3f!j%bJ8#xqw1V6Q$ zSO1Bsr2s=%raL+Z${v0u10F{}wIWJWjS?AXFsLYbOMV;=jKfwVuT3vwV)XVWufWxe zEOLvb*VOT^OJSSXeX5_lNs9OwFHxK+4lAD;5au7}iFsw={!?6S);aktoR`qN2)5pGwOsxT(Ged*3T~-}WqOWJNpZ?4cIpg{vF}Ws zHq8s|?Tt=mqLtz|AXYqWNGUFE!|L4);PSyBM5FtlAiNjD6FuBKOhEZFrFDY1!*<6R z$@Z~LCHc!z`=@oBUbO1+5?9yjj?NRMYZV)6!XxW*Yqyrk73TCMduOdUKR0*tr8X9R z+jRS-h|KwM(d7{-o%M;8XV$7V3G2e0C$>n&tzJ`0rpDI96fTXK9okf%HsiiIGq=x< zv^;t{JIM0Fj{lW;CRSKjUft|Xyix&eHN}~+@!veMNtE7cKO z_MJK}6z)aV%uCDfRsg9i#$qh$+nuy`KPz}9R7u+GD zJ1SRAI)FD=zg?M?cUx}(na^iwgVXy2Wmd=;Nk6S9 zq<$~Yz?NvyzhW)sJh3rxDa3QQ#X0@GP`USNtJsV4-cA7MulCbQuzi{bJr z7Xp{AC)QcMRX*Z&6vxGW&VH>PHaK2l@kYk34jtpxDL5)-c)t}J+7pK%7K}M&0NXJu z_1H;98}7x0Yux&p^IdiVPgh%^xJ|}2N~s1lo@D`-gjcFrkOnpMJS@C*X5v=_hdMu$ z%JONyjRlldgAGwZ*?i+?fD(4#`JsuBZ4K%XE3Tgcs+H>3Y99(f)E2nNhW#HauFnCL%{7m}b!MoU&ym>z%Bp#W-Un2$@k1TgnV||{#S+v3 zDA76R3Al3_%GIWA8fT!<7@amM!@gO**E<6@sRy(=6GO@ z6zjx#SUk~(c0sJ!$`8~XNGV)msak;S_1K|b5|le!LjBZwRGnco^0A``3UYQx{TJj+ z_C)E-)uBII#<8PB8+jZZR;Wi+lj6&1;m9>#!4IT{X+Q>(dJ?X71mL~E$K;qVshF5HG`j+Y>bS*SfKaPu)Kyfqi>(J(!zhAY`H8nA0i)#fESUEZuWQK$J@beSZKw%m%Jj*W%7!^ zJyr+oppar?x^)h|7;QOUI4)55o|vyI*9|&f_Npdy3@8U*I5xn=3jPJ&#uv~=Q2{~Q zY33$qbG99pC)&sG*hniAHfZJboU%!vO!xk-ar%uK45Vpjp3#Hop9rWF>nwTX#`Gzm zV)-oF4W9`;L-CzMRw&_Vk+lKihpdez?l+L{r&}8$2IQ<`I>S6fbcdXttkIZAcM~SE zHVz7|#sHVDy#=l$9oGfcqD6Nopp-|*c!WXFMSKw0E*Sm{Y~&r0!^t-mha(Ajo3D5R zAGMop;aru*kBkC=tEjG{eFe{t@0TlrHqTpDTzQ+XILC#RTr5^Hc-$xnuU5^{F}~s) z8~FbZ^-%hzY!~rmu5a! zR`OU~qBhIDd<9qvd@?;&2P;&5fzJ|oN-oz`e3mQ$J!*7M{#IOlhNA)(`HFh1ZZs}f z>~o1KmJCd~#a@Lx?Mkkj1yyU=Y3xBjrQkGmwaUeO#C2ZaaUDU`T2?ROItd{o;yP89 znZ;FT{rqd0Bw zPR=rlG)j~)SZ0N?Mb_hU(>iDg~x9w27>s_ToF^kEcKX@h?G*yLSSk){D{Xa*f?O)V-^Av=^lZpeID066ubGv zO1q$mVip3^8N-)CYx-ah=`jm|iF6NPBHgn~aLXUK4vxk}Sqy>e94mpHKv6S)AREqN zaBL=uZVGJYpnF7gGZ^T05IU}V1Z}jx1-=U`2ec7B58#`r!vUXS&!tS@yI?@>fa@{} zK=Gg#Tx$t+I_s3j;5|`>7*+7&1DU4`3tCSIkF2nzC3yz2LkFI?Oyjv{~d%E7Pq z9{Otc%QZ(E8jsf8bDNi8e0nhWpl{(@I9 z6+}{E(5u5`a+p&mb1`-L`uQo!I27Sg1Vw~{iB@UAl3O%C(PQpOb>KF# zp^wr~eDDQ7AyT<)^s(2u1SwvqD`~aL1~isYoDfiSkf1&`nM)9d(&K-KS&xeB_ZjZz z`;AdMQ+|L#Ij;E;mE};AIQ1M#w`e#n)Atxt;5y5e3tXNy!#)Mcy!;%Gun1hZ zY9nwF)MYEb1YIQi0@rE0otlVQ%5H$t&o+UJPI1Bo{hCFb;4r#1^EC{+S=?*dB50T1 z=GME3LfQKIR1E#q)MH}6v_pehhbk4|nh$-x*Gkpup>sw*^JYNdjs@q#XKXV;#d(a1 zF0WPM_X8y;(>IN(wrt!c6?*bI%8h(yS15iYnNnK)Yh3aMv0*dtb4?sb#R&Y$Nx5LE<4??zuk;sA)tKAB{3NWqb`JI6#c)RPkHgvrSk}_Akz> z^M*sHQvqF+%e7(?J)roQ|Kw1)hA;S-JeNDyx;uEMs1Z=zXe~zcOJsWgAXJ~>9sxzx z{w+{Mml*N9VJoLgA}Bni6vickkH}{!pqdR=_$*~ym0+yP%6sve6Q>N?fbWub`Iy{Z zuyXb`sBSi2Kv6FD2cRGk%lI1k+y=wx65}-^$3Ze7u?YYh=Ds6#uP>+T3r-iszzLh| z2%k+Kk?#}ZJK8>h57in(FqEI+rWHjQ1kB&{y>tc{EslwHZ$0N1&Q&+WPlxY<2T(sO zkyWvgdz>8cf`@qTjn|}!t|E7nOS!{6aqruX)!VL)tGn)Wn)3Vp#1Y!lT*EetHK5#q zE*+zxum)Xx4I*xXG5%i1be2^LOcW7pw@Q$}Dl7OLA5-?|43D4-Lx_}|Nh4wnj#&e3eI@v5% zlKS_SkLnB)h}tseaTBcFVQ|08h!=~st4};HcGofR(T!sym)n+dxb{-(C3EwM+m=%2 z2V zpw2SVc+g~VBiFmZ@-KRQ$O;8-3s+#szl-Bhm*ZAng;snvr%UBy_8}8X*kr1Lvg*a5 z`=MoLa_vbv9hERc#7`gkfVg-MMqfm-uAMDszvnY2RWBq4FnBpiapW@^6f*+7 z9Ex(r))_&rBznIJm~PUU3jVmM58>OecjVpt+c)rSlcC#&zRTl*x}MstDpb{r?-C4s z*Gav=Jz1P3SBDOd;4R?x3phT;M)^UEL6Wb3pGV!AEb-#^+FFP+s+;W>lvC7g#OPQt zK02T{))w`w{%!1C?SwmRxG28ujBo35C{7YuBXN?n8@BT`;wPML;R+~{L;-ci@C?Vr zPbFOv#W_eL3aAU<7ioW4ipHrlY}detRDw89zvC;?&4L9sG^ZS%*Dh62g7~}u#pg9f zy?}X%Z__CBZRn(Yz9ND6u0AXNY9JrsTWy4NF?WjV=-PBN6trm}mx~O*=zB0%Eu;4b zF&Z#&TMKbBP$9lcRL}^ZqMm=3q9FQrLo7qXmMfPFSM|(rgyVo!F1UqD`hqN6!d1y< zyRc`t{T)iL{G?ag{Kd&;T0x0H$EeQ?SkPiToNSa4LBY!j>EP$ zbmK`)d(TQ98lMuCQcA!*IKZ>>XIj@G!Vv0H9&32sv)~ z4}3gigSZV~wTj@A>ieBTah)fiC{xcz^aPZy^#m@GDT0DbRdX0O!05Zgk5|B?!>gkT z7?a_mj1I+@WJhEQt`4PZ(mAvTyY(?s2BKUc4Mf#N_2<|Lj+Yc?Gr?t8;Z6Wst0@a` z;RwAF96H3nuX_-2%9sKEi*0Tq?Uim$T8ql!ia%^$}W<3Ji$)RthCrn-7!K<9- zPYs2jlFk{O2PEVo89Pe|(H zFW`+f{Bt<}dm&4I{yqg)c%gV3I2}gBtG+qKcd7Y%0r01 zE7$qov9<6SP&^)Lq8KJV^B4>Ndm8_og1`Taf1XC4@45<>j$HO|?K z@KV30oso_6=ftq}^11xEi84zsYfxcB{{obh88!mypt?~0n&Y~zLFIE?5=HYbT5-LK zwJbGJoR!uR@*Mgnl8doe#^Y9)1GsDaPrlyoaG0kiSywv=s(W~~Pl7v{{9ZGBQ*e*x zO%QE2QQicfPpS*#FZtggkNE75y?c#+p2I(Dcdc+04d)fO`zO%0f@kl9oHY+^f}D}& z4A!NL@NoJ#oS?6Hjev^NsUs*(4d(}_p*_iR1V1Q=qw`kGmH0up2r6KRb7TUL=^R;y z2#~jFBA|XIpro3iKkyZpFe0`?(w@(2q#@nF^#j%0)P+Tf=253-I9xY7Aaceg*qI`7 zfcO&6BA_!-GR0%XtG>3Kc-I&0NknrS#AH#=_J4d2QhsJa**!~IFK)=r?|r`gp-2! zrlR!b+{lv51h0S@E%(&UKG2ky)P8<>NlU(e(o%5#mR)<-y9QZr`6qn0MRNH7pQz3c z@{!L!gTF?jIR>kM-4FhDvkj=Mz;2;Pm2R<=VNboos|0?>q0Sion{!v9c7N6?f&Zwf z_!@Cqutt~@Ut9heYm3iBn}Lt=TQLfY7HbHx86E}Sx=lZg)Ni-yC&J43D9`Joj?`DY zwv6kuVE*^1kR-|vS@qW~`pUrTDfy1fM#-74HdW$e8h?THc!1M-dZcY<3w2iOt#F1qwZtZq*IdAoz^d0P2lzOv)e_qnUcCkDl74SQ z76}}#R4bGg?CBuGZwRl{!cL;54)$6$bdnx7oCR<7Nu>->UujS|LstkYBg9~6q`?W- z8sHjnQw_L2;kZ~BBv8jyuU3q_zXGV~8kFq_D85rT->{#7J4&68>LKb&^CJIz%ejqA ztHGSW&$wT6{+wa|;wx+PgNLM_q~Nwm;b+K~8q|EgW*kaCLvF?Ghcz7k|HXX(%!A{) zi<^YvcENwVVKfBIYo5jVB-WGX|C9>wX$R+%*cbobR6x%|U%auVUqc3Gaqcx65LM$g zZ#I+wFL2^=uo$+{XG$e#IK^q;pN(%qdhi*t+l=~Z#=n<2%2pO@NcZ-{8a6W(YdGeL z$ryn}?A2vltRhv;iB)W7jaWr1oUr-1Y@?o_#SEWW$Tr5kf(lsLGd0lO!+!cK(EFtE zb-X0l%U53|*+thrW zgiYOUc#^M^qYu33EO)->rU9rh6{#=ps~582iy@I?kFVfE$+enLF1aO zhA#>>%rJFJVI#iCJ<#v@Oq&~NugFpaj^tq(f=rrXIJQqTY#(UYrlKBW7+W87;U+#SYOahLci{vAFWvMr z>-5b;wSrlvkB4#uhe^t9Jal}`z?ZM%G*g6X~c?Am7rKLezS4p zicQp4Ouy?mV#Rp;PX zyve$w>md`kX#_Z7bw7zb0%-_rMJ^3iIAPI{whhsCem|RzJ*`kia|5xMcD}ITY3B=U zk1M@==d%bTEcBf(a7F64&Kvgg+at<&;L@Z0;sy}yLBfTc%xtRUWGTSJ)iI4nm_9MT zYsCZ|vKKeM$OCi(DlYm~x|FB54&k_@s+puiBo+;1x(*qFT?20hSv)aEX}aM)R>-lD zMp|N(qoJ$NA>VQxBCt)L`~TE-^}$h9SNy&E?%NGX2;oDL7*hxV5)2q1fP}9^Tq1!Y zNElE+B1)(=QV~V`Xc48Uv*v%2KML)Pmey)3BTfqxt=fts)oEu$ zG5fZ^bIyBr-v(6v>0~A|Z};r)o_pW9=iYPg`Jma1tSUfDOF8j&cl4A&YeYOkyU> zK3^Q=>k#CcU|m2`Qt*sRC9S$rcR#yWo9q+*8zy-PtTfUgD@2Eo)D%3UCX3WOyEq-E zF2AGQ{ijiqpD`joPBY7o?Lwg%`S}^XI~KVYRKNp2ac2&yJ>snsl{umA@Z$E}EZc&W zaBf7t9>{Ezx!xGa?DtT)F&&))P3K(Bb3pZ)0~@ePX0tU0D~;zRc22;9GgZbMSnjp! zi9A>2ycm_tAcI7Qi`lN!mY`jUB6tD45WKkC-)^AwZw%;6G#T#;d6g8jIA$KNl4H=S znJa`foyU!_AmQ)XS~m zP$cg;p!+`b*D;)E1m5!ry-?;lxz>f+74Lc4=?Y$4pWvM?jIYp6SA6kl)UM)IV`#){dq z1Z&urVa2RNf>m)Q7U^W+EADF6$*L4sMkh00k()pze8u%cI2VIlmji28AlJedt?7cb z*ZLc;>FOdF{bo&vk|A>$#0LetOJm9s#=Rp}HNJi=uY1uNCj7_08>E`PP z4PObpMI>kxS`58OWR%y>vt&%?aZTvwaSj5>$Ysg6Ex;QL|Dz{ZGOiXnw5lyd>xocRqHjnFVYHce_gPg2<~6d;*@O53VcFR$GpXQ6|L{^{o_6=3;y(=s)23_ zs3Zmw8a!eF3%(ifBw<=d3b#-Qi+_Lzp0_&&IUD- zGS6kUjGFdA%hE7XQ~rM%R*1F1?KFB!b*~54A@61shi__~HYMOFNo1*?a8Qp;&fIUm zO3Ci_P~EoV55c@jCFu?gr|v!m$_fD^f;`zl8M!cqGuzAUwvSU1ySAPtJ;j>_49o%8 zY2(3%R4(gpAR7j(^A4+`dXX+dv1-2+7pKU6*(@VJT%%X3}#X*w3 zo>1X8{VI*WP94I(QMmRA+81`{N3acHDZHP`}s@Ayu1H2{w<_- z6tdp12MyjPs*~0s5r=E*Y$q?T56-F-jT~j*D5x1gy=Vx^wf9jpUL)-l_KLT?3HF=s zcx_(WJNBF21m_;^(Cf&vT9qtSHOZINarKvEE033d;~Q=(Tl;NEg2i-Omkb<;G(csJ zQzngNpZxf*-X43+;nSzzL=ZZXOp`o%x7j~PevmwB|HQjN4OfFvr zQ|bfDKMTV;`=Lr8-f0z=bhMJUc}Eu{M?4;aZ`ANZ)4iNhJB(@R^U*sn^)!VJ+Zof{dL< zdA~h{&imXGpu-6Y`0vyMv3ntlhYlkjqz*7$6UMPVS_wK>#7sAt&ktQb;NYW8rW-}J zKjSs{bc*p5A8{DG6m*mQ0pp2{+Ns`UJjJH0!Sm>j;WZaGk-o*-sSd&ii7a+9c^@_W ziFHIfg=)Xu?TGf^6J=0TfI(y)?in{(g*U>2=#%^byo7bSRi!%xsHGSV{Ag;v2WE# z@HHic29MtbA<1RNx^pj%_}>5Gu^rkL+{h zegZUj!7$a2Ao0+_V=+;U-A!YREE1rSd7sc-!doe4kuZJO>?J6R1n6YmC!D!hHDL`= z#ow-!vBIvvnOHc>LP8&eWWc(bUP*N3OsvV<5zdSN8Mu$~ce4#7C(1ol&(M$hCKUgoR1^QjsxTIexM4tzbWf`lVPDaUo z#eM{%x7T`F?AJo9O~!sL#oF|o`wvhp1r;o@lG9`#5KVez!U!jMgas*~-<1*W%bLfG za4BXi`ML@gX|j1|rJD}vJjr-dnX`oZ5!2}+xT=I^MJ1&35aTWL=^D>Uw*>r=M|BVz z9cWO0)*Ak<0)J049`+zC#n&-jFFFq%hwcLRFnOHUgU-I$%Yp>RnWAM4Fj71SBYd-4 zjL0-tSGafiRqBX$6e|5;j43KKYDBC}TdYUgo_wBin0`OF*-G?w z(*Hb*>5n5?qM7MW!Kfqra;8VRFMTC{e$vO^&7Ysb=Ud$K(Js>YGkvXh1e`OUh?t2G zpLo6>{Gom?o}pR&=g0XOacOvlz3seB&lDpd-*Y5U#D?vQXKZrqEBJfZJpT6-BNd?- zDS38p=jYFYXaDz~@-xQ+A$BH0@j(9XQJf9WpYSvGD;(1jp|}@(pND>3z~7(b=W;xc z=V#n+U|gTViheG!y^Qk((H4As!1IPi=W(J^7!AybaP)E}RV&40DA?|-nW>tPm&HavO!rNxOAWIO&h`9@7e_zCa>G zqRu7}F>a+#(5eng2uG>XxT?AQeQ$}$kPz5=v*9cDhTaX%W>_;(OLpK8Td5f{&a0g; zp>FJjIqUl6oVO9dp|4cF@+ZWGk`*Y9=2Uiq>xggxvGdCO@(^_BQ6AKXu-)N-8i#sR`bklN4wSNTPx>Js$Nhve9n})8y1ZjG4GB>B<>$QtE%x^wM#ZNRt%kY z@5-4AhL3Knsh&Ho(7TLl0cu1kw&~At+aekJ`@wSVOL-qrKcZhA%7{?lYyPWx672z#7RY~)LN8S9bl{!Eq}0S<*mE(`XhG3>2F(T3PtS9p8S zEA%lV5-n!8^X$`w*h`!9z1~#oF*c+L^x!k7SUY~ioZMiyfQoLOhk%TyGV4dyqO5zM zjj_ta;8*lD@Zx7E#wslKYlr=9?yr*vUx1pRCw^{VP>J>0;rqm@so>I#%46P@8F&31 z$R$Pf0(HI$)eF2DG2h{v{tjqs8etoTMH= 3600: - hours += 1 - time_stamp_in_seconds -= 3600 - # get minutes - mins = 0 - while time_stamp_in_seconds >= 60: - mins += 1 - time_stamp_in_seconds -= 60 - time_hours = f"{int(hours):02d}" - time_mins = f"{int(mins):02d}" - time_secs = f"{time_stamp_in_seconds:05.02f}" - fi_time_stamp = time_hours + ":" + time_mins + ":" + time_secs - - return fi_time_stamp - - -def get_timestamp_for_uniform_frame_extraction(num_frames, frame_id, duration): - """ - function: get the timestamp of a frame, 在均匀抽帧时用。 - - num_frames: 总帧数 - frameid_list: 被抽帧的帧的索引 - duration: 视频的总时长 - return: timestamp; xx:xx:xx (str) - """ - time_stamp = duration * 1.0 * frame_id / num_frames - - return time_stamp - - -def render_frame_timestamp(frame, timestamp, font_rate=0.1): - """ - 函数功能, 给frame, 按照顺序将 index 渲染上去 - 逻辑思路: 把index渲染到图片的左上方 - - frame: 帧，PIL.Image object - timestamp: 时间戳，单位是秒 - font_rate: 字体大小占 min(wi, hei)的比率 - """ - - time_stamp = "time: " + timestamp_converting(timestamp) - new_frame = render_single_image_with_timestamp(frame, time_stamp, font_rate) - - return new_frame diff --git a/fastdeploy/input/v1/ernie4_5_vl_processor/utils/video_utils.py b/fastdeploy/input/v1/ernie4_5_vl_processor/utils/video_utils.py deleted file mode 100644 index a4769ca8ecc..00000000000 --- a/fastdeploy/input/v1/ernie4_5_vl_processor/utils/video_utils.py +++ /dev/null @@ -1,83 +0,0 @@ -""" -# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -import io -import os -from tempfile import NamedTemporaryFile as ntf - -import decord - -try: - # moviepy 1.0 - import moviepy.editor as mp -except: - # moviepy 2.0 - import moviepy as mp - - -def is_gif(data: bytes) -> bool: - """ - check if a bytes is a gif based on the magic head - """ - return data[:6] in (b"GIF87a", b"GIF89a") - - -class VideoReaderWrapper(decord.VideoReader): - """ - Solving memory leak bug - - https://github.com/dmlc/decord/issues/208 - """ - - def __init__(self, video_path, *args, **kwargs): - with ntf(delete=True, suffix=".gif") as gif_file: - gif_input = None - self.original_file = None - if isinstance(video_path, str): - self.original_file = video_path - if video_path.lower().endswith(".gif"): - gif_input = video_path - elif isinstance(video_path, bytes): - if is_gif(video_path): - gif_file.write(video_path) - gif_input = gif_file.name - elif isinstance(video_path, io.BytesIO): - video_path.seek(0) - tmp_bytes = video_path.read() - video_path.seek(0) - if is_gif(tmp_bytes): - gif_file.write(tmp_bytes) - gif_input = gif_file.name - - if gif_input is not None: - clip = mp.VideoFileClip(gif_input) - mp4_file = ntf(delete=False, suffix=".mp4") - clip.write_videofile(mp4_file.name, verbose=False, logger=None) - clip.close() - video_path = mp4_file.name - self.original_file = video_path - - super().__init__(video_path, *args, **kwargs) - self.seek(0) - - def __getitem__(self, key): - frames = super().__getitem__(key) - self.seek(0) - return frames - - def __del__(self): - if self.original_file and os.path.exists(self.original_file): - os.remove(self.original_file) diff --git a/fastdeploy/input/v1/paddleocr_vl_processor/__init__.py b/fastdeploy/input/v1/paddleocr_vl_processor/__init__.py deleted file mode 100644 index 8f79e65d634..00000000000 --- a/fastdeploy/input/v1/paddleocr_vl_processor/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -""" -# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -from .paddleocr_vl_processor import PaddleOCRVLProcessor -from .process import DataProcessor - -__all__ = ["DataProcessor", "PaddleOCRVLProcessor"] diff --git a/fastdeploy/input/v1/paddleocr_vl_processor/image_processor.py b/fastdeploy/input/v1/paddleocr_vl_processor/image_processor.py deleted file mode 100644 index 8e333d5bf96..00000000000 --- a/fastdeploy/input/v1/paddleocr_vl_processor/image_processor.py +++ /dev/null @@ -1,275 +0,0 @@ -""" -# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -"""Image processor class for Keye.""" - -# TODO: Support videos - -import json -import logging -import math -from pathlib import Path -from typing import Dict, List, Optional, Union - -import numpy as np -from paddleformers.transformers.feature_extraction_utils import BatchFeature -from paddleformers.transformers.image_processing_utils import BaseImageProcessor -from paddleformers.transformers.image_utils import ( - ImageInput, - is_valid_image, - make_list_of_images, - to_numpy_array, -) - -_OPENAI_CLIP_MEAN = [0.48145466, 0.4578275, 0.40821073] -_OPENAI_CLIP_STD = [0.26862954, 0.26130258, 0.27577711] - - -def make_batched_images(images) -> List[List[ImageInput]]: - """ - Accepts images in list or nested list format, and makes a list of images for preprocessing. - - Args: - images (`Union[List[List[ImageInput]], List[ImageInput], ImageInput]`): - The input image. - - Returns: - list: A list of images. - """ - if isinstance(images, (list, tuple)) and isinstance(images[0], (list, tuple)) and is_valid_image(images[0][0]): - return [img for img_list in images for img in img_list] - - elif isinstance(images, (list, tuple)) and is_valid_image(images[0]): - return images - - elif is_valid_image(images): - return [images] - - raise ValueError(f"Could not make batched images from {images}") - - -def adjust_size(size, patch_size): - num_patches = size // patch_size - if num_patches % 2 != 0: - num_patches -= 1 - return num_patches * patch_size - - -def smart_resize( - height: int, - width: int, - factor: int = 28, - min_pixels: int = 28 * 28 * 130, - max_pixels: int = 28 * 28 * 1280, -): - """Rescales the image so that the following conditions are met: - - 1. Both dimensions (height and width) are divisible by 'factor'. - - 2. The total number of pixels is within the range ['min_pixels', 'max_pixels']. - - 3. The aspect ratio of the image is maintained as closely as possible. - - """ - # if height < factor or width < factor: - # raise ValueError(f"height:{height} or width:{width} must be larger than factor:{factor}") - # if int(height < factor//4) + int(width < factor//4): - # raise ValueError(f"height:{height} or width:{width} must be larger than factor:{factor//4}") - - if height < factor: - logging.debug(f"smart_resize: height={height} < factor={factor}, reset height=factor") - width = round((width * factor) / height) - height = factor - - if width < factor: - logging.debug(f"smart_resize: width={width} < factor={factor}, reset width=factor") - height = round((height * factor) / width) - width = factor - - if max(height, width) / min(height, width) > 200: - raise ValueError( - f"absolute aspect ratio must be smaller than 200, got {max(height, width) / min(height, width)}" - ) - h_bar = round(height / factor) * factor - w_bar = round(width / factor) * factor - if h_bar * w_bar > max_pixels: - beta = math.sqrt((height * width) / max_pixels) - h_bar = math.floor(height / beta / factor) * factor - w_bar = math.floor(width / beta / factor) * factor - elif h_bar * w_bar < min_pixels: - beta = math.sqrt(min_pixels / (height * width)) - h_bar = math.ceil(height * beta / factor) * factor - w_bar = math.ceil(width * beta / factor) * factor - return h_bar, w_bar - - -class ImageProcessor(BaseImageProcessor): - model_input_names = [ - "pixel_values", - "image_grid_thw", - "pixel_values_videos", - "video_grid_thw", - ] - - def __init__( - self, - do_resize: bool = True, - resample: int = 3, - do_rescale: bool = True, - rescale_factor: Union[int, float] = 1 / 255, - do_normalize: bool = True, - image_mean: Optional[Union[float, List[float]]] = None, - image_std: Optional[Union[float, List[float]]] = None, - do_convert_rgb: bool = True, - min_pixels: int = 28 * 28 * 130, - max_pixels: int = 28 * 28 * 1280, - patch_size: int = 14, - temporal_patch_size: int = 1, - merge_size: int = 2, - **kwargs, - ) -> None: - super().__init__() - self.do_resize = do_resize - self.resample = resample - self.do_rescale = do_rescale - self.rescale_factor = rescale_factor - self.do_normalize = do_normalize - self.image_mean = image_mean if image_mean is not None else _OPENAI_CLIP_MEAN - self.image_std = image_std if image_std is not None else _OPENAI_CLIP_STD - self.min_pixels = min_pixels - self.max_pixels = max_pixels - self.patch_size = patch_size - self.temporal_patch_size = temporal_patch_size - self.merge_size = merge_size - self.size = {"min_pixels": min_pixels, "max_pixels": max_pixels} # not used - self.do_convert_rgb = do_convert_rgb - - @classmethod - def from_pretrained(cls, pretrained_model_dir): - pretrained_model_dir = Path(pretrained_model_dir) - image_processor_config_path = pretrained_model_dir / "preprocessor_config.json" - with open(image_processor_config_path, "r", encoding="utf-8") as f: - image_processor_config = json.load(f) - return cls(**image_processor_config) - - def _preprocess( - self, - images, - do_resize: Optional[bool] = None, - do_rescale: Optional[bool] = None, - rescale_factor: Optional[float] = None, - do_normalize: Optional[bool] = None, - image_mean: Optional[Union[float, List[float]]] = None, - image_std: Optional[Union[float, List[float]]] = None, - do_convert_rgb: Optional[bool] = None, - ): - images = make_list_of_images(images) - - if do_convert_rgb: - images = [image.convert("RGB") for image in images] - - width, height = images[0].size - resized_height, resized_width = height, width - processed_images = [] - - for image in images: - if do_resize: - resized_height, resized_width = smart_resize( - height, - width, - factor=self.patch_size * self.merge_size, - min_pixels=self.min_pixels, - max_pixels=self.max_pixels, - ) - - image = image.resize((resized_width, resized_height), resample=self.resample) - - image = to_numpy_array(image) - - if do_rescale: - image = (image * rescale_factor).astype(np.float32) - - if do_normalize: - image = image.astype(np.float32) - image -= np.array(image_mean, dtype=np.float32) - image /= np.array(image_std, dtype=np.float32) - - processed_images.append(image) - - patches = np.array(processed_images) - patches = patches.transpose(0, 3, 1, 2) - if patches.shape[0] == 1: - patches = np.tile(patches, (self.temporal_patch_size, 1, 1, 1)) - channel = patches.shape[1] - grid_t = patches.shape[0] // self.temporal_patch_size - grid_h, grid_w = ( - resized_height // self.patch_size, - resized_width // self.patch_size, - ) - - patches = patches.reshape( - grid_t, - self.temporal_patch_size, - channel, - grid_h, - self.patch_size, - grid_w, - self.patch_size, - ) - patches = patches.transpose(0, 3, 5, 2, 1, 4, 6) - assert self.temporal_patch_size == 1 - flatten_patches = patches.reshape(grid_t * grid_h * grid_w, channel, self.patch_size, self.patch_size) - return flatten_patches, np.array([grid_t, grid_h, grid_w]) - - def preprocess( - self, - images, - videos=None, - do_resize: Optional[bool] = None, - size: Optional[Dict[str, int]] = None, - do_rescale: Optional[bool] = None, - rescale_factor: Optional[float] = None, - do_normalize: Optional[bool] = None, - image_mean: Optional[Union[float, List[float]]] = None, - image_std: Optional[Union[float, List[float]]] = None, - do_convert_rgb: Optional[bool] = None, - return_tensors=None, - ): - do_resize = do_resize if do_resize is not None else self.do_resize - size = size if size is not None else self.size - do_rescale = do_rescale if do_rescale is not None else self.do_rescale - rescale_factor = rescale_factor if rescale_factor is not None else self.rescale_factor - do_normalize = do_normalize if do_normalize is not None else self.do_normalize - image_mean = image_mean if image_mean is not None else self.image_mean - image_std = image_std if image_std is not None else self.image_std - do_convert_rgb = do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb - - if videos is not None: - raise NotImplementedError("Videos are not yet supported") - - patches, image_grid_thw = self._preprocess( - images, - do_resize=do_resize, - do_rescale=do_rescale, - rescale_factor=rescale_factor, - do_normalize=do_normalize, - image_mean=image_mean, - image_std=image_std, - do_convert_rgb=do_convert_rgb, - ) - pixel_values = np.array(patches) - data = {"pixel_values": pixel_values, "grid_thw": image_grid_thw} - return BatchFeature(data=data, tensor_type=return_tensors) diff --git a/fastdeploy/input/v1/paddleocr_vl_processor/paddleocr_vl_processor.py b/fastdeploy/input/v1/paddleocr_vl_processor/paddleocr_vl_processor.py deleted file mode 100644 index f7d7cfbd2b1..00000000000 --- a/fastdeploy/input/v1/paddleocr_vl_processor/paddleocr_vl_processor.py +++ /dev/null @@ -1,322 +0,0 @@ -""" -# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -import numpy as np - -from fastdeploy.engine.request import Request -from fastdeploy.input.v1.text_processor import DataProcessor as TextProcessor -from fastdeploy.utils import data_processor_logger - -from .process import DataProcessor - -_SAMPLING_EPS = 1e-5 -from fastdeploy.input.utils import process_stop_token_ids - - -class PaddleOCRVLProcessor(TextProcessor): - """ - PaddleOCR Vision-Language processor for handling multimodal inputs. - - This processor extends TextProcessor to support: - - Image processing - - Multimodal feature extraction - - Tokenization and position encoding - - Request processing and model input generation - - Attributes: - processor (DataProcessor): Underlying data processor instance - tokenizer: Text tokenizer instance - limit_mm_per_prompt (dict): Limits for multimodal inputs per prompt - """ - - def __init__( - self, - config, - model_name_or_path, - limit_mm_per_prompt=None, - mm_processor_kwargs=None, - reasoning_parser_obj=None, - tool_parser_obj=None, - enable_processor_cache=False, - ): - """ - Initialize PaddleOCRVLProcessor instance. - - Args: - config: Model configuration object - model_name_or_path (str): Pretrained model name or path - limit_mm_per_prompt (dict, optional): Limits for multimodal inputs - mm_processor_kwargs (dict, optional): Multimodal processor arguments - reasoning_parser_obj: Reasoning parser instance - tool_parser_obj: Tool parser instance - """ - super().__init__(model_name_or_path, reasoning_parser_obj, tool_parser_obj) - data_processor_logger.info(f"model_name_or_path: {model_name_or_path}") - processor_kwargs = self._parse_processor_kwargs(mm_processor_kwargs) - self.processor = DataProcessor( - model_path=model_name_or_path, - enable_processor_cache=enable_processor_cache, - tokens_per_second=config.vision_config.tokens_per_second, - tokenizer=self.tokenizer, - **processor_kwargs, - ) - self.image_patch_id = self.processor.image_patch_id - self.limit_mm_per_prompt = self._parse_limits(limit_mm_per_prompt) - - def process_request(self, request, max_model_len=None, **kwargs): - """ - Process incoming request and generate model inputs. - - Args: - request: Input request object - max_model_len (int, optional): Maximum context length - **kwargs: Additional processing parameters - - Returns: - Request: Processed request with model inputs - """ - task = request.to_dict() - task["enable_thinking"] = kwargs.get("enable_thinking", False) - self.process_request_dict(task, max_model_len) - request = Request.from_dict(task) - request = self._apply_default_parameters(request) - return request - - def _parse_processor_kwargs(self, kwargs): - """ - Parse and validate multimodal processor arguments. - - Args: - kwargs (dict): Processor configuration arguments - - Returns: - dict: Validated processor arguments - - Raises: - ValueError: If arguments format is invalid - """ - if not kwargs: - return {} - - try: - if not isinstance(kwargs, dict): - raise ValueError("mm-processor-kwargs must be a dictionary") - - # Validate kwargs types against expected schema - data_processor_logger.info(f"Processing kwargs: {kwargs}") - expected_types = { - "video_max_frames": int, # Maximum video frames parameter - "video_min_frames": int, # Minimum video frames parameter - } - - for key, value in kwargs.items(): - if key in expected_types and not isinstance(value, expected_types[key]): - raise ValueError( - f"Invalid type for {key}: expected {expected_types[key].__name__}, got {type(value).__name__}" - ) - - return kwargs - - except Exception as e: - data_processor_logger.warning(f"Invalid mm-processor-kwargs format: {e}") - return {} - - def _parse_limits(self, limits): - """ - Parse and validate multimodal input limits. - - Args: - limits (dict): Input limits configuration - - Returns: - dict: Validated limits with defaults - - Raises: - ValueError: If limits format is invalid - """ - DEFAULT_LIMITS = {"image": 1, "video": 1, "audio": 1} - - if not limits: - return DEFAULT_LIMITS - - try: - if not isinstance(limits, dict): - raise ValueError("limit-mm-per-prompt must be a dictionary") - data_processor_logger.info(f"_parse_limits:{limits}") - return {**DEFAULT_LIMITS, **limits} - except Exception as e: - data_processor_logger.warning(f"Invalid limit-mm-per-prompt format: {e}, using default limits") - return DEFAULT_LIMITS - - def _check_mm_limits(self, item): - """ - Validate multimodal inputs against configured limits. - - Args: - item: Input request item to validate - - Raises: - ValueError: If input exceeds configured limits - """ - if isinstance(item, dict): - # 请求包含prompt和multi_modal_data - mm_data = item - else: - # 请求包含messages - mm_data = {"image": [], "video": []} - - for message in item: - if isinstance(message.get("content"), list): - for part in message["content"]: - if part.get("type") in ["image_url", "image"]: - mm_data["image"].append(part) - elif part.get("type") in ["video_url", "video"]: - mm_data["video"].append(part) - - for modality, data in mm_data.items(): - if modality in self.limit_mm_per_prompt: - limit = self.limit_mm_per_prompt[modality] - if len(data) > limit: - raise ValueError(f"Too many {modality} items in prompt, " f"got {len(data)} but limit is {limit}") - - def process_request_dict(self, request, max_model_len=None, **kwargs): - """ - Process request dictionary into model inputs. - - Args: - request (dict): Input request dictionary - max_model_len (int, optional): Maximum context length - - Returns: - dict: Processed request with model inputs - - Raises: - ValueError: If request format is invalid - """ - - request = self._apply_default_parameters(request) - if not request.eos_token_ids: - request.eos_token_ids = self.eos_token_ids - - # processing stop_sequences and stop_token_ids - process_stop_token_ids(request, self.update_stop_seq) - - if request.prompt: - multimodal_data = request.multimodal_data - if multimodal_data is None: - multimodal_data = {} - self._check_mm_limits(multimodal_data) - images = multimodal_data.get("image", None) - videos = multimodal_data.get("video", None) - outputs = self.processor.text2ids(request.prompt, images, videos) - - elif request.messages: - messages = request.messages - self._check_mm_limits(messages) - outputs = self.processor.request2ids(request) - - else: - raise ValueError(f"Request must contain 'prompt', or 'messages': {request}") - - metadata = request.metadata - # Handle continuation of previous generation by appending existing tokens - if metadata and metadata.get("generated_token_ids"): - self.append_generated_tokens(outputs, metadata["generated_token_ids"]) - outputs = self.pack_outputs(outputs) - - request.prompt_token_ids = outputs["input_ids"].tolist() - request.prompt_token_ids_len = len(request.prompt_token_ids) - request.multimodal_inputs = outputs - - # Handle prompt truncation if exceeds model context length - if max_model_len is not None and len(request.prompt_token_ids) > max_model_len: - request.prompt_token_ids = request.prompt_token_ids[ - : max_model_len - 1 - ] # Leave space for at least 1 new token - - # Set default max_tokens if not specified - max_tokens = max_model_len - len(request.prompt_token_ids) - if getattr(request.sampling_params, "max_tokens", None) is None: - request.sampling_params.max_tokens = max(1, max_tokens) - else: - request.sampling_params.max_tokens = min(max_tokens, request.sampling_params.max_tokens) - - if request.sampling_params.top_p is not None and request.sampling_params.top_p < _SAMPLING_EPS: - request.sampling_params.top_p = _SAMPLING_EPS - request.sampling_params.top_k = 1 - - if self.reasoning_parser: - model_status = self.reasoning_parser.get_model_status(request.prompt_token_ids) - parts = request.request_id.split("_") - if len(parts) > 1: - real_req_id = parts[0] - index = int(parts[1]) - n = request.get("n", 1) - for idx in range(index * n, (index + 1) * n): - self.model_status_dict[f"{real_req_id}_{idx}"] = model_status - else: - self.model_status_dict[request.request_id] = model_status - request.enable_thinking = model_status == "think_start" - - return request - - def append_generated_tokens(self, multimodal_inputs, generated_token_ids): - """ - Append generated tokens to existing outputs. - - Args: - outputs: Current model outputs - generated_token_ids: Generated tokens to append - """ - num_tokens = len(generated_token_ids) - multimodal_inputs["input_ids"].extend(generated_token_ids) - multimodal_inputs["token_type_ids"].extend([0] * num_tokens) - - pos_ids = self.processor._compute_text_positions(multimodal_inputs["cur_position"], num_tokens) - multimodal_inputs["position_ids"].append(pos_ids) - multimodal_inputs["cur_position"] += num_tokens - - def pack_outputs(self, outputs): - """ - Prepare final output dictionary for model. - - Args: - outputs: Intermediate processing outputs - - Returns: - dict: Packed output dictionary with all required fields - """ - if not outputs["images"]: - outputs["images"] = None # No images case - outputs["grid_thw"] = None # No spatial dimensions - outputs["image_type_ids"] = None # No type IDs - else: - outputs["images"] = np.vstack(outputs["images"]) # Stack image features vertically - outputs["grid_thw"] = np.vstack(outputs["grid_thw"]) # Stack spatial dimensions - outputs["image_type_ids"] = np.array(outputs["image_type_ids"]) # Convert to numpy array - - # Convert all outputs to numpy arrays with appropriate types - outputs["input_ids"] = np.array(outputs["input_ids"], dtype=np.int64) # Token IDs as int64 - outputs["token_type_ids"] = np.array(outputs["token_type_ids"], dtype=np.int64) # Type IDs as int64 - outputs["position_ids"] = np.concatenate( - outputs["position_ids"], axis=1, dtype=np.int64 - ) # Concatenate position ID - - outputs["image_patch_id"] = self.processor.image_token_id - outputs["video_patch_id"] = self.processor.video_token_id - outputs["position_ids"] = outputs["position_ids"].transpose(1, 0) - outputs["mm_num_token_func"] = self.processor.mm_num_tokens - return outputs diff --git a/fastdeploy/input/v1/paddleocr_vl_processor/process.py b/fastdeploy/input/v1/paddleocr_vl_processor/process.py deleted file mode 100644 index ea6b63ee9d8..00000000000 --- a/fastdeploy/input/v1/paddleocr_vl_processor/process.py +++ /dev/null @@ -1,622 +0,0 @@ -""" -# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -import pickle -from typing import Dict, List, Optional, Tuple, Union - -import numpy as np -import paddle -import zmq -from paddleformers.transformers import AutoTokenizer -from PIL import Image - -from fastdeploy.engine.request import ImagePosition, Request -from fastdeploy.entrypoints.chat_utils import parse_chat_messages -from fastdeploy.input.ernie4_5_vl_processor import read_video_decord -from fastdeploy.input.mm_data_processor import MMBaseDataProcessor -from fastdeploy.input.utils import IDS_TYPE_FLAG -from fastdeploy.multimodal.hasher import MultimodalHasher -from fastdeploy.utils import data_processor_logger - -from .image_processor import ImageProcessor -from .process_video import sample_frames - - -class DataProcessor(MMBaseDataProcessor): - """ - Processes multimodal inputs (text, images, videos) into model-ready formats. - - Handles: - - Tokenization of text with special tokens for visual content - - Image and video preprocessing - - Generation of 3D positional embeddings - - Conversion of chat messages to model inputs - - Attributes: - tokenizer: Text tokenizer instance - image_processor: Image/video preprocessor - image_token: Special token for image placeholders - video_token: Special token for video placeholders - vision_start: Token marking start of visual content - """ - - def __init__( - self, - model_path: str, - enable_processor_cache: bool = False, - video_min_frames: int = 4, - video_max_frames: int = 768, - video_target_frames: int = -1, - video_fps: int = -1, - tokens_per_second: int = 2, - tokenizer=None, - **kwargs, - ) -> None: - """ - Initialize the data processor. - - Args: - model_path: Path to pretrained model - video_min_frames: Minimum frames to sample from videos - video_max_frames: Maximum frames to sample from videos - tokens_per_second: Temporal resolution for positional embeddings - **kwargs: Additional configuration - """ - super().__init__() - self.min_frames = video_min_frames - self.max_frames = video_max_frames - self.target_frames = video_target_frames - self.fps = video_fps - - # Initialize tokenizer with left padding and fast tokenizer - if tokenizer is None: - self.tokenizer = AutoTokenizer.from_pretrained(model_path, padding_side="left", use_fast=True) - self.tokenizer.ignored_index = -100 # Set ignored index for loss calculation - else: - self.tokenizer = tokenizer - self.image_processor = ImageProcessor.from_pretrained(model_path) # Initialize image processor - self.enable_processor_cache = enable_processor_cache - - # Convolution sizes for patch aggregation - self.spatial_conv_size = self.image_processor.merge_size - self.temporal_conv_size = self.image_processor.temporal_patch_size - - # Special tokens and IDs - self.image_token = "<|IMAGE_PLACEHOLDER|>" - self.video_token = "<|video_pad|>" - - self.image_token_id = self.tokenizer.convert_tokens_to_ids(self.image_token) - self.video_token_id = self.tokenizer.convert_tokens_to_ids(self.video_token) - self.image_patch_id = self.image_token_id - - self.vision_start = "<|IMAGE_START|>" - self.vision_start_id = self.tokenizer.convert_tokens_to_ids(self.vision_start) - - self.tokens_per_second = tokens_per_second - - self.role_prefixes = { - "system": "", - "user": "User: ", - "bot": "Assistant: ", - "assistant": "Assistant: ", - } - - @staticmethod - def mm_num_tokens(grid_thw: list | list[list[int]] | np.ndarray | paddle.Tensor) -> int | list[int]: - """ - Calculate the number of tokens in the multimodal input. - """ - if isinstance(grid_thw, paddle.Tensor): - grid_thw = grid_thw.numpy() - - if len(grid_thw) == 0: - return 0 - - def calc_one(thw): - t, h, w = map(int, thw) - return t * h * w // 4 - - if isinstance(grid_thw[0], (list, tuple, np.ndarray)): - return [calc_one(x) for x in grid_thw] - - return calc_one(grid_thw) - - def text2ids(self, text, images=None, videos=None, image_uuid=None, video_uuid=None): - """ - Convert text with image/video placeholders into model inputs. - - Args: - text: Input text with <|image@placeholder|> and <|video@placeholder|> markers - images: List of PIL Images corresponding to image placeholders - videos: List of video data corresponding to video placeholders - image_uuid: List of unique identifiers for each image, used for caching or hashing. - video_uuid: List of unique identifiers for each video, used for caching or hashing. - - Returns: - Dict containing: - - input_ids: Token IDs - - token_type_ids: Type identifiers (text/image/video) - - position_ids: 3D positional embeddings - - images: Preprocessed visual features - - grid_thw: Spatial/temporal dimensions - - image_type_ids: Visual content type (0=image, 1=video) - """ - - outputs = { - "input_ids": [], - "token_type_ids": [], - "position_ids": [], - "images": [], - "grid_thw": [], - "image_type_ids": [], - "labels": [], - "cur_position": 0, - "video_cnt": 0, - "num_input_image_tokens": 0, - "num_input_video_tokens": 0, - "fps": [], - "mm_positions": [], - "mm_hashes": [], - "vit_seqlen": [], - "vit_position_ids": [], - } - - # Define placeholders and their lengths - IMAGE_PLACEHOLDER = self.image_token - VIDEO_PLACEHOLDER = self.video_token - IMAGE_PLACEHOLDER_LEN = len(IMAGE_PLACEHOLDER) - VIDEO_PLACEHOLDER_LEN = len(VIDEO_PLACEHOLDER) - - # Initialize tracking variables for text parsing - st, image_idx, video_idx = 0, 0, 0 # Start position, image counter, video counter - while st < len(text): - # Find next image or video placeholder in text - image_pos = text.find(IMAGE_PLACEHOLDER, st) - image_pos = len(text) if image_pos == -1 else image_pos # Set to end if not found - video_pos = text.find(VIDEO_PLACEHOLDER, st) - video_pos = len(text) if video_pos == -1 else video_pos # Set to end if not found - ed = min(image_pos, video_pos) # End position is first placeholder found - - self._add_text(text[st:ed], outputs) - if ed == len(text): - break - - if ed == image_pos: - image = images[image_idx] - uuid = image_uuid[image_idx] if image_uuid else None - if not isinstance(image, tuple): - self._add_image(image, outputs, uuid) - else: - self._add_processed_image(image, outputs, uuid) - image_idx += 1 - st = ed + IMAGE_PLACEHOLDER_LEN - else: - item = videos[video_idx] - uuid = video_uuid[video_idx] if video_uuid else None - if not isinstance(item, tuple): - if isinstance(item, dict): - frames, meta = self._load_and_process_video(item["video"], item) - else: - frames, meta = self._load_and_process_video(item, {}) - self._add_video(frames, meta, outputs, uuid) - else: - # cached frames are already processed - self._add_processed_video(item, outputs, uuid) - video_idx += 1 - st = ed + VIDEO_PLACEHOLDER_LEN - - return outputs - - def request2ids( - self, request: Request, tgts: List[str] = None - ) -> Dict[str, Union[np.ndarray, List[np.ndarray], None]]: - """ - Convert chat request with multimodal messages into model inputs. - - Args: - request: Dictionary containing: - - messages: List of chat messages with text/image/video content - - request_id: Unique identifier for logging - tgts: Optional target sequences - - Returns: - Dict with same structure as text2ids() output - """ - - # Parse and validate chat messages - messages = parse_chat_messages(request.messages) - mm_items = [] - for msg in messages: - role = msg.get("role") - assert role in self.role_prefixes, f"Unsupported role: {role}" - - # Normalize content to list format - content = msg.get("content") - if not isinstance(content, list): - content = [content] - # Collect all visual content items - for item in content: - if item.get("type") in ["image", "video"]: - mm_items.append(item) - - missing_hashes, missing_idx = [], [] - for idx, item in enumerate(mm_items): - if not item.get("data"): - # raw data not provided, should be retrieved from processor cache - missing_hashes.append(item.get("uuid")) - missing_idx.append(idx) - - if len(missing_hashes) > 0 and not self.enable_processor_cache: - raise ValueError("Missing items cannot be retrieved without processor cache.") - - if self.enable_processor_cache: - context = zmq.Context() - dealer = context.socket(zmq.DEALER) - dealer.connect("ipc:///dev/shm/processor_cache.ipc") - - missing_items = self.get_processor_cache(dealer, missing_hashes) - for idx in range(len(missing_items)): - if not missing_items[idx]: - raise ValueError(f"Missing item {idx} not found in processor cache") - mm_items[missing_idx[idx]]["data"] = missing_items[idx] - - images, videos = [], [] - image_uuid, video_uuid = [], [] - for item in mm_items: - if item.get("type") == "image": - images.append(item["data"]) - image_uuid.append(item["uuid"]) - elif item.get("type") == "video": - videos.append(item["data"]) - video_uuid.append(item["uuid"]) - else: - raise ValueError(f"Unsupported multimodal type: {item.get('type')}") - - if self.tokenizer.chat_template is None: - raise ValueError("This model does not support chat template.") - - chat_template_kwargs = request.chat_template_kwargs if request.chat_template_kwargs else {} - prompt = self.tokenizer.apply_chat_template( - messages, - tokenize=False, - add_generation_prompt=request.add_generation_prompt if request.add_generation_prompt is not None else True, - **chat_template_kwargs, - ) - request.prompt_tokens = prompt - - outputs = self.text2ids(prompt, images, videos, image_uuid, video_uuid) - - if self.enable_processor_cache: - missing_idx = set(missing_idx) - hashes_to_cache, items_to_cache = [], [] - for idx in range(len(mm_items)): - if idx in missing_idx: - continue - meta = {} - t, h, w = outputs["grid_thw"][idx] - meta["thw"] = (t, h, w) - meta["fps"] = outputs["fps"][idx] - hashes_to_cache.append(outputs["mm_hashes"][idx]) - items_to_cache.append((outputs["images"][idx], meta)) - self.update_processor_cache(dealer, hashes_to_cache, items_to_cache) - - return outputs - - def _add_text(self, tokens, outputs: Dict) -> None: - """ - Add text tokens to model inputs dictionary. - - Args: - tokens: Text string or already tokenized IDs - outputs: Dictionary accumulating model inputs - - Note: - - Handles both raw text and pre-tokenized inputs - - Updates position IDs for 3D embeddings - """ - if not tokens: - return None - - if isinstance(tokens, str): - tokens_str = self.tokenizer.tokenize(tokens) - tokens = self.tokenizer.convert_tokens_to_ids(tokens_str) - - num_tokens = len(tokens) - outputs["input_ids"].extend(tokens) - outputs["token_type_ids"].extend([IDS_TYPE_FLAG["text"]] * num_tokens) - - pos_ids = self._compute_text_positions(outputs["cur_position"], num_tokens) - outputs["position_ids"].append(pos_ids) - outputs["cur_position"] = pos_ids.max() + 1 - - def _compute_text_positions(self, start_pos: int, num_tokens: int) -> np.ndarray: - """ - Generate 3D positional embeddings for text tokens. - - Args: - start_pos: Starting position index - num_tokens: Number of tokens to generate positions for - - Returns: - numpy.ndarray: 3D position IDs shaped (3, num_tokens) - """ - text_array = np.arange(num_tokens).reshape(1, -1) - text_index = np.broadcast_to(text_array, (3, num_tokens)) - position = text_index + start_pos - return position - - def _add_image(self, img, outputs: Dict, uuid: Optional[str]) -> None: - """ - Add image data to model inputs dictionary. - - Args: - img: PIL Image to process - outputs: Dictionary accumulating model inputs - - Note: - - Preprocesses image and calculates spatial dimensions - - Adds image token IDs and type markers - - Generates appropriate position embeddings - """ - ret = self.image_processor.preprocess(images=[img.convert("RGB")]) - num_tokens = ret["grid_thw"].prod() // self.image_processor.merge_size**2 - grid_thw = ret["grid_thw"].tolist() - - outputs["mm_positions"].append(ImagePosition(len(outputs["input_ids"]), num_tokens)) - outputs["input_ids"].extend([self.image_token_id] * num_tokens) - outputs["token_type_ids"].extend([IDS_TYPE_FLAG["image"]] * num_tokens) - outputs["num_input_image_tokens"] += int(num_tokens) - - outputs["images"].append(ret["pixel_values"]) - if not uuid: - outputs["mm_hashes"].append(MultimodalHasher.hash_features(ret["pixel_values"])) - else: - outputs["mm_hashes"].append(uuid) - outputs["grid_thw"].append(grid_thw) - outputs["image_type_ids"].append(0) - - # position_ids - t, h, w = grid_thw - pos_ids = self._compute_vision_positions(outputs["cur_position"], t, h, w, 0) - outputs["position_ids"].append(pos_ids) - outputs["cur_position"] = pos_ids.max() + 1 - outputs["fps"].append(0) - numel = h * w - outputs["vit_seqlen"].append(numel) - outputs["vit_position_ids"].append(np.arange(numel) % numel) - - def _add_processed_image(self, img_cache: Tuple[np.ndarray, dict], outputs: Dict, uuid: str) -> None: - img, meta = img_cache - num_tokens = img.shape[0] // self.image_processor.merge_size**2 - - outputs["mm_positions"].append(ImagePosition(len(outputs["input_ids"]), num_tokens)) - outputs["input_ids"].extend([self.image_patch_id] * num_tokens) - outputs["token_type_ids"].extend([IDS_TYPE_FLAG["image"]] * num_tokens) - - _, h, w = meta["thw"] - pos_ids = self._compute_vision_positions(outputs["cur_position"], 1, h, w, 0) - outputs["position_ids"].append(pos_ids) - outputs["cur_position"] = pos_ids.max() + 1 - - outputs["images"].append(img) - outputs["mm_hashes"].append(uuid) - outputs["grid_thw"].append(np.array([[1, h, w]])) - outputs["image_type_ids"].append(0) - - outputs["fps"].append(0) - - def _add_video(self, frames, meta: Dict, outputs: Dict, uuid: Optional[str]) -> None: - """ - Add video data to model inputs dictionary. - - Args: - frames: Video frames as numpy array - meta: Video metadata containing fps/duration - outputs: Dictionary accumulating model inputs - - Note: - - Handles temporal dimension in position embeddings - - Uses video-specific token IDs and type markers - """ - ret = self.image_processor.preprocess(images=frames) - - num_tokens = ret["image_grid_thw"].prod() // self.image_processor.merge_size**2 - grid_thw = ret["image_grid_thw"].tolist() - - outputs["mm_positions"].append(ImagePosition(len(outputs["input_ids"]), num_tokens)) - outputs["input_ids"].extend([self.video_token_id] * num_tokens) - outputs["token_type_ids"].extend([IDS_TYPE_FLAG["video"]] * num_tokens) - outputs["num_input_video_tokens"] += int(num_tokens) - - outputs["images"].append(ret["pixel_values"]) - if not uuid: - outputs["mm_hashes"].append(MultimodalHasher.hash_features(ret["pixel_values"])) - else: - outputs["mm_hashes"].append(uuid) - outputs["grid_thw"].append(grid_thw) - outputs["image_type_ids"].extend([1] * grid_thw[0]) - - fps = meta["fps"] - second_per_grid_t = self.temporal_conv_size / fps - t, h, w = grid_thw - pos_ids = self._compute_vision_positions(outputs["cur_position"], t, h, w, second_per_grid_t) - - outputs["position_ids"].append(pos_ids) - outputs["cur_position"] = pos_ids.max() + 1 - outputs["fps"].append(fps) - numel = h * w - outputs["vit_seqlen"].append(numel) - outputs["vit_position_ids"].append(np.arange(numel) % numel) - - def _add_processed_video(self, frames_cache: Tuple[np.ndarray, dict], outputs: Dict, uuid: str) -> None: - frames, meta = frames_cache - num_tokens = frames.shape[0] // self.image_processor.merge_size**2 - - t, h, w = meta["thw"] - outputs["images"].append(frames) - outputs["mm_hashes"].append(uuid) - outputs["grid_thw"].append(np.array([[t, h, w]])) - - outputs["mm_positions"].append(ImagePosition(len(outputs["input_ids"]), num_tokens)) - outputs["input_ids"].extend([self.image_patch_id] * num_tokens) - outputs["token_type_ids"].extend([IDS_TYPE_FLAG["video"]] * num_tokens) - outputs["image_type_ids"].extend([1] * t) - - fps = meta["fps"] - second_per_grid_t = self.temporal_conv_size / fps - pos_ids = self._compute_vision_positions(outputs["cur_position"], t, h, w, second_per_grid_t) - outputs["position_ids"].append(pos_ids) - outputs["cur_position"] = pos_ids.max() + 1 - - outputs["fps"].append(fps) - - def _compute_vision_positions( - self, start_pos: int, t: int, h: int, w: int, second_per_grid_t: float - ) -> np.ndarray: - """ - Generate 3D position IDs for visual inputs. - - Args: - start_pos: Base position in sequence - t: Temporal patches (1 for images) - h: Height in patches - w: Width in patches - second_per_grid_t: Time per temporal patch - - Returns: - np.ndarray: Position IDs for [t,h,w] dimensions - """ - h //= self.spatial_conv_size - w //= self.spatial_conv_size - - tn = np.arange(t).reshape(-1, 1) - tn = np.broadcast_to(tn, (t, h * w)) - tn = tn * int(second_per_grid_t) * self.tokens_per_second - t_index = tn.flatten() - - hn = np.arange(h).reshape(1, -1, 1) - h_index = np.broadcast_to(hn, (t, h, w)).flatten() - - wn = np.arange(w).reshape(1, 1, -1) - w_index = np.broadcast_to(wn, (t, h, w)).flatten() - - position = np.stack([t_index, h_index, w_index]) + start_pos - return position - - def _load_and_process_video(self, url: str, item: Dict) -> Tuple[np.ndarray, Dict]: - """ - Load and preprocess video into frames. - - Args: - url: Video file path or bytes - item: Dictionary containing processing parameters - - Returns: - tuple: (frames, metadata) where: - - frames: Processed video frames as numpy array - - metadata: Updated video metadata dictionary - """ - reader, meta, _ = read_video_decord(url, save_to_disk=False) - - # Apply frame sampling if fps or target_frames specified - fps = item.get("fps", self.fps) - num_frames = item.get("target_frames", self.target_frames) - - frame_indices = list(range(meta["num_of_frame"])) - if fps > 0 or num_frames > 0: - # Get frame sampling constraints - min_frames = item.get("min_frames", self.min_frames) - max_frames = item.get("max_frames", self.max_frames) - - # Sample frames according to specifications - frame_indices = sample_frames( - frame_factor=self.temporal_conv_size, # Ensure divisible by temporal patch size - min_frames=min_frames, - max_frames=max_frames, - metadata=meta, - fps=fps, - num_frames=num_frames, - ) - - # Update metadata with new frame count and fps - meta["num_of_frame"] = len(frame_indices) - if fps is not None: - meta["fps"] = fps # Use specified fps - meta["duration"] = len(frame_indices) / fps - else: - meta["fps"] = len(frame_indices) / meta["duration"] # Calculate fps from sampled frames - - frames = [] - for idx in frame_indices: - frame = reader[idx].asnumpy() - image = Image.fromarray(frame, "RGB") - frames.append(image) - frames = np.stack([np.array(f.convert("RGB")) for f in frames], axis=0) - - return frames, meta - - def get_processor_cache(self, socket, mm_hashes: list[str]) -> list: - """ - get cache correspond to given hash values - """ - req = pickle.dumps(mm_hashes) - socket.send_multipart([b"", req]) - _, resp = socket.recv_multipart() - mm_items = pickle.loads(resp) - data_processor_logger.info(f"Get cache of mm_hashes: {mm_hashes}") - - return mm_items - - def update_processor_cache(self, socket, mm_hashes: list[str], mm_items): - """ - update cache data - """ - req = pickle.dumps((mm_hashes, mm_items)) - socket.send_multipart([b"", req]) - data_processor_logger.info(f"Update cache of mm_hashes: {mm_hashes}") - - def apply_chat_template(self, request): - """ - Apply chat template to convert messages into token sequence. - - Args: - request: Dictionary containing chat messages - - Returns: - List of token IDs - - Raises: - ValueError: If model doesn't support chat templates - """ - if self.tokenizer.chat_template is None: - raise ValueError("This model does not support chat_template.") - - raw_prompt = self.tokenizer.apply_chat_template( - request["messages"], - tokenize=False, - add_generation_prompt=request.get("add_generation_prompt", True), - chat_template=request.get("chat_template", None), - ) - prompt_token_str = raw_prompt.replace(self.image_token, "").replace(self.video_token, "") - request["text_after_process"] = raw_prompt - - tokens = self.tokenizer.tokenize(prompt_token_str) - token_ids = self.tokenizer.convert_tokens_to_ids(tokens) - data_processor_logger.info( - f"req_id:{request.get('request_id', ''), } prompt: {raw_prompt} tokens: {tokens}, token_ids: {token_ids}" - ) - return token_ids diff --git a/fastdeploy/input/v1/paddleocr_vl_processor/process_video.py b/fastdeploy/input/v1/paddleocr_vl_processor/process_video.py deleted file mode 100644 index c7089d26dc2..00000000000 --- a/fastdeploy/input/v1/paddleocr_vl_processor/process_video.py +++ /dev/null @@ -1,82 +0,0 @@ -""" -# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -import math -from typing import Optional, Union - -import numpy as np - - -def sample_frames( - frame_factor: int, - min_frames: int, - max_frames: int, - metadata: Optional[dict] = None, - fps: Optional[Union[int, float]] = None, - num_frames: Optional[int] = None, -): - """ - Sample frames from video according to specified criteria. - - Args: - frame_factor: Ensure sampled frames are multiples of this factor - min_frames: Minimum number of frames to sample - max_frames: Maximum number of frames to sample - metadata: Video metadata containing fps information - fps: Target frames per second for sampling - num_frames: Exact number of frames to sample - - Returns: - np.ndarray: Sampled video frames - - Raises: - ValueError: If both fps and num_frames are specified, - or if required metadata is missing, - or if requested frames exceed available frames - """ - if fps > 0 and num_frames > 0: - raise ValueError("`num_frames` and `fps` are mutually exclusive arguments, please use only one!") - - total_num_frames = metadata["num_of_frame"] - - # If num_frames is not given but fps is, calculate num_frames from fps - if num_frames > 0: - num_frames = round(num_frames / frame_factor) * frame_factor - elif fps > 0: - if metadata is None: - raise ValueError( - "Asked to sample `fps` frames per second but no video metadata was provided which is required when sampling with `fps`. " - "Please pass in `VideoMetadata` object or use a fixed `num_frames` per input video" - ) - max_frames = math.floor(min(max_frames, total_num_frames) / frame_factor) * frame_factor - num_frames = total_num_frames / metadata["fps"] * fps - num_frames = min(min(max(num_frames, min_frames), max_frames), total_num_frames) - num_frames = math.floor(num_frames / frame_factor) * frame_factor - if num_frames > total_num_frames: - raise ValueError( - f"Video can't be sampled. The inferred `num_frames={num_frames}` exceeds `total_num_frames={total_num_frames}`. " - "Decrease `num_frames` or `fps` for sampling." - ) - - # Calculate frame indices based on sampling strategy - if num_frames > 0: - # Evenly spaced sampling for target frame count - indices = np.arange(0, total_num_frames, total_num_frames / num_frames).astype(np.int32) - else: - # Keep all frames if no sampling requested - indices = np.arange(0, total_num_frames).astype(np.int32) - - return indices diff --git a/fastdeploy/input/v1/qwen3_vl_processor/__init__.py b/fastdeploy/input/v1/qwen3_vl_processor/__init__.py deleted file mode 100644 index 9f959610c8d..00000000000 --- a/fastdeploy/input/v1/qwen3_vl_processor/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -""" -# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -from .process import DataProcessor -from .qwen3_vl_processor import Qwen3VLProcessor - -__all__ = [ - "DataProcessor", - "Qwen3VLProcessor", -] diff --git a/fastdeploy/input/v1/qwen3_vl_processor/image_processor.py b/fastdeploy/input/v1/qwen3_vl_processor/image_processor.py deleted file mode 100644 index 167f3e340db..00000000000 --- a/fastdeploy/input/v1/qwen3_vl_processor/image_processor.py +++ /dev/null @@ -1,413 +0,0 @@ -""" -# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -import math -from typing import List, Optional, Union - -import numpy as np -import paddle -import PIL -from paddleformers.transformers.feature_extraction_utils import BatchFeature -from paddleformers.transformers.image_processing_utils import BaseImageProcessor -from paddleformers.transformers.image_transforms import ( - normalize, - rescale, - resize, - to_channel_dimension_format, -) -from paddleformers.transformers.image_utils import ( - ChannelDimension, - ImageInput, - PILImageResampling, - get_image_size, - infer_channel_dimension_format, - make_list_of_images, - to_numpy_array, - valid_images, -) -from paddleformers.transformers.legacy.tokenizer_utils_base import TensorType -from PIL import Image - -from fastdeploy.utils import data_processor_logger - -IMAGE_MEAN = [0.5, 0.5, 0.5] -IMAGE_STD = [0.5, 0.5, 0.5] - -MIN_PIXELS = 65536 -MAX_PIXELS = 16777216 - - -VideoInput = Union[ - List["PIL.Image.Image"], - "np.ndarray", - "paddle.Tensor", - List["np.ndarray"], - List["paddle.Tensor"], - List[List["PIL.Image.Image"]], - List[List["np.ndarray"]], - List[List["paddle.Tensor"]], -] - - -def round_by_factor(number: int, factor: int) -> int: - return round(number / factor) * factor - - -def ceil_by_factor(number: int, factor: int) -> int: - return math.ceil(number / factor) * factor - - -def floor_by_factor(number: int, factor: int) -> int: - return math.floor(number / factor) * factor - - -def smart_resize(height: int, width: int, factor: int, min_pixels: int, max_pixels: int, max_ratio: int = 200): - """ - Smart image resizing that maintains aspect ratio and respects constraints. - - Args: - height: Original image height - width: Original image width - factor: Patch size factor - min_pixels: Minimum allowed pixels - max_pixels: Maximum allowed pixels - max_ratio: Maximum allowed aspect ratio - - Returns: - tuple: (new_height, new_width) - - Raises: - ValueError: If calculated dimensions are invalid - """ - if max(height, width) / min(height, width) > max_ratio: - if height > width: - new_width = max(factor, round_by_factor(width, factor)) - new_height = floor_by_factor(new_width * max_ratio, factor) - else: - new_height = max(factor, round_by_factor(height, factor)) - new_width = floor_by_factor(new_height * max_ratio, factor) - - data_processor_logger.info( - f"absolute aspect ratio must be smaller than {max_ratio}, got {max(height, width) / min(height, width)},\ - resize to {max(new_height, new_width) / min(new_height, new_width)}" - ) - - height = new_height - width = new_width - - h_bar = max(factor, round_by_factor(height, factor)) - w_bar = max(factor, round_by_factor(width, factor)) - if h_bar * w_bar > max_pixels: - beta = math.sqrt((height * width) / max_pixels) - h_bar = floor_by_factor(height / beta, factor) - w_bar = floor_by_factor(width / beta, factor) - elif h_bar * w_bar < min_pixels: - beta = math.sqrt(min_pixels / (height * width)) - h_bar = ceil_by_factor(height * beta, factor) - w_bar = ceil_by_factor(width * beta, factor) - - if min_pixels > h_bar * w_bar or h_bar * w_bar > max_pixels: - raise ValueError(f"encounter invalid h_bar: {h_bar}, w_bar: {w_bar}") - - return h_bar, w_bar - - -def is_scaled_image(image: np.ndarray) -> bool: - """ - Check if image pixel values are already normalized to [0, 1] range. - - Args: - image: Input image array - - Returns: - bool: True if image is already scaled - """ - if image.dtype == np.uint8: - return False - - # It's possible the image has pixel values in [0, 255] but is of floating type - return np.min(image) >= 0 and np.max(image) <= 1 - - -class ImageProcessor(BaseImageProcessor): - """ - Adaptive image processor for dynamic image resizing and preprocessing. - - This processor handles image resizing, rescaling, normalization and format conversion. - It dynamically adjusts image dimensions based on original size and specified constraints. - """ - - def __init__( - self, - patch_size: int = 16, - merge_size: int = 2, - temporal_patch_size: int = 2, - min_pixels: int = MIN_PIXELS, - max_pixels: int = MAX_PIXELS, - image_mean: Union[float, List[float]] = IMAGE_MEAN, - image_std: Union[float, List[float]] = IMAGE_STD, - rescale_factor: float = 1 / 255, - do_rescale: bool = True, - do_normalize: bool = True, - resample: PILImageResampling = PILImageResampling.BICUBIC, - **kwargs, - ) -> None: - """ - Initialize image processor with configuration parameters. - - Args: - patch_size (int): Spatial patch size for vision encoder - merge_size (int): Merge size between vision and LLM encoders - temporal_patch_size (int): Temporal patch size for video processing - min_pixels (int): Minimum allowed pixels in resized image - max_pixels (int): Maximum allowed pixels in resized image - image_mean (float/list): Mean values for normalization per channel - image_std (float/list): Std values for normalization per channel - rescale_factor (float): Scaling factor for pixel values (default 1/255) - do_rescale (bool): Whether to rescale images - do_normalize (bool): Whether to normalize images - resample: Resampling method for image resizing - **kwargs: Additional base class arguments - """ - super().__init__(**kwargs) - self.patch_size = patch_size - self.merge_size = merge_size - self.temporal_patch_size = temporal_patch_size - - self.min_pixels = min_pixels - self.max_pixels = max_pixels - - self.image_mean = image_mean - self.image_std = image_std - self.rescale_factor = rescale_factor - self.do_rescale = do_rescale - self.do_normalize = do_normalize - - self.resample = resample - - def _preprocess( - self, - images: Union[ImageInput, VideoInput], - min_pixels: int, - max_pixels: int, - image_mean: Optional[Union[float, List[float]]], - image_std: Optional[Union[float, List[float]]], - rescale_factor: float, - do_rescale: bool, - do_normalize: bool, - resample: PILImageResampling, - data_format: Optional[ChannelDimension], - input_data_format: Optional[Union[str, ChannelDimension]], - ): - """ - Internal method for image preprocessing pipeline. - - Args: - images: Input image or batch of images - min_pixels: Minimum allowed pixels in output - max_pixels: Maximum allowed pixels in output - image_mean: Normalization mean values - image_std: Normalization std values - rescale_factor: Pixel value scaling factor - do_rescale: Whether to rescale pixel values - do_normalize: Whether to normalize pixel values - resample: Resampling method - data_format: Output channel format - input_data_format: Input channel format - - Returns: - tuple: (flatten_patches, grid_dimensions) - - flatten_patches: Flattened image patches - - grid_dimensions: Grid dimensions [t, h, w] - """ - images = make_list_of_images(images) - - # All transformations expect numpy arrays. - images = [to_numpy_array(image) for image in images] - - if is_scaled_image(images[0]) and do_rescale: - data_processor_logger.warning( - "It looks like you are trying to rescale already rescaled images. If the input" - " images have pixel values between 0 and 1, set `do_rescale=False` to avoid rescaling them again." - ) - if input_data_format is None: - # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(images[0]) - - # Get original dimensions and calculate optimal resize dimensions - height, width = get_image_size(images[0], channel_dim=input_data_format) - resized_height, resized_width = smart_resize( - height, - width, - factor=self.patch_size * self.merge_size, # Combine patch and merge factors - min_pixels=min_pixels, - max_pixels=max_pixels, - ) - - processed_images = [] - for image in images: - if height != resized_height or width != resized_width: - # Convert to uint8 before resizing to avoid double scaling - image = image.astype("uint8") - # Convert to PIL Image and resize - image = Image.fromarray(image) - image = resize( - image, - size=(resized_height, resized_width), - resample=resample, - data_format=input_data_format, - ) - - if do_rescale and do_normalize: - # Adjust mean and std for combined rescale+normalize - image_mean = np.array(image_mean, dtype=np.float32) * (1.0 / rescale_factor) - image_std = np.array(image_std, dtype=np.float32) * (1.0 / rescale_factor) - do_rescale = False # Skip separate rescale step - - # mutual exclusion and upper branch - if do_rescale: - image = image.astype(np.float32) - image = rescale(image, scale=rescale_factor, data_format=input_data_format) - - if do_normalize: - image = image.astype(np.float32) - image = normalize( - image=image, - mean=image_mean, - std=image_std, - data_format=input_data_format, - ) - - image = to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) # [C, H, W] - processed_images.append(image) - - # Convert processed images to numpy array - patches = np.array(processed_images) - - # Pad temporal dimension if needed - if patches.shape[0] % self.temporal_patch_size != 0: - repeats = np.repeat( - patches[-1][np.newaxis], - self.temporal_patch_size - (patches.shape[0] % self.temporal_patch_size), - axis=0, - ) - patches = np.concatenate([patches, repeats], axis=0) - - # Convert to channels-first format if needed - if data_format == ChannelDimension.LAST: - patches = patches.transpose([0, 3, 1, 2]) # [N, H, W, C] -> [N, C, H, W] - - grid_t, channel = patches.shape[:2] - grid_t = grid_t // self.temporal_patch_size - - grid_h, grid_w = ( - resized_height // self.patch_size, - resized_width // self.patch_size, - ) - # Reshape into hierarchical patch structure - patches = patches.reshape( - [ - grid_t, - self.temporal_patch_size, - channel, - grid_h // self.merge_size, - self.merge_size, - self.patch_size, - grid_w // self.merge_size, - self.merge_size, - self.patch_size, - ] - ) - # Reorder dimensions for better memory access pattern - # [grid_t, grid_h/merge_size, grid_w/merge_size, merge_size, merge_size, C, temporal_patch_size, psz, psz] - patches = patches.transpose([0, 3, 6, 4, 7, 2, 1, 5, 8]) - - flatten_patches = patches.reshape( - [ - grid_t * grid_h * grid_w, - channel * self.temporal_patch_size * self.patch_size * self.patch_size, - ] - ) - - return flatten_patches, np.array([grid_t, grid_h, grid_w]) - - def preprocess( - self, - images: Union[ImageInput, VideoInput], - min_pixels: Optional[int] = None, - max_pixels: Optional[int] = None, - image_mean: Optional[Union[float, List[float]]] = None, - image_std: Optional[Union[float, List[float]]] = None, - rescale_factor: Optional[float] = None, - do_rescale: Optional[bool] = None, - do_normalize: Optional[bool] = None, - resample: Optional[PILImageResampling] = None, - return_tensors: Optional[Union[str, TensorType]] = None, - data_format: Optional[ChannelDimension] = ChannelDimension.FIRST, - input_data_format: Optional[Union[str, ChannelDimension]] = ChannelDimension.LAST, - ): - """ - Main preprocessing method for images/videos. - - Args: - images: Input image/video data - min_pixels: Override for minimum pixels - max_pixels: Override for maximum pixels - image_mean: Override for normalization mean - image_std: Override for normalization std - rescale_factor: Override for rescaling factor - do_rescale: Override for rescaling flag - do_normalize: Override for normalization flag - resample: Override for resampling method - return_tensors: Desired output tensor format - data_format: Output channel dimension format - input_data_format: Input channel dimension format - - Returns: - BatchFeature: Processed features containing: - - pixel_values: Preprocessed pixel data - - grid_thw: Grid dimensions [temporal, height, width] - - Raises: - ValueError: For invalid image types or dimensions - """ - min_pixels = min_pixels if min_pixels is not None else self.min_pixels - max_pixels = max_pixels if max_pixels is not None else self.max_pixels - image_mean = image_mean if image_mean is not None else self.image_mean - image_std = image_std if image_std is not None else self.image_std - rescale_factor = rescale_factor if rescale_factor is not None else self.rescale_factor - do_rescale = do_rescale if do_rescale is not None else self.do_rescale - do_normalize = do_normalize if do_normalize is not None else self.do_normalize - resample = resample if resample is not None else self.resample - - if images is not None and not valid_images(images): - raise ValueError("Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, " "paddle.Tensor.") - - pixel_values, grid_thw = self._preprocess( - images, - min_pixels=min_pixels, - max_pixels=max_pixels, - image_mean=image_mean, - image_std=image_std, - rescale_factor=rescale_factor, - do_rescale=do_rescale, - do_normalize=do_normalize, - resample=resample, - data_format=data_format, - input_data_format=input_data_format, - ) - data = {"pixel_values": pixel_values, "grid_thw": grid_thw} - return BatchFeature(data=data, tensor_type=return_tensors) diff --git a/fastdeploy/input/v1/qwen3_vl_processor/process.py b/fastdeploy/input/v1/qwen3_vl_processor/process.py deleted file mode 100644 index 0731d04972d..00000000000 --- a/fastdeploy/input/v1/qwen3_vl_processor/process.py +++ /dev/null @@ -1,814 +0,0 @@ -""" -# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -import pickle -from typing import Dict, List, Optional, Tuple, Union - -import numpy as np -import paddle -import zmq -from paddleformers.transformers import AutoTokenizer -from PIL import Image - -from fastdeploy.engine.request import ImagePosition, Request -from fastdeploy.entrypoints.chat_utils import parse_chat_messages -from fastdeploy.input.ernie4_5_vl_processor import read_video_decord -from fastdeploy.input.mm_data_processor import MMBaseDataProcessor -from fastdeploy.input.utils import IDS_TYPE_FLAG -from fastdeploy.multimodal.hasher import MultimodalHasher -from fastdeploy.utils import data_processor_logger - -from .image_processor import ImageProcessor, ceil_by_factor, floor_by_factor - -VIDEO_MIN_PIXELS = 128 * 28 * 28 -VIDEO_MAX_PIXELS = 768 * 28 * 28 -FRAME_FACTOR = 2 -FPS = 2.0 -FPS_MIN_FRAMES = 4 -FPS_MAX_FRAMES = 768 - - -def sample_frames( - frame_factor: int, - min_frames: int, - max_frames: int, - metadata: Optional[dict] = None, - fps: Optional[Union[int, float]] = -1, - num_frames: Optional[int] = -1, -): - """ - Sample frames from video according to specified criteria. - - Args: - frame_factor: Ensure sampled frames are multiples of this factor - min_frames: Minimum number of frames to sample - max_frames: Maximum number of frames to sample - metadata: Video metadata containing fps information - fps: Target frames per second for sampling - num_frames: Exact number of frames to sample - - Returns: - np.ndarray: Sampled video frames - - Raises: - ValueError: If both fps and num_frames are specified, - or if required metadata is missing, - or if requested frames exceed available frames - """ - if fps > 0 and num_frames > 0: - raise ValueError("`num_frames` and `fps` are mutually exclusive arguments, please use only one!") - - total_num_frames = metadata["num_of_frame"] - - # If num_frames is not given but fps is, calculate num_frames from fps - if num_frames > 0: - num_frames = round(num_frames / frame_factor) * frame_factor - elif fps > 0: - if metadata is None: - raise ValueError( - "Asked to sample `fps` frames per second but no video metadata was provided which is required when sampling with `fps`. " - "Please pass in `VideoMetadata` object or use a fixed `num_frames` per input video" - ) - # max_frames = math.floor(min(max_frames, total_num_frames) / frame_factor) * frame_factor - min_frames = ceil_by_factor(min_frames, frame_factor) - max_frames = floor_by_factor(min(max_frames, total_num_frames), frame_factor) - - num_frames = total_num_frames / metadata["fps"] * fps - - if num_frames > total_num_frames: - data_processor_logger.warning(f"smart_nframes: nframes[{num_frames}] > total_frames[{total_num_frames}]") - - num_frames = min(min(max(num_frames, min_frames), max_frames), total_num_frames) - num_frames = floor_by_factor(num_frames, frame_factor) - - if num_frames > total_num_frames: - raise ValueError( - f"Video can't be sampled. The inferred `num_frames={num_frames}` exceeds `total_num_frames={total_num_frames}`. " - "Decrease `num_frames` or `fps` for sampling." - ) - - # Hack code ensures that num_frames can always be divided by 4 - # due to sched/resource_manager_v1.py 中 grid_thw.extend([[2, h, w]] * (t // 2)) - if num_frames > 2 and num_frames % 4 != 0: - num_frames = (num_frames // 4) * 4 # 向下取整到 4 的倍数 - total_num_frames = (total_num_frames // 4) * 4 - num_frames = min(min(max(num_frames, min_frames), max_frames), total_num_frames) - - # Calculate frame indices based on sampling strategy - if num_frames > 0: - # Evenly spaced sampling for target frame count - indices = np.arange(0, total_num_frames, total_num_frames / num_frames).astype(np.int32) - else: - # Keep all frames if no sampling requested - indices = np.arange(0, total_num_frames).astype(np.int32) - - return indices - - -class DataProcessor(MMBaseDataProcessor): - """ - Processes multimodal inputs (text, images, videos) into model-ready formats. - - Handles: - - Tokenization of text with special tokens for visual content - - Image and video preprocessing - - Generation of 3D positional embeddings - - Conversion of chat messages to model inputs - - Attributes: - tokenizer: Text tokenizer instance - image_processor: Image/video preprocessor - image_token: Special token for image placeholders - video_token: Special token for video placeholders - vision_start: Token marking start of visual content - """ - - def __init__( - self, - model_path: str, - enable_processor_cache: bool = False, - video_min_frames: int = FPS_MIN_FRAMES, - video_max_frames: int = FPS_MAX_FRAMES, - video_target_frames: int = -1, - video_fps: int = FPS, - tokens_per_second: int = 2, - tokenizer=None, - **kwargs, - ) -> None: - """ - Initialize the data processor. - - Args: - model_path: Path to pretrained model - video_min_frames: Minimum frames to sample from videos - video_max_frames: Maximum frames to sample from videos - tokens_per_second: Temporal resolution for positional embeddings - **kwargs: Additional configuration - """ - super().__init__() - self.min_frames = video_min_frames - self.max_frames = video_max_frames - self.target_frames = video_target_frames - self.fps = video_fps - self.frame_factor = FRAME_FACTOR - - # Initialize tokenizer with left padding and fast tokenizer - if tokenizer is None: - self.tokenizer = AutoTokenizer.from_pretrained(model_path, padding_side="left", use_fast=True) - self.tokenizer.ignored_index = -100 # Set ignored index for loss calculation - else: - self.tokenizer = tokenizer - - self.image_processor = ImageProcessor.from_pretrained(model_path) # Initialize image processor - self.enable_processor_cache = enable_processor_cache - - # Convolution sizes for patch aggregation - self.spatial_conv_size = self.image_processor.merge_size - self.temporal_conv_size = self.image_processor.temporal_patch_size - - # Special tokens and IDs - self.image_token = "<|image_pad|>" - self.video_token = "<|video_pad|>" - - self.image_token_id = self.tokenizer.convert_tokens_to_ids(self.image_token) - self.video_token_id = self.tokenizer.convert_tokens_to_ids(self.video_token) - - self.vision_start = "<|vision_start|>" - self.vision_start_id = self.tokenizer.convert_tokens_to_ids(self.vision_start) - - self.tokens_per_second = tokens_per_second - - self.role_prefixes = { - "system": "", - "user": "User: ", - "bot": "Assistant: ", - "assistant": "Assistant: ", - } - - @staticmethod - def mm_num_tokens(grid_thw: list | list[list[int]] | np.ndarray | paddle.Tensor) -> int | list[int]: - """ - Calculate the number of tokens in the multimodal input. - """ - if isinstance(grid_thw, paddle.Tensor): - grid_thw = grid_thw.numpy() - - if len(grid_thw) == 0: - return 0 - - def calc_one(thw): - t, h, w = map(int, thw) - return t * h * w // 4 - - if isinstance(grid_thw[0], (list, tuple, np.ndarray)): - return [calc_one(x) for x in grid_thw] - - return calc_one(grid_thw) - - def text2ids(self, text, images=None, videos=None, image_uuid=None, video_uuid=None): - """ - Convert text with image/video placeholders into model inputs. - - Args: - text: Input text with <|image@placeholder|> and <|video@placeholder|> markers - images: List of PIL Images corresponding to image placeholders - videos: List of video data corresponding to video placeholders - image_uuid: List of unique identifiers for each image, used for caching or hashing. - video_uuid: List of unique identifiers for each video, used for caching or hashing. - - Returns: - Dict containing: - - input_ids: Token IDs - - token_type_ids: Type identifiers (text/image/video) - - position_ids: 3D positional embeddings - - images: Preprocessed visual features - - grid_thw: Spatial/temporal dimensions - - image_type_ids: Visual content type (0=image, 1=video) - """ - - outputs = { - "input_ids": [], - "token_type_ids": [], - "position_ids": [], - "images": [], - "grid_thw": [], - "image_type_ids": [], - "labels": [], - "cur_position": 0, - "video_cnt": 0, - "num_input_image_tokens": 0, - "num_input_video_tokens": 0, - "fps": [], - "mm_positions": [], - "mm_hashes": [], - } - - # Define placeholders and their lengths - IMAGE_PLACEHOLDER = "<|image_pad|>" - VIDEO_PLACEHOLDER = "<|video_pad|>" - IMAGE_PLACEHOLDER_LEN = len(IMAGE_PLACEHOLDER) - VIDEO_PLACEHOLDER_LEN = len(VIDEO_PLACEHOLDER) - - # Initialize tracking variables for text parsing - st, image_idx, video_idx = 0, 0, 0 # Start position, image counter, video counter - while st < len(text): - # Find next image or video placeholder in text - image_pos = text.find(IMAGE_PLACEHOLDER, st) - image_pos = len(text) if image_pos == -1 else image_pos # Set to end if not found - video_pos = text.find(VIDEO_PLACEHOLDER, st) - video_pos = len(text) if video_pos == -1 else video_pos # Set to end if not found - ed = min(image_pos, video_pos) # End position is first placeholder found - - self._add_text(text[st:ed], outputs) - if ed == len(text): - break - - if ed == image_pos: - image = images[image_idx] - uuid = image_uuid[image_idx] if image_uuid else None - if not isinstance(image, tuple): - self._add_image(image, outputs, uuid) - else: - self._add_processed_image(image, outputs, uuid) - image_idx += 1 - st = ed + IMAGE_PLACEHOLDER_LEN - else: - item = videos[video_idx] - uuid = video_uuid[video_idx] if video_uuid else None - if not isinstance(item, tuple): - if isinstance(item, dict): - frames, meta = self._load_and_process_video(item["video"], item) - else: - frames, meta = self._load_and_process_video(item, {}) - self._add_video(frames, meta, outputs, uuid) - else: - # cached frames are already processed - self._add_processed_video(item, outputs, uuid) - video_idx += 1 - st = ed + VIDEO_PLACEHOLDER_LEN - - return outputs - - def prompt_token_ids2outputs( - self, request: Request, tgts: List[str] = None - ) -> Dict[str, Union[np.ndarray, List[np.ndarray], None]]: - outputs = { - "input_ids": [], - "token_type_ids": [], - "position_ids": [], - "images": [], - "grid_thw": [], - "image_type_ids": [], - "labels": [], - "cur_position": 0, - "video_cnt": 0, - "num_input_image_tokens": 0, - "num_input_video_tokens": 0, - "fps": [], - "mm_positions": [], - "mm_hashes": [], - } - prompt_token_ids = request.prompt_token_ids if request.prompt_token_ids else [] - prompt_token_ids_len = len(prompt_token_ids) - - if not request.messages: - self._add_text(prompt_token_ids, outputs) - return outputs - - messages = parse_chat_messages(request.messages) - mm_items = [] - for msg in messages: - role = msg.get("role") - assert role in self.role_prefixes, f"Unsupported role: {role}" - - content = msg.get("content") - if not isinstance(content, list): - content = [content] - for item in content: - if item.get("type") in ["image", "video"]: - mm_items.append(item) - - missing_hashes, missing_idx = [], [] - for idx, item in enumerate(mm_items): - if not item.get("data"): - missing_hashes.append(item.get("uuid")) - missing_idx.append(idx) - - if len(missing_hashes) > 0 and not self.enable_processor_cache: - raise ValueError("Missing items cannot be retrieved without processor cache.") - - dealer = None - if self.enable_processor_cache: - context = zmq.Context() - dealer = context.socket(zmq.DEALER) - dealer.connect("ipc:///dev/shm/processor_cache.ipc") - - missing_items = self.get_processor_cache(dealer, missing_hashes) - for idx in range(len(missing_items)): - if not missing_items[idx]: - raise ValueError(f"Missing item {idx} not found in processor cache") - mm_items[missing_idx[idx]]["data"] = missing_items[idx] - - st, mm_idx = 0, 0 - while st < prompt_token_ids_len: - if prompt_token_ids[st] != self.image_token_id: - cur_idx = st - while cur_idx < prompt_token_ids_len and prompt_token_ids[cur_idx] != self.image_token_id: - cur_idx += 1 - self._add_text(prompt_token_ids[st:cur_idx], outputs) - st = cur_idx - continue - - if mm_idx >= len(mm_items): - raise ValueError("prompt token ids has more multimodal placeholder than in messages") - - cur_idx = st - while cur_idx < prompt_token_ids_len and prompt_token_ids[cur_idx] == self.image_token_id: - cur_idx += 1 - - item = mm_items[mm_idx] - uuid = item.get("uuid") - token_len = cur_idx - st - if item.get("type") == "image": - image = item.get("data") - if not isinstance(image, tuple): - self._add_image(image, outputs, uuid, token_len) - else: - self._add_processed_image(image, outputs, uuid, token_len) - elif item.get("type") == "video": - video = item.get("data") - if not isinstance(video, tuple): - if isinstance(video, dict): - frames, meta = self._load_and_process_video(video["video"], video) - else: - frames, meta = self._load_and_process_video(video, {}) - self._add_video(frames, meta, outputs, uuid, token_len) - else: - self._add_processed_video(video, outputs, uuid, token_len) - else: - raise ValueError(f"Unsupported multimodal type: {item.get('type')}") - mm_idx += 1 - st = cur_idx - - if mm_idx != len(mm_items): - raise ValueError("number of multimodal items does not match prompt token ids") - - if self.enable_processor_cache: - missing_idx = set(missing_idx) - hashes_to_cache, items_to_cache = [], [] - for idx in range(len(mm_items)): - if idx in missing_idx: - continue - meta = {} - grid_thw = np.asarray(outputs["grid_thw"][idx]) - if grid_thw.ndim > 1: - t, h, w = grid_thw[0] - else: - t, h, w = grid_thw - meta["thw"] = (int(t), int(h), int(w)) - meta["fps"] = outputs["fps"][idx] - hashes_to_cache.append(outputs["mm_hashes"][idx]) - items_to_cache.append((outputs["images"][idx], meta)) - if hashes_to_cache: - self.update_processor_cache(dealer, hashes_to_cache, items_to_cache) - - return outputs - - def request2ids( - self, request: Request, tgts: List[str] = None - ) -> Dict[str, Union[np.ndarray, List[np.ndarray], None]]: - """ - Convert chat request with multimodal messages into model inputs. - - Args: - request: Request containing: - - messages: List of chat messages with text/image/video content - - request_id: Unique identifier for logging - tgts: Optional target sequences - - Returns: - Dict with same structure as text2ids() output - """ - - messages = parse_chat_messages(request.messages) - mm_items = [] - for msg in messages: - role = msg.get("role") - assert role in self.role_prefixes, f"Unsupported role: {role}" - - content = msg.get("content") - if not isinstance(content, list): - content = [content] - for item in content: - if item.get("type") in ["image", "video"]: - mm_items.append(item) - - missing_hashes, missing_idx = [], [] - for idx, item in enumerate(mm_items): - if not item.get("data"): - missing_hashes.append(item.get("uuid")) - missing_idx.append(idx) - - if len(missing_hashes) > 0 and not self.enable_processor_cache: - raise ValueError("Missing items cannot be retrieved without processor cache.") - - if self.enable_processor_cache: - context = zmq.Context() - dealer = context.socket(zmq.DEALER) - dealer.connect("ipc:///dev/shm/processor_cache.ipc") - - missing_items = self.get_processor_cache(dealer, missing_hashes) - for idx in range(len(missing_items)): - if not missing_items[idx]: - raise ValueError(f"Missing item {idx} not found in processor cache") - mm_items[missing_idx[idx]]["data"] = missing_items[idx] - - images, videos = [], [] - image_uuid, video_uuid = [], [] - for item in mm_items: - if item.get("type") == "image": - images.append(item["data"]) - image_uuid.append(item["uuid"]) - elif item.get("type") == "video": - videos.append(item["data"]) - video_uuid.append(item["uuid"]) - else: - raise ValueError(f"Unsupported multimodal type: {item.get('type')}") - - if self.tokenizer.chat_template is None: - raise ValueError("This model does not support chat template.") - - chat_template_kwargs = request.chat_template_kwargs if request.chat_template_kwargs else {} - prompt = self.tokenizer.apply_chat_template( - messages, - tokenize=False, - add_generation_prompt=request.add_generation_prompt if request.add_generation_prompt is not None else True, - **chat_template_kwargs, - ) - request.prompt_tokens = prompt - - outputs = self.text2ids(prompt, images, videos, image_uuid, video_uuid) - - if self.enable_processor_cache: - missing_idx = set(missing_idx) - hashes_to_cache, items_to_cache = [], [] - for idx in range(len(mm_items)): - if idx in missing_idx: - continue - meta = {} - grid_thw = np.asarray(outputs["grid_thw"][idx]) - if grid_thw.ndim > 1: - t, h, w = grid_thw[0] - else: - t, h, w = grid_thw - meta["thw"] = (int(t), int(h), int(w)) - meta["fps"] = outputs["fps"][idx] - hashes_to_cache.append(outputs["mm_hashes"][idx]) - items_to_cache.append((outputs["images"][idx], meta)) - if hashes_to_cache: - self.update_processor_cache(dealer, hashes_to_cache, items_to_cache) - - return outputs - - def _add_text(self, tokens, outputs: Dict) -> None: - """ - Add text tokens to model inputs dictionary. - - Args: - tokens: Text string or already tokenized IDs - outputs: Dictionary accumulating model inputs - - Note: - - Handles both raw text and pre-tokenized inputs - - Updates position IDs for 3D embeddings - """ - if not tokens: - return None - - if isinstance(tokens, str): - tokens_str = self.tokenizer.tokenize(tokens) - tokens = self.tokenizer.convert_tokens_to_ids(tokens_str) - - num_tokens = len(tokens) - outputs["input_ids"].extend(tokens) - outputs["token_type_ids"].extend([IDS_TYPE_FLAG["text"]] * num_tokens) - - pos_ids = self._compute_text_positions(outputs["cur_position"], num_tokens) - outputs["position_ids"].append(pos_ids) - outputs["cur_position"] = pos_ids.max() + 1 - - def _compute_text_positions(self, start_pos: int, num_tokens: int) -> np.ndarray: - """ - Generate 3D positional embeddings for text tokens. - - Args: - start_pos: Starting position index - num_tokens: Number of tokens to generate positions for - - Returns: - numpy.ndarray: 3D position IDs shaped (3, num_tokens) - """ - text_array = np.arange(num_tokens).reshape(1, -1) - text_index = np.broadcast_to(text_array, (3, num_tokens)) - position = text_index + start_pos - return position - - def _add_image(self, img, outputs: Dict, uuid: Optional[str], token_len: Optional[int] = None) -> None: - """ - Add image data to model inputs dictionary. - - Args: - img: PIL Image to process - outputs: Dictionary accumulating model inputs - - Note: - - Preprocesses image and calculates spatial dimensions - - Adds image token IDs and type markers - - Generates appropriate position embeddings - """ - ret = self.image_processor.preprocess(images=[img.convert("RGB")]) - num_tokens = ret["grid_thw"].prod() // self.image_processor.merge_size**2 - grid_thw = ret["grid_thw"].tolist() - if token_len is not None and token_len != num_tokens: - raise ValueError("image tokens num not match the size") - - outputs["mm_positions"].append(ImagePosition(len(outputs["input_ids"]), num_tokens)) - outputs["input_ids"].extend([self.image_token_id] * num_tokens) - outputs["token_type_ids"].extend([IDS_TYPE_FLAG["image"]] * num_tokens) - outputs["num_input_image_tokens"] += int(num_tokens) - - outputs["images"].append(ret["pixel_values"]) - if not uuid: - outputs["mm_hashes"].append(MultimodalHasher.hash_features(ret["pixel_values"])) - else: - outputs["mm_hashes"].append(uuid) - outputs["grid_thw"].append(grid_thw) - outputs["image_type_ids"].append(0) - - t, h, w = grid_thw - pos_ids = self._compute_vision_positions(outputs["cur_position"], t, h, w, 0) - - outputs["position_ids"].append(pos_ids) - outputs["cur_position"] = pos_ids.max() + 1 - - outputs["fps"].append(0) - - def _add_processed_image( - self, img_cache: Tuple[np.ndarray, dict], outputs: Dict, uuid: str, token_len: Optional[int] = None - ) -> None: - img, meta = img_cache - num_tokens = img.shape[0] // self.image_processor.merge_size**2 - if token_len is not None and token_len != num_tokens: - raise ValueError("image tokens num not match the size") - - outputs["mm_positions"].append(ImagePosition(len(outputs["input_ids"]), num_tokens)) - outputs["input_ids"].extend([self.image_token_id] * num_tokens) - outputs["token_type_ids"].extend([IDS_TYPE_FLAG["image"]] * num_tokens) - - _, h, w = meta["thw"] - pos_ids = self._compute_vision_positions(outputs["cur_position"], 1, h, w, 0) - outputs["position_ids"].append(pos_ids) - outputs["cur_position"] = pos_ids.max() + 1 - - outputs["images"].append(img) - outputs["mm_hashes"].append(uuid) - outputs["grid_thw"].append(np.array([[1, h, w]])) - outputs["image_type_ids"].append(0) - - outputs["fps"].append(0) - - def _add_video( - self, frames, meta: Dict, outputs: Dict, uuid: Optional[str], token_len: Optional[int] = None - ) -> None: - """ - Add video data to model inputs dictionary. - - Args: - frames: Video frames as numpy array - meta: Video metadata containing fps/duration - outputs: Dictionary accumulating model inputs - - Note: - - Handles temporal dimension in position embeddings - - Uses video-specific token IDs and type markers - """ - ret = self.image_processor.preprocess( - images=frames, - min_pixels=VIDEO_MIN_PIXELS, - max_pixels=VIDEO_MAX_PIXELS, - ) - - num_tokens = ret["grid_thw"].prod() // self.image_processor.merge_size**2 - grid_thw = ret["grid_thw"].tolist() - if token_len is not None and token_len != num_tokens: - raise ValueError("video tokens num not match the size") - - outputs["mm_positions"].append(ImagePosition(len(outputs["input_ids"]), num_tokens)) - # Hack code. In order to adapt to the framework, only image_token can be passed - # The correct way should be to use [self.video_token_id] * num_tokens - outputs["input_ids"].extend([self.image_token_id] * num_tokens) - outputs["token_type_ids"].extend([IDS_TYPE_FLAG["video"]] * num_tokens) - outputs["num_input_video_tokens"] += int(num_tokens) - - outputs["images"].append(ret["pixel_values"]) - if not uuid: - outputs["mm_hashes"].append(MultimodalHasher.hash_features(ret["pixel_values"])) - else: - outputs["mm_hashes"].append(uuid) - outputs["grid_thw"].append(grid_thw) - outputs["image_type_ids"].extend([1] * grid_thw[0]) - - fps = meta["fps"] - second_per_grid_t = self.temporal_conv_size / fps - t, h, w = grid_thw - pos_ids = self._compute_vision_positions(outputs["cur_position"], t, h, w, second_per_grid_t) - - outputs["position_ids"].append(pos_ids) - outputs["cur_position"] = pos_ids.max() + 1 - - outputs["fps"].append(fps) - - def _add_processed_video( - self, frames_cache: Tuple[np.ndarray, dict], outputs: Dict, uuid: str, token_len: Optional[int] = None - ) -> None: - frames, meta = frames_cache - num_tokens = frames.shape[0] // self.image_processor.merge_size**2 - if token_len is not None and token_len != num_tokens: - raise ValueError("video tokens num not match the size") - - t, h, w = meta["thw"] - outputs["images"].append(frames) - outputs["mm_hashes"].append(uuid) - outputs["grid_thw"].append(np.array([[t, h, w]])) - - outputs["mm_positions"].append(ImagePosition(len(outputs["input_ids"]), num_tokens)) - outputs["input_ids"].extend([self.image_token_id] * num_tokens) - outputs["token_type_ids"].extend([IDS_TYPE_FLAG["video"]] * num_tokens) - outputs["image_type_ids"].extend([1] * t) - - fps = meta["fps"] - second_per_grid_t = self.temporal_conv_size / fps - pos_ids = self._compute_vision_positions(outputs["cur_position"], t, h, w, second_per_grid_t) - outputs["position_ids"].append(pos_ids) - outputs["cur_position"] = pos_ids.max() + 1 - - outputs["fps"].append(fps) - - def _compute_vision_positions( - self, start_pos: int, t: int, h: int, w: int, second_per_grid_t: float - ) -> np.ndarray: - """ - Generate 3D position IDs for visual inputs. - - Args: - start_pos: Base position in sequence - t: Temporal patches (1 for images) - h: Height in patches - w: Width in patches - second_per_grid_t: Time per temporal patch - - Returns: - np.ndarray: Position IDs for [t,h,w] dimensions - """ - h //= self.spatial_conv_size - w //= self.spatial_conv_size - - tn = np.arange(t).reshape(-1, 1) - tn = np.broadcast_to(tn, (t, h * w)) - tn = tn * int(second_per_grid_t) * self.tokens_per_second - t_index = tn.flatten() - - hn = np.arange(h).reshape(1, -1, 1) - h_index = np.broadcast_to(hn, (t, h, w)).flatten() - - wn = np.arange(w).reshape(1, 1, -1) - w_index = np.broadcast_to(wn, (t, h, w)).flatten() - - position = np.stack([t_index, h_index, w_index]) + start_pos - return position - - def _load_and_process_video(self, url: str, item: Dict) -> Tuple[np.ndarray, Dict]: - """ - Load and preprocess video into frames. - - Args: - url: Video file path or bytes - item: Dictionary containing processing parameters - - Returns: - tuple: (frames, metadata) where: - - frames: Processed video frames as numpy array - - metadata: Updated video metadata dictionary - """ - reader, meta, _ = read_video_decord(url, save_to_disk=False) - - # Apply frame sampling if fps or target_frames specified - fps = item.get("fps", self.fps) - num_frames = item.get("target_frames", self.target_frames) - - frame_indices = list(range(meta["num_of_frame"])) - if fps > 0 or num_frames > 0: - # Get frame sampling constraints - min_frames = item.get("min_frames", self.min_frames) - max_frames = item.get("max_frames", self.max_frames) - - # Sample frames according to specifications - frame_indices = sample_frames( - frame_factor=self.frame_factor, # Ensure divisible by temporal patch size - min_frames=min_frames, - max_frames=max_frames, - metadata=meta, - fps=fps, - num_frames=num_frames, - ) - - # Update metadata with new frame count and fps - meta["num_of_frame"] = len(frame_indices) - if fps is not None: - meta["fps"] = fps # Use specified fps - meta["duration"] = len(frame_indices) / fps - else: - meta["fps"] = len(frame_indices) / meta["duration"] # Calculate fps from sampled frames - - frames = [] - for idx in frame_indices: - frame = reader[idx].asnumpy() - image = Image.fromarray(frame, "RGB") - frames.append(image) - frames = np.stack([np.array(f.convert("RGB")) for f in frames], axis=0) - - return frames, meta - - def get_processor_cache(self, socket, mm_hashes: list[str]) -> list: - """ - get cache correspond to given hash values - """ - req = pickle.dumps(mm_hashes) - socket.send_multipart([b"", req]) - _, resp = socket.recv_multipart() - mm_items = pickle.loads(resp) - data_processor_logger.info(f"Get cache of mm_hashes: {mm_hashes}") - - return mm_items - - def update_processor_cache(self, socket, mm_hashes: list[str], mm_items): - """ - update cache data - """ - req = pickle.dumps((mm_hashes, mm_items)) - socket.send_multipart([b"", req]) - data_processor_logger.info(f"Update cache of mm_hashes: {mm_hashes}") diff --git a/fastdeploy/input/v1/qwen3_vl_processor/qwen3_vl_processor.py b/fastdeploy/input/v1/qwen3_vl_processor/qwen3_vl_processor.py deleted file mode 100644 index c72de49bcc3..00000000000 --- a/fastdeploy/input/v1/qwen3_vl_processor/qwen3_vl_processor.py +++ /dev/null @@ -1,341 +0,0 @@ -""" -# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -import numpy as np - -from fastdeploy.engine.request import Request -from fastdeploy.input.v1.text_processor import DataProcessor as TextProcessor -from fastdeploy.utils import data_processor_logger - -from .process import DataProcessor - - -class Qwen3VLProcessor(TextProcessor): - """ - Qwen Vision-Language processor for handling multimodal inputs. - - This processor extends TextProcessor to support: - - Image and video processing - - Multimodal feature extraction - - Tokenization and position encoding - - Request processing and model input generation - - Attributes: - processor (DataProcessor): Underlying data processor instance - tokenizer: Text tokenizer instance - limit_mm_per_prompt (dict): Limits for multimodal inputs per prompt - """ - - def __init__( - self, - config, - model_name_or_path, - limit_mm_per_prompt=None, - mm_processor_kwargs=None, - reasoning_parser_obj=None, - tool_parser_obj=None, - enable_processor_cache=False, - ): - """ - Initialize QwenVLProcessor instance. - - Args: - config: Model configuration object - model_name_or_path (str): Pretrained model name or path - limit_mm_per_prompt (dict, optional): Limits for multimodal inputs - mm_processor_kwargs (dict, optional): Multimodal processor arguments - reasoning_parser_obj: Reasoning parser instance - tool_parser_obj: Tool parser instance - """ - super().__init__(model_name_or_path, reasoning_parser_obj, tool_parser_obj) - - data_processor_logger.info(f"model_name_or_path: {model_name_or_path}") - processor_kwargs = self._parse_processor_kwargs(mm_processor_kwargs) - self.processor = DataProcessor( - model_path=model_name_or_path, - enable_processor_cache=enable_processor_cache, - # tokens_per_second=config.vision_config.tokens_per_second, - tokenizer=self.tokenizer, - **processor_kwargs, - ) - self.image_patch_id = self.processor.image_token_id - self.limit_mm_per_prompt = self._parse_limits(limit_mm_per_prompt) - - def _parse_processor_kwargs(self, kwargs): - """ - Parse and validate multimodal processor arguments. - - Args: - kwargs (dict): Processor configuration arguments - - Returns: - dict: Validated processor arguments - - Raises: - ValueError: If arguments format is invalid - """ - if not kwargs: - return {} - - try: - if not isinstance(kwargs, dict): - raise ValueError("mm-processor-kwargs must be a dictionary") - - # Validate kwargs types against expected schema - data_processor_logger.info(f"Processing kwargs: {kwargs}") - expected_types = { - "video_max_frames": int, # Maximum video frames parameter - "video_min_frames": int, # Minimum video frames parameter - } - - for key, value in kwargs.items(): - if key in expected_types and not isinstance(value, expected_types[key]): - raise ValueError( - f"Invalid type for {key}: expected {expected_types[key].__name__}, got {type(value).__name__}" - ) - - return kwargs - - except Exception as e: - data_processor_logger.warning(f"Invalid mm-processor-kwargs format: {e}") - return {} - - def _parse_limits(self, limits): - """ - Parse and validate multimodal input limits. - - Args: - limits (dict): Input limits configuration - - Returns: - dict: Validated limits with defaults - - Raises: - ValueError: If limits format is invalid - """ - DEFAULT_LIMITS = {"image": 1, "video": 1, "audio": 1} - - if not limits: - return DEFAULT_LIMITS - - try: - if not isinstance(limits, dict): - raise ValueError("limit-mm-per-prompt must be a dictionary") - data_processor_logger.info(f"_parse_limits:{limits}") - return {**DEFAULT_LIMITS, **limits} - except Exception as e: - data_processor_logger.warning(f"Invalid limit-mm-per-prompt format: {e}, using default limits") - return DEFAULT_LIMITS - - def _check_mm_limits(self, item): - """ - Validate multimodal inputs against configured limits. - - Args: - item: Input request item to validate - - Raises: - ValueError: If input exceeds configured limits - """ - if isinstance(item, dict): - # 请求包含prompt和multi_modal_data - mm_data = item - else: - # 请求包含messages - mm_data = {"image": [], "video": []} - - for message in item: - if isinstance(message.get("content"), list): - for part in message["content"]: - if part.get("type") in ["image_url", "image"]: - mm_data["image"].append(part) - elif part.get("type") in ["video_url", "video"]: - mm_data["video"].append(part) - - for modality, data in mm_data.items(): - if modality in self.limit_mm_per_prompt: - limit = self.limit_mm_per_prompt[modality] - if len(data) > limit: - raise ValueError(f"Too many {modality} items in prompt, " f"got {len(data)} but limit is {limit}") - - def process_request(self, request, max_model_len=None, **kwargs): - """ - Process incoming request and generate model inputs. - - Args: - request: Input request object - max_model_len (int, optional): Maximum context length - **kwargs: Additional processing parameters - - Returns: - Request: Processed request with model inputs - """ - task = request.to_dict() - task["enable_thinking"] = kwargs.get("enable_thinking", False) - self.process_request_dict(task, max_model_len) - request = Request.from_dict(task) - request = self._apply_default_parameters(request) - return request - - def process_request_dict(self, request, max_model_len=None, **kwargs): - """ - Process request dictionary into model inputs. - - Args: - request Request: Input request dictionary - max_model_len (int, optional): Maximum context length - - Returns: - Request: Processed request with model inputs - - Raises: - ValueError: If request format is invalid - """ - - request = self._apply_default_parameters(request) - if not request.eos_token_ids: - request.eos_token_ids = self.eos_token_ids - - stop_sequences = request.sampling_params.stop - if stop_sequences: - stop_seqs, stop_seqs_len = self.update_stop_seq(stop_sequences) - request.sampling_params.stop_token_ids = stop_seqs - request.sampling_params.stop_seqs_len = stop_seqs_len - - bad_words = request.sampling_params.bad_words - bad_words_token_ids = request.sampling_params.bad_words_token_ids - if bad_words: - bad_words_token_ids = self.update_bad_words(bad_words, bad_words_token_ids) - request.sampling_params.bad_words_token_ids = bad_words_token_ids - - if request.prompt_token_ids: - messages = request.messages - if messages: - self._check_mm_limits(messages) - if request.enable_thinking is None: - request.enable_thinking = kwargs.get("enable_thinking", False) - outputs = self.processor.prompt_token_ids2outputs(request) - - elif request.prompt: - multimodal_data = request.multimodal_data - if multimodal_data is None: - multimodal_data = {} - self._check_mm_limits(multimodal_data) - images = multimodal_data.get("image", None) - videos = multimodal_data.get("video", None) - outputs = self.processor.text2ids(request.prompt, images, videos) - - elif request.messages: - messages = request.messages - self._check_mm_limits(messages) - chat_template_kwargs = request.chat_template_kwargs - if chat_template_kwargs: - if isinstance(chat_template_kwargs, dict): - for k, v in chat_template_kwargs.items(): - if getattr(request, k, None) is None: - setattr(request, k, v) - else: - raise ValueError("Invalid input: chat_template_kwargs must be a dict") - if request.enable_thinking is None: - request.enable_thinking = kwargs.get("enable_thinking", False) - outputs = self.processor.request2ids(request) - delattr(request, "chat_template_kwargs") - else: - raise ValueError(f"Request must contain 'prompt', or 'messages': {request}") - - # Handle continuation of previous generation by appending existing tokens - if request.completion_token_ids: - self.append_completion_tokens(outputs, request.completion_token_ids) - - # qwen25_vl not support thinking - request.enable_thinking = False - - outputs = self.pack_outputs(outputs) - - request.prompt_token_ids = ( - outputs["input_ids"].tolist() - if not getattr(request, "prompt_token_ids", None) - else request.prompt_token_ids - ) - request.prompt_token_ids_len = len(request.prompt_token_ids) - request.multimodal_inputs = outputs - - # Handle prompt truncation if exceeds model context length - if max_model_len is not None and len(request.prompt_token_ids) > max_model_len: - request.prompt_token_ids = request.prompt_token_ids[ - : max_model_len - 1 - ] # Leave space for at least 1 new token - - # Set default max_tokens if not specified - max_tokens = max_model_len - len(request.prompt_token_ids) - if getattr(request.sampling_params, "max_tokens", None) is None: - request.sampling_params.max_tokens = max(1, max_tokens) - else: - request.sampling_params.max_tokens = min(max_tokens, request.sampling_params.max_tokens) - data_processor_logger.info(f"Processed request {request}") - - return request - - def append_completion_tokens(self, multimodal_inputs, completion_token_ids): - """ - Append completion tokens to existing outputs. - - Args: - outputs: Current model outputs - completion_token_ids: completion tokens to append - """ - - num_tokens = len(completion_token_ids) - multimodal_inputs["input_ids"].extend(completion_token_ids) - multimodal_inputs["token_type_ids"].extend([0] * num_tokens) - - pos_ids = self.processor._compute_text_positions(multimodal_inputs["cur_position"], num_tokens) - multimodal_inputs["position_ids"].append(pos_ids) - multimodal_inputs["cur_position"] += num_tokens - - def pack_outputs(self, outputs): - """ - Prepare final output dictionary for model. - - Args: - outputs: Intermediate processing outputs - - Returns: - dict: Packed output dictionary with all required fields - """ - if not outputs["images"]: - outputs["images"] = None # No images case - outputs["grid_thw"] = None # No spatial dimensions - outputs["image_type_ids"] = None # No type IDs - else: - outputs["images"] = np.vstack(outputs["images"]) # Stack image features vertically - outputs["grid_thw"] = np.vstack(outputs["grid_thw"]) # Stack spatial dimensions - outputs["image_type_ids"] = np.array(outputs["image_type_ids"]) # Convert to numpy array - - # Convert all outputs to numpy arrays with appropriate types - outputs["input_ids"] = np.array(outputs["input_ids"], dtype=np.int64) # Token IDs as int64 - outputs["token_type_ids"] = np.array(outputs["token_type_ids"], dtype=np.int64) # Type IDs as int64 - outputs["position_ids"] = np.concatenate( - outputs["position_ids"], axis=1, dtype=np.int64 - ) # Concatenate position ID - - outputs["image_patch_id"] = self.processor.image_token_id - outputs["video_patch_id"] = self.processor.video_token_id - outputs["position_ids"] = outputs["position_ids"].transpose(1, 0) - - outputs["mm_num_token_func"] = self.processor.mm_num_tokens - - return outputs diff --git a/fastdeploy/input/v1/qwen_vl_processor/__init__.py b/fastdeploy/input/v1/qwen_vl_processor/__init__.py deleted file mode 100644 index c876cde7125..00000000000 --- a/fastdeploy/input/v1/qwen_vl_processor/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -""" -# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -from .process import DataProcessor -from .qwen_vl_processor import QwenVLProcessor - -__all__ = [ - "DataProcessor", - "QwenVLProcessor", -] diff --git a/fastdeploy/input/v1/qwen_vl_processor/image_processor.py b/fastdeploy/input/v1/qwen_vl_processor/image_processor.py deleted file mode 100644 index b6a1db19bc5..00000000000 --- a/fastdeploy/input/v1/qwen_vl_processor/image_processor.py +++ /dev/null @@ -1,442 +0,0 @@ -""" -# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -import math -from typing import List, Optional, Union - -import numpy as np -import paddle -import PIL -from paddleformers.transformers.feature_extraction_utils import BatchFeature -from paddleformers.transformers.image_processing_utils import BaseImageProcessor -from paddleformers.transformers.image_transforms import ( - normalize, - rescale, - resize, - to_channel_dimension_format, -) -from paddleformers.transformers.image_utils import ( - ChannelDimension, - ImageInput, - PILImageResampling, - get_image_size, - infer_channel_dimension_format, - make_list_of_images, - to_numpy_array, - valid_images, -) -from paddleformers.transformers.legacy.tokenizer_utils_base import TensorType -from PIL import Image - -from fastdeploy.utils import data_processor_logger - -OPENAI_CLIP_MEAN = [0.48145466, 0.4578275, 0.40821073] -OPENAI_CLIP_STD = [0.26862954, 0.26130258, 0.27577711] - -MIN_PIXELS = 4 * 28 * 28 -MAX_PIXELS = 16384 * 28 * 28 - - -VideoInput = Union[ - List["PIL.Image.Image"], - "np.ndarray", - "paddle.Tensor", - List["np.ndarray"], - List["paddle.Tensor"], - List[List["PIL.Image.Image"]], - List[List["np.ndarray"]], - List[List["paddle.Tensor"]], -] - - -def round_by_factor(number: int, factor: int) -> int: - """ - Round number to nearest multiple of factor. - - Args: - number: Input number to round - factor: Rounding factor - - Returns: - int: Rounded number - """ - return round(number / factor) * factor - - -def ceil_by_factor(number: int, factor: int) -> int: - """ - Round number up to nearest multiple of factor. - - Args: - number: Input number to round - factor: Rounding factor - - Returns: - int: Rounded number - """ - return math.ceil(number / factor) * factor - - -def floor_by_factor(number: int, factor: int) -> int: - """ - Round number down to nearest multiple of factor. - - Args: - number: Input number to round - factor: Rounding factor - - Returns: - int: Rounded number - """ - return math.floor(number / factor) * factor - - -def smart_resize(height: int, width: int, factor: int, min_pixels: int, max_pixels: int, max_ratio: int = 200): - """ - Smart image resizing that maintains aspect ratio and respects constraints. - - Args: - height: Original image height - width: Original image width - factor: Patch size factor - min_pixels: Minimum allowed pixels - max_pixels: Maximum allowed pixels - max_ratio: Maximum allowed aspect ratio - - Returns: - tuple: (new_height, new_width) - - Raises: - ValueError: If calculated dimensions are invalid - """ - if max(height, width) / min(height, width) > max_ratio: - if height > width: - new_width = max(factor, round_by_factor(width, factor)) - new_height = floor_by_factor(new_width * max_ratio, factor) - else: - new_height = max(factor, round_by_factor(height, factor)) - new_width = floor_by_factor(new_height * max_ratio, factor) - - data_processor_logger.info( - f"absolute aspect ratio must be smaller than {max_ratio}, got {max(height, width) / min(height, width)},\ - resize to {max(new_height, new_width) / min(new_height, new_width)}" - ) - - height = new_height - width = new_width - - h_bar = max(factor, round_by_factor(height, factor)) - w_bar = max(factor, round_by_factor(width, factor)) - if h_bar * w_bar > max_pixels: - beta = math.sqrt((height * width) / max_pixels) - h_bar = floor_by_factor(height / beta, factor) - w_bar = floor_by_factor(width / beta, factor) - elif h_bar * w_bar < min_pixels: - beta = math.sqrt(min_pixels / (height * width)) - h_bar = ceil_by_factor(height * beta, factor) - w_bar = ceil_by_factor(width * beta, factor) - - if min_pixels > h_bar * w_bar or h_bar * w_bar > max_pixels: - raise ValueError(f"encounter invalid h_bar: {h_bar}, w_bar: {w_bar}") - - return h_bar, w_bar - - -def is_scaled_image(image: np.ndarray) -> bool: - """ - Check if image pixel values are already normalized to [0, 1] range. - - Args: - image: Input image array - - Returns: - bool: True if image is already scaled - """ - if image.dtype == np.uint8: - return False - - # It's possible the image has pixel values in [0, 255] but is of floating type - return np.min(image) >= 0 and np.max(image) <= 1 - - -class ImageProcessor(BaseImageProcessor): - """ - Adaptive image processor for dynamic image resizing and preprocessing. - - This processor handles image resizing, rescaling, normalization and format conversion. - It dynamically adjusts image dimensions based on original size and specified constraints. - """ - - def __init__( - self, - patch_size: int = 14, - merge_size: int = 2, - temporal_patch_size: int = 2, - min_pixels: int = MIN_PIXELS, - max_pixels: int = MAX_PIXELS, - image_mean: Union[float, List[float]] = OPENAI_CLIP_MEAN, - image_std: Union[float, List[float]] = OPENAI_CLIP_STD, - rescale_factor: float = 1 / 255, - do_rescale: bool = True, - do_normalize: bool = True, - resample: PILImageResampling = PILImageResampling.BICUBIC, - **kwargs, - ) -> None: - """ - Initialize image processor with configuration parameters. - - Args: - patch_size (int): Spatial patch size for vision encoder - merge_size (int): Merge size between vision and LLM encoders - temporal_patch_size (int): Temporal patch size for video processing - min_pixels (int): Minimum allowed pixels in resized image - max_pixels (int): Maximum allowed pixels in resized image - image_mean (float/list): Mean values for normalization per channel - image_std (float/list): Std values for normalization per channel - rescale_factor (float): Scaling factor for pixel values (default 1/255) - do_rescale (bool): Whether to rescale images - do_normalize (bool): Whether to normalize images - resample: Resampling method for image resizing - **kwargs: Additional base class arguments - """ - super().__init__(**kwargs) - self.patch_size = patch_size - self.merge_size = merge_size - self.temporal_patch_size = temporal_patch_size - - self.min_pixels = min_pixels - self.max_pixels = max_pixels - - self.image_mean = image_mean - self.image_std = image_std - self.rescale_factor = rescale_factor - self.do_rescale = do_rescale - self.do_normalize = do_normalize - - self.resample = resample - - def _preprocess( - self, - images: Union[ImageInput, VideoInput], - min_pixels: int, - max_pixels: int, - image_mean: Optional[Union[float, List[float]]], - image_std: Optional[Union[float, List[float]]], - rescale_factor: float, - do_rescale: bool, - do_normalize: bool, - resample: PILImageResampling, - data_format: Optional[ChannelDimension], - input_data_format: Optional[Union[str, ChannelDimension]], - ): - """ - Internal method for image preprocessing pipeline. - - Args: - images: Input image or batch of images - min_pixels: Minimum allowed pixels in output - max_pixels: Maximum allowed pixels in output - image_mean: Normalization mean values - image_std: Normalization std values - rescale_factor: Pixel value scaling factor - do_rescale: Whether to rescale pixel values - do_normalize: Whether to normalize pixel values - resample: Resampling method - data_format: Output channel format - input_data_format: Input channel format - - Returns: - tuple: (flatten_patches, grid_dimensions) - - flatten_patches: Flattened image patches - - grid_dimensions: Grid dimensions [t, h, w] - """ - images = make_list_of_images(images) - - # All transformations expect numpy arrays. - images = [to_numpy_array(image) for image in images] - - if is_scaled_image(images[0]) and do_rescale: - data_processor_logger.warning( - "It looks like you are trying to rescale already rescaled images. If the input" - " images have pixel values between 0 and 1, set `do_rescale=False` to avoid rescaling them again." - ) - if input_data_format is None: - # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(images[0]) - - # Get original dimensions and calculate optimal resize dimensions - height, width = get_image_size(images[0], channel_dim=input_data_format) - resized_height, resized_width = smart_resize( - height, - width, - factor=self.patch_size * self.merge_size, # Combine patch and merge factors - min_pixels=min_pixels, - max_pixels=max_pixels, - ) - - processed_images = [] - for image in images: - if height != resized_height or width != resized_width: - # Convert to uint8 before resizing to avoid double scaling - image = image.astype("uint8") - # Convert to PIL Image and resize - image = Image.fromarray(image) - image = resize( - image, - size=(resized_height, resized_width), - resample=resample, - data_format=input_data_format, - ) - - if do_rescale and do_normalize: - # Adjust mean and std for combined rescale+normalize - image_mean = np.array(image_mean, dtype=np.float32) * (1.0 / rescale_factor) - image_std = np.array(image_std, dtype=np.float32) * (1.0 / rescale_factor) - do_rescale = False # Skip separate rescale step - - if do_rescale: - image = image.astype(np.float32) - image = rescale(image, scale=rescale_factor, data_format=input_data_format) - - if do_normalize: - image = image.astype(np.float32) - image = normalize( - image=image, - mean=image_mean, - std=image_std, - data_format=input_data_format, - ) - - image = to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) # [C, H, W] - processed_images.append(image) - - # Convert processed images to numpy array - patches = np.array(processed_images) - - # Pad temporal dimension if needed - if patches.shape[0] % self.temporal_patch_size != 0: - repeats = np.repeat( - patches[-1][np.newaxis], - self.temporal_patch_size - (patches.shape[0] % self.temporal_patch_size), - axis=0, - ) - patches = np.concatenate([patches, repeats], axis=0) - - # Convert to channels-first format if needed - if data_format == ChannelDimension.LAST: - patches = patches.transpose([0, 3, 1, 2]) # [N, H, W, C] -> [N, C, H, W] - - grid_t, channel = patches.shape[:2] - grid_t = grid_t // self.temporal_patch_size - - grid_h, grid_w = ( - resized_height // self.patch_size, - resized_width // self.patch_size, - ) - # Reshape into hierarchical patch structure - patches = patches.reshape( - [ - grid_t, - self.temporal_patch_size, - channel, - grid_h // self.merge_size, - self.merge_size, - self.patch_size, - grid_w // self.merge_size, - self.merge_size, - self.patch_size, - ] - ) - # Reorder dimensions for better memory access pattern - # [grid_t, grid_h/merge_size, grid_w/merge_size, merge_size, merge_size, C, temporal_patch_size, psz, psz] - patches = patches.transpose([0, 3, 6, 4, 7, 2, 1, 5, 8]) - - flatten_patches = patches.reshape( - [ - grid_t * grid_h * grid_w, - channel * self.temporal_patch_size * self.patch_size * self.patch_size, - ] - ) - - return flatten_patches, np.array([grid_t, grid_h, grid_w]) - - def preprocess( - self, - images: Union[ImageInput, VideoInput], - min_pixels: Optional[int] = None, - max_pixels: Optional[int] = None, - image_mean: Optional[Union[float, List[float]]] = None, - image_std: Optional[Union[float, List[float]]] = None, - rescale_factor: Optional[float] = None, - do_rescale: Optional[bool] = None, - do_normalize: Optional[bool] = None, - resample: Optional[PILImageResampling] = None, - return_tensors: Optional[Union[str, TensorType]] = None, - data_format: Optional[ChannelDimension] = ChannelDimension.FIRST, - input_data_format: Optional[Union[str, ChannelDimension]] = ChannelDimension.LAST, - ): - """ - Main preprocessing method for images/videos. - - Args: - images: Input image/video data - min_pixels: Override for minimum pixels - max_pixels: Override for maximum pixels - image_mean: Override for normalization mean - image_std: Override for normalization std - rescale_factor: Override for rescaling factor - do_rescale: Override for rescaling flag - do_normalize: Override for normalization flag - resample: Override for resampling method - return_tensors: Desired output tensor format - data_format: Output channel dimension format - input_data_format: Input channel dimension format - - Returns: - BatchFeature: Processed features containing: - - pixel_values: Preprocessed pixel data - - grid_thw: Grid dimensions [temporal, height, width] - - Raises: - ValueError: For invalid image types or dimensions - """ - min_pixels = min_pixels if min_pixels is not None else self.min_pixels - max_pixels = max_pixels if max_pixels is not None else self.max_pixels - image_mean = image_mean if image_mean is not None else self.image_mean - image_std = image_std if image_std is not None else self.image_std - rescale_factor = rescale_factor if rescale_factor is not None else self.rescale_factor - do_rescale = do_rescale if do_rescale is not None else self.do_rescale - do_normalize = do_normalize if do_normalize is not None else self.do_normalize - resample = resample if resample is not None else self.resample - - if images is not None and not valid_images(images): - raise ValueError("Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, " "paddle.Tensor.") - - pixel_values, grid_thw = self._preprocess( - images, - min_pixels=min_pixels, - max_pixels=max_pixels, - image_mean=image_mean, - image_std=image_std, - rescale_factor=rescale_factor, - do_rescale=do_rescale, - do_normalize=do_normalize, - resample=resample, - data_format=data_format, - input_data_format=input_data_format, - ) - data = {"pixel_values": pixel_values, "grid_thw": grid_thw} - return BatchFeature(data=data, tensor_type=return_tensors) diff --git a/fastdeploy/input/v1/qwen_vl_processor/process.py b/fastdeploy/input/v1/qwen_vl_processor/process.py deleted file mode 100644 index 3aaade025cf..00000000000 --- a/fastdeploy/input/v1/qwen_vl_processor/process.py +++ /dev/null @@ -1,591 +0,0 @@ -""" -# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -import pickle -from typing import Dict, List, Optional, Tuple, Union - -import numpy as np -import paddle -import zmq -from paddleformers.transformers import AutoTokenizer -from PIL import Image - -from fastdeploy.engine.request import ImagePosition, Request -from fastdeploy.entrypoints.chat_utils import parse_chat_messages -from fastdeploy.input.ernie4_5_vl_processor import read_video_decord -from fastdeploy.input.mm_data_processor import MMBaseDataProcessor -from fastdeploy.input.utils import IDS_TYPE_FLAG -from fastdeploy.multimodal.hasher import MultimodalHasher -from fastdeploy.utils import data_processor_logger - -from .image_processor import ImageProcessor -from .process_video import sample_frames - -FRAME_FACTOR = 2 -FPS = 2.0 -FPS_MIN_FRAMES = 4 -FPS_MAX_FRAMES = 768 - - -class DataProcessor(MMBaseDataProcessor): - """ - Processes multimodal inputs (text, images, videos) into model-ready formats. - - Handles: - - Tokenization of text with special tokens for visual content - - Image and video preprocessing - - Generation of 3D positional embeddings - - Conversion of chat messages to model inputs - - Attributes: - tokenizer: Text tokenizer instance - image_processor: Image/video preprocessor - image_token: Special token for image placeholders - video_token: Special token for video placeholders - vision_start: Token marking start of visual content - """ - - def __init__( - self, - model_path: str, - enable_processor_cache: bool = False, - video_min_frames: int = FPS_MIN_FRAMES, - video_max_frames: int = FPS_MAX_FRAMES, - video_target_frames: int = -1, - video_fps: int = FPS, - tokens_per_second: int = 2, - tokenizer=None, - **kwargs, - ) -> None: - """ - Initialize the data processor. - - Args: - model_path: Path to pretrained model - video_min_frames: Minimum frames to sample from videos - video_max_frames: Maximum frames to sample from videos - tokens_per_second: Temporal resolution for positional embeddings - **kwargs: Additional configuration - """ - super().__init__() - self.min_frames = video_min_frames - self.max_frames = video_max_frames - self.target_frames = video_target_frames - self.fps = video_fps - self.frame_factor = FRAME_FACTOR - - # Initialize tokenizer with left padding and fast tokenizer - if tokenizer is None: - self.tokenizer = AutoTokenizer.from_pretrained(model_path, padding_side="left", use_fast=True) - self.tokenizer.ignored_index = -100 # Set ignored index for loss calculation - else: - self.tokenizer = tokenizer - self.image_processor = ImageProcessor.from_pretrained(model_path) # Initialize image processor - self.enable_processor_cache = enable_processor_cache - - # Convolution sizes for patch aggregation - self.spatial_conv_size = self.image_processor.merge_size - self.temporal_conv_size = self.image_processor.temporal_patch_size - - # Special tokens and IDs - self.image_token = "<|image_pad|>" - self.video_token = "<|video_pad|>" - - self.image_token_id = self.tokenizer.convert_tokens_to_ids(self.image_token) - self.video_token_id = self.tokenizer.convert_tokens_to_ids(self.video_token) - - self.vision_start = "<|vision_start|>" - self.vision_start_id = self.tokenizer.convert_tokens_to_ids(self.vision_start) - - self.tokens_per_second = tokens_per_second - - self.role_prefixes = { - "system": "", - "user": "User: ", - "bot": "Assistant: ", - "assistant": "Assistant: ", - } - - @staticmethod - def mm_num_tokens(grid_thw: list | list[list[int]] | np.ndarray | paddle.Tensor) -> int | list[int]: - """ - Calculate the number of tokens in the multimodal input. - """ - if isinstance(grid_thw, paddle.Tensor): - grid_thw = grid_thw.numpy() - - if len(grid_thw) == 0: - return 0 - - def calc_one(thw): - t, h, w = map(int, thw) - return t * h * w // 4 - - if isinstance(grid_thw[0], (list, tuple, np.ndarray)): - return [calc_one(x) for x in grid_thw] - - return calc_one(grid_thw) - - def text2ids(self, text, images=None, videos=None, image_uuid=None, video_uuid=None): - """ - Convert text with image/video placeholders into model inputs. - - Args: - text: Input text with <|image@placeholder|> and <|video@placeholder|> markers - images: List of PIL Images corresponding to image placeholders - videos: List of video data corresponding to video placeholders - image_uuid: List of unique identifiers for each image, used for caching or hashing. - video_uuid: List of unique identifiers for each video, used for caching or hashing. - - Returns: - Dict containing: - - input_ids: Token IDs - - token_type_ids: Type identifiers (text/image/video) - - position_ids: 3D positional embeddings - - images: Preprocessed visual features - - grid_thw: Spatial/temporal dimensions - - image_type_ids: Visual content type (0=image, 1=video) - """ - - outputs = { - "input_ids": [], - "token_type_ids": [], - "position_ids": [], - "images": [], - "grid_thw": [], - "image_type_ids": [], - "labels": [], - "cur_position": 0, - "video_cnt": 0, - "num_input_image_tokens": 0, - "num_input_video_tokens": 0, - "fps": [], - "mm_positions": [], - "mm_hashes": [], - } - - # Define placeholders and their lengths - IMAGE_PLACEHOLDER = "<|image_pad|>" - VIDEO_PLACEHOLDER = "<|video_pad|>" - IMAGE_PLACEHOLDER_LEN = len(IMAGE_PLACEHOLDER) - VIDEO_PLACEHOLDER_LEN = len(VIDEO_PLACEHOLDER) - - # Initialize tracking variables for text parsing - st, image_idx, video_idx = 0, 0, 0 # Start position, image counter, video counter - while st < len(text): - # Find next image or video placeholder in text - image_pos = text.find(IMAGE_PLACEHOLDER, st) - image_pos = len(text) if image_pos == -1 else image_pos # Set to end if not found - video_pos = text.find(VIDEO_PLACEHOLDER, st) - video_pos = len(text) if video_pos == -1 else video_pos # Set to end if not found - ed = min(image_pos, video_pos) # End position is first placeholder found - - self._add_text(text[st:ed], outputs) - if ed == len(text): - break - - if ed == image_pos: - image = images[image_idx] - uuid = image_uuid[image_idx] if image_uuid else None - if not isinstance(image, tuple): - self._add_image(image, outputs, uuid) - else: - self._add_processed_image(image, outputs, uuid) - image_idx += 1 - st = ed + IMAGE_PLACEHOLDER_LEN - else: - item = videos[video_idx] - uuid = video_uuid[video_idx] if video_uuid else None - if not isinstance(item, tuple): - if isinstance(item, dict): - frames, meta = self._load_and_process_video(item["video"], item) - else: - frames, meta = self._load_and_process_video(item, {}) - self._add_video(frames, meta, outputs, uuid) - else: - # cached frames are already processed - self._add_processed_video(item, outputs, uuid) - video_idx += 1 - st = ed + VIDEO_PLACEHOLDER_LEN - - return outputs - - def request2ids( - self, request: Request, tgts: List[str] = None - ) -> Dict[str, Union[np.ndarray, List[np.ndarray], None]]: - """ - Convert chat request with multimodal messages into model inputs. - - Args: - request: Dictionary containing: - - messages: List of chat messages with text/image/video content - - request_id: Unique identifier for logging - tgts: Optional target sequences - - Returns: - Dict with same structure as text2ids() output - """ - - # Parse and validate chat messages - messages = parse_chat_messages(request.messages) - mm_items = [] - for msg in messages: - role = msg.get("role") - assert role in self.role_prefixes, f"Unsupported role: {role}" - - # Normalize content to list format - content = msg.get("content") - if not isinstance(content, list): - content = [content] - # Collect all visual content items - for item in content: - if item.get("type") in ["image", "video"]: - mm_items.append(item) - - missing_hashes, missing_idx = [], [] - for idx, item in enumerate(mm_items): - if not item.get("data"): - # raw data not provided, should be retrieved from processor cache - missing_hashes.append(item.get("uuid")) - missing_idx.append(idx) - - if len(missing_hashes) > 0 and not self.enable_processor_cache: - raise ValueError("Missing items cannot be retrieved without processor cache.") - - if self.enable_processor_cache: - context = zmq.Context() - dealer = context.socket(zmq.DEALER) - dealer.connect("ipc:///dev/shm/processor_cache.ipc") - - missing_items = self.get_processor_cache(dealer, missing_hashes) - for idx in range(len(missing_items)): - if not missing_items[idx]: - raise ValueError(f"Missing item {idx} not found in processor cache") - mm_items[missing_idx[idx]]["data"] = missing_items[idx] - - images, videos = [], [] - image_uuid, video_uuid = [], [] - for item in mm_items: - if item.get("type") == "image": - images.append(item["data"]) - image_uuid.append(item["uuid"]) - elif item.get("type") == "video": - videos.append(item["data"]) - video_uuid.append(item["uuid"]) - else: - raise ValueError(f"Unsupported multimodal type: {item.get('type')}") - - if self.tokenizer.chat_template is None: - raise ValueError("This model does not support chat template.") - - chat_template_kwargs = request.chat_template_kwargs if request.chat_template_kwargs else {} - prompt = self.tokenizer.apply_chat_template( - messages, - tokenize=False, - add_generation_prompt=request.add_generation_prompt if request.add_generation_prompt is not None else True, - **chat_template_kwargs, - ) - request.prompt_tokens = prompt - - outputs = self.text2ids(prompt, images, videos, image_uuid, video_uuid) - - if self.enable_processor_cache: - missing_idx = set(missing_idx) - hashes_to_cache, items_to_cache = [], [] - for idx in range(len(mm_items)): - if idx in missing_idx: - continue - meta = {} - t, h, w = outputs["grid_thw"][idx] - meta["thw"] = (t, h, w) - meta["fps"] = outputs["fps"][idx] - hashes_to_cache.append(outputs["mm_hashes"][idx]) - items_to_cache.append((outputs["images"][idx], meta)) - self.update_processor_cache(dealer, hashes_to_cache, items_to_cache) - - return outputs - - def _add_text(self, tokens, outputs: Dict) -> None: - """ - Add text tokens to model inputs dictionary. - - Args: - tokens: Text string or already tokenized IDs - outputs: Dictionary accumulating model inputs - - Note: - - Handles both raw text and pre-tokenized inputs - - Updates position IDs for 3D embeddings - """ - if not tokens: - return None - - if isinstance(tokens, str): - tokens_str = self.tokenizer.tokenize(tokens) - tokens = self.tokenizer.convert_tokens_to_ids(tokens_str) - - num_tokens = len(tokens) - outputs["input_ids"].extend(tokens) - outputs["token_type_ids"].extend([IDS_TYPE_FLAG["text"]] * num_tokens) - - pos_ids = self._compute_text_positions(outputs["cur_position"], num_tokens) - outputs["position_ids"].append(pos_ids) - outputs["cur_position"] = pos_ids.max() + 1 - - def _compute_text_positions(self, start_pos: int, num_tokens: int) -> np.ndarray: - """ - Generate 3D positional embeddings for text tokens. - - Args: - start_pos: Starting position index - num_tokens: Number of tokens to generate positions for - - Returns: - numpy.ndarray: 3D position IDs shaped (3, num_tokens) - """ - text_array = np.arange(num_tokens).reshape(1, -1) - text_index = np.broadcast_to(text_array, (3, num_tokens)) - position = text_index + start_pos - return position - - def _add_image(self, img, outputs: Dict, uuid: Optional[str]) -> None: - """ - Add image data to model inputs dictionary. - - Args: - img: PIL Image to process - outputs: Dictionary accumulating model inputs - - Note: - - Preprocesses image and calculates spatial dimensions - - Adds image token IDs and type markers - - Generates appropriate position embeddings - """ - ret = self.image_processor.preprocess(images=[img.convert("RGB")]) - num_tokens = ret["grid_thw"].prod() // self.image_processor.merge_size**2 - grid_thw = ret["grid_thw"].tolist() - - outputs["mm_positions"].append(ImagePosition(len(outputs["input_ids"]), num_tokens)) - outputs["input_ids"].extend([self.image_token_id] * num_tokens) - outputs["token_type_ids"].extend([IDS_TYPE_FLAG["image"]] * num_tokens) - outputs["num_input_image_tokens"] += int(num_tokens) - - outputs["images"].append(ret["pixel_values"]) - if not uuid: - outputs["mm_hashes"].append(MultimodalHasher.hash_features(ret["pixel_values"])) - else: - outputs["mm_hashes"].append(uuid) - outputs["grid_thw"].append(grid_thw) - outputs["image_type_ids"].append(0) - - t, h, w = grid_thw - pos_ids = self._compute_vision_positions(outputs["cur_position"], t, h, w, 0) - - outputs["position_ids"].append(pos_ids) - outputs["cur_position"] = pos_ids.max() + 1 - - outputs["fps"].append(0) - - def _add_processed_image(self, img_cache: Tuple[np.ndarray, dict], outputs: Dict, uuid: str) -> None: - img, meta = img_cache - num_tokens = img.shape[0] // self.image_processor.merge_size**2 - - outputs["mm_positions"].append(ImagePosition(len(outputs["input_ids"]), num_tokens)) - outputs["input_ids"].extend([self.image_patch_id] * num_tokens) - outputs["token_type_ids"].extend([IDS_TYPE_FLAG["image"]] * num_tokens) - - _, h, w = meta["thw"] - pos_ids = self._compute_vision_positions(outputs["cur_position"], 1, h, w, 0) - outputs["position_ids"].append(pos_ids) - outputs["cur_position"] = pos_ids.max() + 1 - - outputs["images"].append(img) - outputs["mm_hashes"].append(uuid) - outputs["grid_thw"].append(np.array([[1, h, w]])) - outputs["image_type_ids"].append(0) - - outputs["fps"].append(0) - - def _add_video(self, frames, meta: Dict, outputs: Dict, uuid: Optional[str]) -> None: - """ - Add video data to model inputs dictionary. - - Args: - frames: Video frames as numpy array - meta: Video metadata containing fps/duration - outputs: Dictionary accumulating model inputs - - Note: - - Handles temporal dimension in position embeddings - - Uses video-specific token IDs and type markers - """ - ret = self.image_processor.preprocess(images=frames) - - num_tokens = ret["grid_thw"].prod() // self.image_processor.merge_size**2 - grid_thw = ret["grid_thw"].tolist() - - outputs["mm_positions"].append(ImagePosition(len(outputs["input_ids"]), num_tokens)) - # Hack code. In order to adapt to the framework, only image_token can be passed - # The correct way should be to use [self.video_token_id] * num_tokens - outputs["input_ids"].extend([self.image_token_id] * num_tokens) - outputs["token_type_ids"].extend([IDS_TYPE_FLAG["video"]] * num_tokens) - outputs["num_input_video_tokens"] += int(num_tokens) - - outputs["images"].append(ret["pixel_values"]) - if not uuid: - outputs["mm_hashes"].append(MultimodalHasher.hash_features(ret["pixel_values"])) - else: - outputs["mm_hashes"].append(uuid) - outputs["grid_thw"].append(grid_thw) - outputs["image_type_ids"].extend([1] * grid_thw[0]) - - fps = meta["fps"] - second_per_grid_t = self.temporal_conv_size / fps - t, h, w = grid_thw - pos_ids = self._compute_vision_positions(outputs["cur_position"], t, h, w, second_per_grid_t) - - outputs["position_ids"].append(pos_ids) - outputs["cur_position"] = pos_ids.max() + 1 - - outputs["fps"].append(fps) - - def _add_processed_video(self, frames_cache: Tuple[np.ndarray, dict], outputs: Dict, uuid: str) -> None: - frames, meta = frames_cache - num_tokens = frames.shape[0] // self.image_processor.merge_size**2 - - t, h, w = meta["thw"] - outputs["images"].append(frames) - outputs["mm_hashes"].append(uuid) - outputs["grid_thw"].append(np.array([[t, h, w]])) - - outputs["mm_positions"].append(ImagePosition(len(outputs["input_ids"]), num_tokens)) - outputs["input_ids"].extend([self.image_patch_id] * num_tokens) - outputs["token_type_ids"].extend([IDS_TYPE_FLAG["video"]] * num_tokens) - outputs["image_type_ids"].extend([1] * t) - - fps = meta["fps"] - second_per_grid_t = self.temporal_conv_size / fps - pos_ids = self._compute_vision_positions(outputs["cur_position"], t, h, w, second_per_grid_t) - outputs["position_ids"].append(pos_ids) - outputs["cur_position"] = pos_ids.max() + 1 - - outputs["fps"].append(fps) - - def _compute_vision_positions( - self, start_pos: int, t: int, h: int, w: int, second_per_grid_t: float - ) -> np.ndarray: - """ - Generate 3D position IDs for visual inputs. - - Args: - start_pos: Base position in sequence - t: Temporal patches (1 for images) - h: Height in patches - w: Width in patches - second_per_grid_t: Time per temporal patch - - Returns: - np.ndarray: Position IDs for [t,h,w] dimensions - """ - h //= self.spatial_conv_size - w //= self.spatial_conv_size - - tn = np.arange(t).reshape(-1, 1) - tn = np.broadcast_to(tn, (t, h * w)) - tn = tn * int(second_per_grid_t) * self.tokens_per_second - t_index = tn.flatten() - - hn = np.arange(h).reshape(1, -1, 1) - h_index = np.broadcast_to(hn, (t, h, w)).flatten() - - wn = np.arange(w).reshape(1, 1, -1) - w_index = np.broadcast_to(wn, (t, h, w)).flatten() - - position = np.stack([t_index, h_index, w_index]) + start_pos - return position - - def _load_and_process_video(self, url: str, item: Dict) -> Tuple[np.ndarray, Dict]: - """ - Load and preprocess video into frames. - - Args: - url: Video file path or bytes - item: Dictionary containing processing parameters - - Returns: - tuple: (frames, metadata) where: - - frames: Processed video frames as numpy array - - metadata: Updated video metadata dictionary - """ - reader, meta, _ = read_video_decord(url, save_to_disk=False) - - # Apply frame sampling if fps or target_frames specified - fps = item.get("fps", self.fps) - num_frames = item.get("target_frames", self.target_frames) - - frame_indices = list(range(meta["num_of_frame"])) - if fps > 0 or num_frames > 0: - # Get frame sampling constraints - min_frames = item.get("min_frames", self.min_frames) - max_frames = item.get("max_frames", self.max_frames) - - # Sample frames according to specifications - frame_indices = sample_frames( - frame_factor=self.frame_factor, # Ensure divisible by temporal patch size - min_frames=min_frames, - max_frames=max_frames, - metadata=meta, - fps=fps, - num_frames=num_frames, - ) - - # Update metadata with new frame count and fps - meta["num_of_frame"] = len(frame_indices) - if fps is not None: - meta["fps"] = fps # Use specified fps - meta["duration"] = len(frame_indices) / fps - else: - meta["fps"] = len(frame_indices) / meta["duration"] # Calculate fps from sampled frames - - frames = [] - for idx in frame_indices: - frame = reader[idx].asnumpy() - image = Image.fromarray(frame, "RGB") - frames.append(image) - frames = np.stack([np.array(f.convert("RGB")) for f in frames], axis=0) - - return frames, meta - - def get_processor_cache(self, socket, mm_hashes: list[str]) -> list: - """ - get cache correspond to given hash values - """ - req = pickle.dumps(mm_hashes) - socket.send_multipart([b"", req]) - _, resp = socket.recv_multipart() - mm_items = pickle.loads(resp) - data_processor_logger.info(f"Get cache of mm_hashes: {mm_hashes}") - - return mm_items - - def update_processor_cache(self, socket, mm_hashes: list[str], mm_items): - """ - update cache data - """ - req = pickle.dumps((mm_hashes, mm_items)) - socket.send_multipart([b"", req]) - data_processor_logger.info(f"Update cache of mm_hashes: {mm_hashes}") diff --git a/fastdeploy/input/v1/qwen_vl_processor/process_video.py b/fastdeploy/input/v1/qwen_vl_processor/process_video.py deleted file mode 100644 index 891f272033b..00000000000 --- a/fastdeploy/input/v1/qwen_vl_processor/process_video.py +++ /dev/null @@ -1,100 +0,0 @@ -""" -# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -from typing import Optional, Union - -import numpy as np - -from fastdeploy.utils import data_processor_logger - -from .image_processor import ceil_by_factor, floor_by_factor - - -def sample_frames( - frame_factor: int, - min_frames: int, - max_frames: int, - metadata: Optional[dict] = None, - fps: Optional[Union[int, float]] = -1, - num_frames: Optional[int] = -1, -): - """ - Sample frames from video according to specified criteria. - - Args: - frame_factor: Ensure sampled frames are multiples of this factor - min_frames: Minimum number of frames to sample - max_frames: Maximum number of frames to sample - metadata: Video metadata containing fps information - fps: Target frames per second for sampling - num_frames: Exact number of frames to sample - - Returns: - np.ndarray: Sampled video frames - - Raises: - ValueError: If both fps and num_frames are specified, - or if required metadata is missing, - or if requested frames exceed available frames - """ - if fps > 0 and num_frames > 0: - raise ValueError("`num_frames` and `fps` are mutually exclusive arguments, please use only one!") - - total_num_frames = metadata["num_of_frame"] - - # If num_frames is not given but fps is, calculate num_frames from fps - if num_frames > 0: - num_frames = round(num_frames / frame_factor) * frame_factor - elif fps > 0: - if metadata is None: - raise ValueError( - "Asked to sample `fps` frames per second but no video metadata was provided which is required when sampling with `fps`. " - "Please pass in `VideoMetadata` object or use a fixed `num_frames` per input video" - ) - # max_frames = math.floor(min(max_frames, total_num_frames) / frame_factor) * frame_factor - min_frames = ceil_by_factor(min_frames, frame_factor) - max_frames = floor_by_factor(min(max_frames, total_num_frames), frame_factor) - - num_frames = total_num_frames / metadata["fps"] * fps - - if num_frames > total_num_frames: - data_processor_logger.warning(f"smart_nframes: nframes[{num_frames}] > total_frames[{total_num_frames}]") - - num_frames = min(min(max(num_frames, min_frames), max_frames), total_num_frames) - num_frames = floor_by_factor(num_frames, frame_factor) - - if num_frames > total_num_frames: - raise ValueError( - f"Video can't be sampled. The inferred `num_frames={num_frames}` exceeds `total_num_frames={total_num_frames}`. " - "Decrease `num_frames` or `fps` for sampling." - ) - - # Hack code ensures that num_frames can always be divided by 4 - # due to sched/resource_manager_v1.py 中 grid_thw.extend([[2, h, w]] * (t // 2)) - if num_frames > 2 and num_frames % 4 != 0: - num_frames = (num_frames // 4) * 4 # 向下取整到 4 的倍数 - total_num_frames = (total_num_frames // 4) * 4 - num_frames = min(min(max(num_frames, min_frames), max_frames), total_num_frames) - - # Calculate frame indices based on sampling strategy - if num_frames > 0: - # Evenly spaced sampling for target frame count - indices = np.arange(0, total_num_frames, total_num_frames / num_frames).astype(np.int32) - else: - # Keep all frames if no sampling requested - indices = np.arange(0, total_num_frames).astype(np.int32) - - return indices diff --git a/fastdeploy/input/v1/qwen_vl_processor/qwen_vl_processor.py b/fastdeploy/input/v1/qwen_vl_processor/qwen_vl_processor.py deleted file mode 100644 index e0d846d53a2..00000000000 --- a/fastdeploy/input/v1/qwen_vl_processor/qwen_vl_processor.py +++ /dev/null @@ -1,338 +0,0 @@ -""" -# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -import numpy as np - -from fastdeploy.engine.request import Request -from fastdeploy.input.utils import process_stop_token_ids -from fastdeploy.input.v1.text_processor import DataProcessor as TextProcessor -from fastdeploy.utils import data_processor_logger - -from .process import DataProcessor - - -class QwenVLProcessor(TextProcessor): - """ - Qwen Vision-Language processor for handling multimodal inputs. - - This processor extends TextProcessor to support: - - Image and video processing - - Multimodal feature extraction - - Tokenization and position encoding - - Request processing and model input generation - - Attributes: - processor (DataProcessor): Underlying data processor instance - tokenizer: Text tokenizer instance - limit_mm_per_prompt (dict): Limits for multimodal inputs per prompt - """ - - def __init__( - self, - config, - model_name_or_path, - limit_mm_per_prompt=None, - mm_processor_kwargs=None, - reasoning_parser_obj=None, - tool_parser_obj=None, - enable_processor_cache=False, - ): - """ - Initialize QwenVLProcessor instance. - - Args: - config: Model configuration object - model_name_or_path (str): Pretrained model name or path - limit_mm_per_prompt (dict, optional): Limits for multimodal inputs - mm_processor_kwargs (dict, optional): Multimodal processor arguments - reasoning_parser_obj: Reasoning parser instance - tool_parser_obj: Tool parser instance - """ - super().__init__(model_name_or_path, reasoning_parser_obj, tool_parser_obj) - - data_processor_logger.info(f"model_name_or_path: {model_name_or_path}") - processor_kwargs = self._parse_processor_kwargs(mm_processor_kwargs) - self.processor = DataProcessor( - model_path=model_name_or_path, - enable_processor_cache=enable_processor_cache, - tokens_per_second=config.vision_config.tokens_per_second, - tokenizer=self.tokenizer, - **processor_kwargs, - ) - self.image_patch_id = self.processor.image_token_id - self.limit_mm_per_prompt = self._parse_limits(limit_mm_per_prompt) - - def _parse_processor_kwargs(self, kwargs): - """ - Parse and validate multimodal processor arguments. - - Args: - kwargs (dict): Processor configuration arguments - - Returns: - dict: Validated processor arguments - - Raises: - ValueError: If arguments format is invalid - """ - if not kwargs: - return {} - - try: - if not isinstance(kwargs, dict): - raise ValueError("mm-processor-kwargs must be a dictionary") - - # Validate kwargs types against expected schema - data_processor_logger.info(f"Processing kwargs: {kwargs}") - expected_types = { - "video_max_frames": int, # Maximum video frames parameter - "video_min_frames": int, # Minimum video frames parameter - } - - for key, value in kwargs.items(): - if key in expected_types and not isinstance(value, expected_types[key]): - raise ValueError( - f"Invalid type for {key}: expected {expected_types[key].__name__}, got {type(value).__name__}" - ) - - return kwargs - - except Exception as e: - data_processor_logger.warning(f"Invalid mm-processor-kwargs format: {e}") - return {} - - def _parse_limits(self, limits): - """ - Parse and validate multimodal input limits. - - Args: - limits (dict): Input limits configuration - - Returns: - dict: Validated limits with defaults - - Raises: - ValueError: If limits format is invalid - """ - DEFAULT_LIMITS = {"image": 1, "video": 1, "audio": 1} - - if not limits: - return DEFAULT_LIMITS - - try: - if not isinstance(limits, dict): - raise ValueError("limit-mm-per-prompt must be a dictionary") - data_processor_logger.info(f"_parse_limits:{limits}") - return {**DEFAULT_LIMITS, **limits} - except Exception as e: - data_processor_logger.warning(f"Invalid limit-mm-per-prompt format: {e}, using default limits") - return DEFAULT_LIMITS - - def _check_mm_limits(self, item): - """ - Validate multimodal inputs against configured limits. - - Args: - item: Input request item to validate - - Raises: - ValueError: If input exceeds configured limits - """ - if isinstance(item, dict): - # 请求包含prompt和multi_modal_data - mm_data = item - else: - # 请求包含messages - mm_data = {"image": [], "video": []} - - for message in item: - if isinstance(message.get("content"), list): - for part in message["content"]: - if part.get("type") in ["image_url", "image"]: - mm_data["image"].append(part) - elif part.get("type") in ["video_url", "video"]: - mm_data["video"].append(part) - - for modality, data in mm_data.items(): - if modality in self.limit_mm_per_prompt: - limit = self.limit_mm_per_prompt[modality] - if len(data) > limit: - raise ValueError(f"Too many {modality} items in prompt, " f"got {len(data)} but limit is {limit}") - - def process_request(self, request, max_model_len=None, **kwargs): - """ - Process incoming request and generate model inputs. - - Args: - request: Input request object - max_model_len (int, optional): Maximum context length - **kwargs: Additional processing parameters - - Returns: - Request: Processed request with model inputs - """ - task = request.to_dict() - task["enable_thinking"] = kwargs.get("enable_thinking", False) - self.process_request_dict(task, max_model_len) - request = Request.from_dict(task) - request = self._apply_default_parameters(request) - return request - - def process_request_dict(self, request, max_model_len=None, **kwargs): - """ - Process request dictionary into model inputs. - - Args: - request (dict): Input request dictionary - max_model_len (int, optional): Maximum context length - - Returns: - dict: Processed request with model inputs - - Raises: - ValueError: If request format is invalid - """ - - request = self._apply_default_parameters(request) - if not request.eos_token_ids: - request.eos_token_ids = self.eos_token_ids - - # processing stop_sequences and stop_token_ids - process_stop_token_ids(request, self.update_stop_seq) - - bad_words = request.sampling_params.bad_words - bad_words_token_ids = request.sampling_params.bad_words_token_ids - if bad_words: - bad_words_token_ids = self.update_bad_words(bad_words, bad_words_token_ids) - request.sampling_params.bad_words_token_ids = bad_words_token_ids - - if request.prompt: - multimodal_data = request.multimodal_data - if multimodal_data is None: - multimodal_data = {} - self._check_mm_limits(multimodal_data) - images = multimodal_data.get("image", None) - videos = multimodal_data.get("video", None) - outputs = self.processor.text2ids(request.prompt, images, videos) - - elif request.messages: - messages = request.messages - self._check_mm_limits(messages) - chat_template_kwargs = request.chat_template_kwargs - if chat_template_kwargs: - if isinstance(chat_template_kwargs, dict): - for k, v in chat_template_kwargs.items(): - if getattr(request, k, v): - setattr(request, k, v) - else: - raise ValueError("Invalid input: chat_template_kwargs must be a dict") - if getattr(request, "enable_thinking") is None: - setattr(request, "enable_thinking", True) - outputs = self.processor.request2ids(request) - delattr(request, "chat_template_kwargs") - else: - raise ValueError(f"Request must contain 'prompt', or 'messages': {request}") - - # Handle continuation of previous generation by appending existing tokens - if request.completion_token_ids: - self.append_completion_tokens(outputs, request.completion_token_ids) - - # qwen25_vl not support thinking - request.enable_thinking = False - - outputs = self.pack_outputs(outputs) - - request.prompt_token_ids = outputs["input_ids"].tolist() - request.prompt_token_ids_len = len(request.prompt_token_ids) - request.multimodal_inputs = outputs - - # Handle prompt truncation if exceeds model context length - if max_model_len is not None and len(request.prompt_token_ids) > max_model_len: - request.prompt_token_ids = request.prompt_token_ids[ - : max_model_len - 1 - ] # Leave space for at least 1 new token - - # Set default max_tokens if not specified - max_tokens = max_model_len - len(request.prompt_token_ids) - if getattr(request.sampling_params, "max_tokens", None) is None: - request.sampling_params.max_tokens = max(1, max_tokens) - else: - request.sampling_params.max_tokens = min(max_tokens, request.sampling_params.max_tokens) - if self.reasoning_parser: - model_status = self.reasoning_parser.get_model_status(request.prompt_token_ids) - parts = request.request_id.split("_") - if len(parts) > 1: - real_req_id = parts[0] - index = int(parts[1]) - n = request.sampling_params.n or 1 - for idx in range(index * n, (index + 1) * n): - self.model_status_dict[f"{real_req_id}_{idx}"] = model_status - else: - self.model_status_dict[request.request_id] = model_status - request.enable_thinking = model_status == "think_start" - data_processor_logger.info(f"Processed request {request}") - - return request - - def append_completion_tokens(self, multimodal_inputs, completion_token_ids): - """ - Append completion tokens to existing outputs. - - Args: - outputs: Current model outputs - completion_token_ids: completion tokens to append - """ - - num_tokens = len(completion_token_ids) - multimodal_inputs["input_ids"].extend(completion_token_ids) - multimodal_inputs["token_type_ids"].extend([0] * num_tokens) - - pos_ids = self.processor._compute_text_positions(multimodal_inputs["cur_position"], num_tokens) - multimodal_inputs["position_ids"].append(pos_ids) - multimodal_inputs["cur_position"] += num_tokens - - def pack_outputs(self, outputs): - """ - Prepare final output dictionary for model. - - Args: - outputs: Intermediate processing outputs - - Returns: - dict: Packed output dictionary with all required fields - """ - if not outputs["images"]: - outputs["images"] = None # No images case - outputs["grid_thw"] = None # No spatial dimensions - outputs["image_type_ids"] = None # No type IDs - else: - outputs["images"] = np.vstack(outputs["images"]) # Stack image features vertically - outputs["grid_thw"] = np.vstack(outputs["grid_thw"]) # Stack spatial dimensions - outputs["image_type_ids"] = np.array(outputs["image_type_ids"]) # Convert to numpy array - - # Convert all outputs to numpy arrays with appropriate types - outputs["input_ids"] = np.array(outputs["input_ids"], dtype=np.int64) # Token IDs as int64 - outputs["token_type_ids"] = np.array(outputs["token_type_ids"], dtype=np.int64) # Type IDs as int64 - outputs["position_ids"] = np.concatenate( - outputs["position_ids"], axis=1, dtype=np.int64 - ) # Concatenate position ID - - outputs["image_patch_id"] = self.processor.image_token_id - outputs["video_patch_id"] = self.processor.video_token_id - outputs["position_ids"] = outputs["position_ids"].transpose(1, 0) - - outputs["mm_num_token_func"] = self.processor.mm_num_tokens - return outputs diff --git a/fastdeploy/input/v1/text_processor.py b/fastdeploy/input/v1/text_processor.py deleted file mode 100644 index f83a0e0f12e..00000000000 --- a/fastdeploy/input/v1/text_processor.py +++ /dev/null @@ -1,925 +0,0 @@ -""" -# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License" -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -from abc import ABC, abstractmethod -from collections import OrderedDict - -import numpy as np -from paddleformers.generation import GenerationConfig -from paddleformers.transformers import Llama3Tokenizer, LlamaTokenizer - -from fastdeploy import envs -from fastdeploy.input.utils import process_stop_token_ids -from fastdeploy.utils import data_processor_logger - -_SAMPLING_EPS = 1e-5 - - -class BaseDataProcessor(ABC): - """base class for data processor""" - - def __init__(self): - """ - Returns: - None - """ - self.tokenizer = self._load_tokenizer() - self.tokenizer.bos_token_id = self.tokenizer._convert_token_to_id(self.tokenizer.bos_token) - self.tokenizer.cls_token_id = self.tokenizer._convert_token_to_id(self.tokenizer.cls_token) - self.tokenizer.sep_token_id = self.tokenizer._convert_token_to_id(self.tokenizer.sep_token) - self.tokenizer.eos_token_id = self.tokenizer._convert_token_to_id(self.tokenizer.eos_token) - self.tokenizer.mask_token_id = self.tokenizer._convert_token_to_id(self.tokenizer.mask_token) - data_processor_logger.info( - ( - f"tokenizer information: bos_token is {self.tokenizer.bos_token}, {self.tokenizer.bos_token_id}, ", - f"cls_token is {self.tokenizer.cls_token}, {self.tokenizer.cls_token_id}, " - f"sep_token is {self.tokenizer.sep_token}, {self.tokenizer.sep_token_id}, " - f"eos_token is {self.tokenizer.eos_token}, {self.tokenizer.eos_token_id}, " - f"mask_token is {self.tokenizer.mask_token}, {self.tokenizer.mask_token_id}", - ) - ) - self._tokenize_cache = OrderedDict() - self._tokenize_cache_capacity = 128 - - def _apply_default_parameters(self, request): - """ - Apply default value for parameters in request - """ - - def set_value(req, key, value): - value = getattr(self.generation_config, key, value) - if getattr(req.sampling_params, key) is None: - setattr(req.sampling_params, key, value) - - set_value(request, "top_p", 0.7) - set_value(request, "temperature", 1.0) - set_value(request, "repetition_penalty", 1.0) - set_value(request, "frequency_penalty", 0.0) - set_value(request, "presence_penalty", 0.0) - return request - - @abstractmethod - def process_request_dict(self, request, **kwargs): - """ - Preprocess the request - - Args: - request Request: may contain text and messages fields - **kwargs: others - - Returns: - bool: Whether preprocessing is successful - str: error message - """ - raise NotImplementedError - - @abstractmethod - def process_response_dict(self, response_obj): - """ - Preprocess the response - - Args: - response_obj RequestOutput: response for engine, contain ids fields - - Returns: - RequestOutput: response contain text fields - """ - raise NotImplementedError - - def text2ids(self, text, max_model_len=None): - """ - text to token ids - - Args: - text (str): text - - Returns: - List[int]: token ids list - """ - raise NotImplementedError - - def encode_with_cache(self, text, max_model_len=None, add_special_tokens=False): - """ - Encode text into token ids with a small LRU cache. - """ - if not hasattr(self, "_tokenize_cache"): - self._tokenize_cache = OrderedDict() - self._tokenize_cache_capacity = getattr(self, "_tokenize_cache_capacity", 128) - key = (text, bool(add_special_tokens)) - cached = self._tokenize_cache.get(key) - if cached is not None: - self._tokenize_cache.move_to_end(key) - return cached - token_ids = self.text2ids(text, max_model_len, add_special_tokens=add_special_tokens) - if hasattr(token_ids, "tolist"): - token_ids = token_ids.tolist() - elif not isinstance(token_ids, list): - token_ids = list(token_ids) - self._tokenize_cache[key] = token_ids - if len(self._tokenize_cache) > self._tokenize_cache_capacity: - self._tokenize_cache.popitem(last=False) - return token_ids - - def _encode_literal_text_with_cache(self, text): - if not hasattr(self, "_tokenize_cache"): - self._tokenize_cache = OrderedDict() - self._tokenize_cache_capacity = getattr(self, "_tokenize_cache_capacity", 128) - key = ("literal_text", text) - cached = self._tokenize_cache.get(key) - if cached is not None: - self._tokenize_cache.move_to_end(key) - return cached - token_ids = self.tokenizer.convert_tokens_to_ids(self.tokenizer.tokenize(text)) - if hasattr(token_ids, "tolist"): - token_ids = token_ids.tolist() - elif not isinstance(token_ids, list): - token_ids = list(token_ids) - self._tokenize_cache[key] = token_ids - if len(self._tokenize_cache) > self._tokenize_cache_capacity: - self._tokenize_cache.popitem(last=False) - return token_ids - - def messages2ids(self, messages): - """ - Convert multi-turn messages into ID sequences. - - Args: - messages (List[List[Dict[str, Any]]]): multi-turn messages. - - Returns: - List[int]: ID sequences - """ - raise NotImplementedError - - def _get_think_token_ids(self): - think_token_ids = getattr(self, "_think_token_ids", None) - if think_token_ids is not None: - return think_token_ids - tokenizer = getattr(self, "tokenizer", None) - vocab = tokenizer.get_vocab() if tokenizer is not None else {} - think_start_id = vocab.get("", -1) - think_end_id = vocab.get("", -1) - self._think_token_ids = (think_start_id, think_end_id) - return self._think_token_ids - - def _prepare_think_stop_sentence(self, logits_processors_args, max_model_len=None): - if not isinstance(logits_processors_args, dict): - return logits_processors_args - think_stop_sentence = logits_processors_args.get("think_stop_sentence") - if isinstance(think_stop_sentence, str) and think_stop_sentence: - sentence_token_ids = self._encode_literal_text_with_cache(think_stop_sentence) - logits_processors_args["think_stop_sentence_token_ids"] = sentence_token_ids - logits_processors_args.pop("think_stop_sentence", None) - return logits_processors_args - - def _update_thinking_prompt_state(self, prompt_token_ids, logits_processors_args): - if not isinstance(logits_processors_args, dict): - return logits_processors_args - thinking_budget = logits_processors_args.get("thinking_budget") - if thinking_budget is None or not isinstance(thinking_budget, int) or thinking_budget < 0: - return logits_processors_args - if logits_processors_args.get("think_prompt_checked"): - return logits_processors_args - if prompt_token_ids is None: - return logits_processors_args - token_len = getattr(prompt_token_ids, "size", None) or len(prompt_token_ids) - if token_len == 0: - return logits_processors_args - think_start_id, think_end_id = self._get_think_token_ids() - if think_start_id < 0 or think_end_id < 0: - return logits_processors_args - - if hasattr(prompt_token_ids, "tolist"): - token_list = prompt_token_ids.tolist() - else: - token_list = list(prompt_token_ids) - - started = False - ended = False - tokens_after_start = 0 - last_token_id = None - in_thinking = False - for token_id in token_list: - if token_id == think_start_id: - started = True - ended = False - in_thinking = True - elif token_id == think_end_id and in_thinking: - ended = True - in_thinking = False - if started and token_list: - # Align with operator-level reasoning_max_tokens: prompt-side tokens - # inside do not consume thinking budget. - last_token_id = int(token_list[-1]) - - logits_processors_args["think_prompt_checked"] = True - logits_processors_args["think_prompt_started"] = started - logits_processors_args["think_prompt_ended"] = ended - logits_processors_args["think_prompt_tokens_after_start"] = tokens_after_start - if last_token_id is not None: - logits_processors_args["think_prompt_last_token_id"] = last_token_id - else: - logits_processors_args.pop("think_prompt_last_token_id", None) - return logits_processors_args - - def ids2tokens(self, token_id, task_id=None): - """ - token ids to strings - - Args: - token_id (List[int]): token id - task_id (str): task id - - Returns: - List[str]: strings - """ - raise NotImplementedError - - @abstractmethod - def _load_tokenizer(self): - """ - load tokenizer - - Returns: - tokenizer (AutoTokenizer) - """ - raise NotImplementedError - - -class DataProcessor(BaseDataProcessor): - def __init__(self, model_name_or_path, reasoning_parser_obj=None, tool_parser_obj=None): - """ - Initializes the DecodeStatus object. - - Args: - model_name_or_path (str): The name or path of the pre-trained model to be loaded. - Can also be a path to a directory containing the pre-trained model file. - - Returns: - None. - - Raises: - None. - """ - - self.model_name_or_path = model_name_or_path - - # Generation config - try: - self.generation_config = GenerationConfig.from_pretrained(self.model_name_or_path) - except Exception as e: - data_processor_logger.warning( - f"Can't find generation config: {e}, so it will not use generation_config field in the model config" - ) - self.generation_config = None - - self.decode_status = dict() - self.model_status_dict = dict() - self.tool_parser_dict = dict() - self.tokenizer = self._load_tokenizer() - self._tokenize_cache = OrderedDict() - self._tokenize_cache_capacity = 128 - data_processor_logger.info( - f"tokenizer information: bos_token is {self.tokenizer.bos_token}, {self.tokenizer.bos_token_id}, \ - eos_token is {self.tokenizer.eos_token}, {self.tokenizer.eos_token_id} " - ) - - try: - from paddleformers.trl.llm_utils import get_eos_token_id - except Exception: - from paddleformers.cli.utils.llm_utils import get_eos_token_id - - self.eos_token_ids = get_eos_token_id(self.tokenizer, self.generation_config) - data_processor_logger.info( - f"The eos_token_ids obtained by merging tokenizer and generation_config is {self.eos_token_ids}" - ) - self.eos_token_id_len = len(self.eos_token_ids) - self.pad_token_id = self.get_pad_id() - self.reasoning_parser = None - self.tool_parser_obj = tool_parser_obj - if reasoning_parser_obj: - self.reasoning_parser = reasoning_parser_obj(self.tokenizer) - self.tokenizer.pad_token_id = self.pad_token_id - - def process_request(self, request, max_model_len=None, **kwargs): - """ - Preprocess the request - - Args: - request (Dict): may contain text and messages fields - - Returns: - bool: Whether preprocessing is successful - str: error message - """ - data_processor_logger.info(f"Start processing request: {request}") - request = self._apply_default_parameters(request) - if request.get("eos_token_ids") is None or len(request.eos_token_ids) == 0: - request.eos_token_ids = self.eos_token_ids - - # processing stop_sequences and stop_token_ids - process_stop_token_ids(request, self.update_stop_seq) - - # processing bad_words - bad_words = request.get("bad_words") - bad_words_token_ids = request.get("bad_words_token_ids") - if bad_words: - bad_words_token_ids = self.update_bad_words(bad_words, bad_words_token_ids) - request["bad_words_token_ids"] = bad_words_token_ids - - logits_processors_args = self._prepare_think_stop_sentence( - request.get("logits_processors_args") or {}, max_model_len - ) - request["logits_processors_args"] = logits_processors_args - - # processing prompt_token_ids - if request.prompt_token_ids is None or len(request.prompt_token_ids) == 0: - if request.prompt is not None: - prompt = request.prompt - add_special_tokens = request.get("add_special_tokens", False) - assert isinstance(prompt, str) or ( - isinstance(prompt, list) and all([isinstance(t, int) for t in prompt]) - ), f"prompt must be a string or a list of integers, but got {type(prompt)}" - if isinstance(prompt, list): # if prompt is a token id list - request.prompt_token_ids = prompt - else: - request.prompt_token_ids = self.text2ids( - request.prompt, max_model_len, add_special_tokens=add_special_tokens - ) - elif request.messages is not None: - if self.tokenizer.chat_template is None: - raise ValueError("This model does not support chat_template.") - task = request.to_dict() - chat_template_kwargs = kwargs.get("chat_template_kwargs", {}) - if chat_template_kwargs: - if isinstance(chat_template_kwargs, dict): - for k, v in chat_template_kwargs.items(): - if k not in task or task[k] is None: - task[k] = v - else: - raise ValueError("Invalid input: chat_template_kwargs must be a dict") - task.setdefault("enable_thinking", True) - request.prompt_token_ids = self.messages2ids(task, **chat_template_kwargs) - else: - raise ValueError(f"The request should have `input_ids`, `text` or `messages`: {request}.") - - if len(request.prompt_token_ids) == 0: - raise ValueError("Invalid input: prompt_token_ids must be a non-empty sequence of token IDs") - - # truncate prompts that exceed the length limit - if max_model_len is not None and len(request.prompt_token_ids) > max_model_len: - request.prompt_token_ids = request.prompt_token_ids[: max_model_len - 1] - - logits_processors_args = request.get("logits_processors_args") or {} - logits_processors_args = self._update_thinking_prompt_state(request.prompt_token_ids, logits_processors_args) - request["logits_processors_args"] = logits_processors_args - - max_tokens = max_model_len - len(request.prompt_token_ids) - if request.get("max_tokens") is None: - request.set("max_tokens", max(1, max_tokens)) - else: - request.set("max_tokens", min(max_tokens, request.get("max_tokens"))) - if request.get("temperature") < _SAMPLING_EPS: - # zero temperature means greedy decoding: set top_k=1 to force argmax - request.set("temperature", 1) - request.set("top_k", 1) - if request.get("top_p") < _SAMPLING_EPS: - request.set("top_p", _SAMPLING_EPS) - request.set("top_k", 1) - if self.reasoning_parser: - model_status = self.reasoning_parser.get_model_status(request.prompt_token_ids) - parts = request.request_id.split("_") - if len(parts) > 1: - real_req_id = parts[0] - index = int(parts[1]) - n = request.get("n", 1) - for idx in range(index * n, (index + 1) * n): - self.model_status_dict[f"{real_req_id}_{idx}"] = model_status - else: - self.model_status_dict[request.request_id] = model_status - request.enable_thinking = model_status == "think_start" - - data_processor_logger.info(f"Processed request: {request}") - return request - - def process_request_dict(self, request, max_model_len=None, **kwargs): - """ - Preprocess the request - - Args: - request Request: may contain text and messages fields - - Returns: - bool: Whether preprocessing is successful - str: error message - """ - data_processor_logger.info(f"Start processing request: {request}") - request = self._apply_default_parameters(request) - if not request.eos_token_ids: - request.eos_token_ids = self.eos_token_ids - - # processing stop_sequences and stop_token_ids - process_stop_token_ids(request, self.update_stop_seq) - - # processing bad_words - bad_words = request.sampling_params.bad_words - bad_words_token_ids = request.sampling_params.bad_words_token_ids - if bad_words: - bad_words_token_ids = self.update_bad_words(bad_words, bad_words_token_ids) - request.sampling_params.bad_words_token_ids = bad_words_token_ids - - logits_processors_args = self._prepare_think_stop_sentence( - getattr(request.sampling_params, "logits_processors_args", None) or {}, max_model_len - ) - request.sampling_params.logits_processors_args = logits_processors_args - - # processing prompt_token_ids - if not request.prompt_token_ids: - if request.prompt: - prompt = request.prompt - add_special_tokens = getattr(request, "add_special_tokens", None) or False - assert isinstance(prompt, str) or ( - isinstance(prompt, list) and all([isinstance(t, int) for t in prompt]) - ), f"prompt must be a string or a list of integers, but got {type(prompt)}" - if isinstance(prompt, list): # if prompt is a token id list - request.prompt_token_ids = prompt - else: - request.prompt_token_ids = self.text2ids( - request.prompt, max_model_len, add_special_tokens=add_special_tokens - ).tolist() - elif request.messages: - if self.tokenizer.chat_template is None: - raise ValueError("This model does not support chat_template.") - chat_template_kwargs = kwargs.get("chat_template_kwargs", {}) - if not chat_template_kwargs: - chat_template_kwargs = request.chat_template_kwargs if request.chat_template_kwargs else {} - if chat_template_kwargs: - if isinstance(chat_template_kwargs, dict): - for k, v in chat_template_kwargs.items(): - if not getattr(request, k, None): - setattr(request, k, v) - else: - raise ValueError("Invalid input: chat_template_kwargs must be a dict") - if getattr(request, "enable_thinking") is None: - setattr(request, "enable_thinking", True) - request.prompt_token_ids = self.messages2ids(request, **chat_template_kwargs) - delattr(request, "chat_template_kwargs") - else: - raise ValueError(f"Request must contain 'prompt_token_ids', 'prompt', or 'messages': {request}") - - if len(request.prompt_token_ids) == 0: - raise ValueError("Invalid input: prompt_token_ids must be a non-empty sequence of token IDs") - - # truncate prompts that exceed the length limit - if max_model_len is not None and len(request.prompt_token_ids) > max_model_len: - request.prompt_token_ids = request.prompt_token_ids[: max_model_len - 1] - logits_processors_args = getattr(request.sampling_params, "logits_processors_args", None) or {} - logits_processors_args = self._update_thinking_prompt_state(request.prompt_token_ids, logits_processors_args) - request.sampling_params.logits_processors_args = logits_processors_args - - max_tokens = max_model_len - len(request.prompt_token_ids) - if getattr(request.sampling_params, "max_tokens", None) is None: - request.sampling_params.max_tokens = max(1, max_tokens) - else: - request.sampling_params.max_tokens = min(max_tokens, request.sampling_params.max_tokens) - - if request.sampling_params.temperature < _SAMPLING_EPS: - # zero temperature means greedy decoding: set top_k=1 to force argmax - request.sampling_params.temperature = 1 - request.sampling_params.top_k = 1 - if request.sampling_params.top_p < _SAMPLING_EPS: - request.sampling_params.top_p = _SAMPLING_EPS - request.sampling_params.top_k = 1 - if self.reasoning_parser: - model_status = self.reasoning_parser.get_model_status(request.prompt_token_ids) - parts = request.request_id.split("_") - if len(parts) > 1: - real_req_id = parts[0] - index = int(parts[1]) - n = request.sampling_params.n or 1 - for idx in range(index * n, (index + 1) * n): - self.model_status_dict[f"{real_req_id}_{idx}"] = model_status - else: - self.model_status_dict[request.request_id] = model_status - request.enable_thinking = model_status == "think_start" - - data_processor_logger.info(f"Processed request: {request}") - return request - - def process_logprob_response(self, token_ids, **kwargs): - full_text = self.tokenizer.decode(token_ids, **kwargs) - return full_text - - def process_response(self, response_dict, **kwargs): - """ - Preprocess the response - - Args: - response_dict (Dict): response for engine, contain ids fields - - Returns: - Dict: response contain text fields - """ - req_id = response_dict.request_id - token_ids = response_dict.outputs.token_ids - if token_ids[-1] == self.tokenizer.eos_token_id: - token_ids = token_ids[:-1] - full_text = self.tokenizer.decode(token_ids) - response_dict.outputs.text = full_text - if self.reasoning_parser: - reasoning_content, text = self.reasoning_parser.extract_reasoning_content( - full_text, response_dict, self.model_status_dict[req_id] - ) - response_dict.outputs.text = text - response_dict.outputs.reasoning_content = reasoning_content - if self.tool_parser_obj: - tool_parser = self.tool_parser_obj(self.tokenizer) - tool_call_info = tool_parser.extract_tool_calls(full_text, response_dict) - if tool_call_info.tools_called: - response_dict.outputs.tool_calls = tool_call_info.tool_calls - response_dict.outputs.text = tool_call_info.content - if req_id in self.model_status_dict: - del self.model_status_dict[req_id] - data_processor_logger.info(f"req_id:{req_id}, token_ids: {token_ids}") - - return response_dict - - def process_response_obj_normal(self, response_obj, **kwargs): - """ - Preprocess the response - - Args: - response_obj :response for engine, contain ids fields - - Returns: - RequestOutput: response contain text fields - """ - output = response_obj.outputs - token_ids = output.token_ids - is_end = response_obj.finished - req_id = response_obj.request_id - request = kwargs.get("request", None) - if is_end and len(token_ids) > 0 and not kwargs.get("include_stop_str_in_output"): - if token_ids[-1] in self.eos_token_ids: - token_ids = token_ids[:-1] - delta_text, _, previous_texts = self.ids2tokens(token_ids, req_id) - if is_end: - full_text = previous_texts + delta_text - response_obj.outputs.completion_tokens = full_text - response_obj.outputs.text = full_text - if self.reasoning_parser: - reasoning_content, text = self.reasoning_parser.extract_reasoning_content( - full_text, - request, - self.model_status_dict[req_id], - ) - response_obj.outputs.text = text - response_obj.outputs.reasoning_content = reasoning_content - reasoning_tokens = self.tokenizer.tokenize(reasoning_content) if reasoning_content else [] - response_obj.outputs.reasoning_token_num = len(reasoning_tokens) - if self.tool_parser_obj: - tool_parser = self.tool_parser_obj(self.tokenizer) - tool_call_info = tool_parser.extract_tool_calls(full_text, request) - if tool_call_info.tools_called: - response_obj.outputs.tool_calls = tool_call_info.tool_calls - response_obj.outputs.text = tool_call_info.content - data_processor_logger.info(f"req_id:{req_id}, decode_status: {self.decode_status[req_id]}") - del self.decode_status[req_id] - if req_id in self.model_status_dict: - del self.model_status_dict[req_id] - return response_obj - - def process_response_obj_streaming(self, response_obj, **kwargs): - """ - Preprocess the response - - Args: - response_obj : response for engine, contain ids fields - - Returns: - RequestOutput: response contain text fields - """ - output = response_obj.outputs - token_ids = output.token_ids - is_end = response_obj.finished - req_id = response_obj.request_id - request = kwargs.get("request", None) - - if is_end and len(token_ids) > 0 and not kwargs.get("include_stop_str_in_output"): - if token_ids[-1] in self.eos_token_ids: - token_ids = token_ids[:-1] - delta_text, previous_token_ids, previous_texts = self.ids2tokens(token_ids, req_id) - response_obj.outputs.completion_tokens = delta_text - if self.reasoning_parser: - reasoning_delta_message = self.reasoning_parser.extract_reasoning_content_streaming( - previous_texts, - previous_texts + delta_text, - delta_text, - previous_token_ids, - previous_token_ids + token_ids, - token_ids, - self.model_status_dict[req_id], - ) - response_obj.outputs.delta_message = reasoning_delta_message - reasoning_content = reasoning_delta_message.reasoning_content if reasoning_delta_message else None - reasoning_tokens = self.tokenizer.tokenize(reasoning_content) if reasoning_content else [] - response_obj.outputs.reasoning_token_num = len(reasoning_tokens) - if self.tool_parser_obj: - if req_id not in self.tool_parser_dict: - self.tool_parser_dict[req_id] = self.tool_parser_obj(self.tokenizer) - tool_parser = self.tool_parser_dict[req_id] - tool_call = tool_parser.extract_tool_calls_streaming( - previous_texts, - previous_texts + delta_text, - delta_text, - previous_token_ids, - previous_token_ids + token_ids, - token_ids, - request, - ) - if tool_call is None or tool_call.tool_calls: - response_obj.outputs.delta_message = tool_call - response_obj.outputs.text = delta_text - if is_end: - data_processor_logger.info(f"req_id:{req_id}, decode_status: {self.decode_status[req_id]}") - del self.decode_status[req_id] - if req_id in self.tool_parser_dict: - del self.tool_parser_dict[req_id] - if req_id in self.model_status_dict: - del self.model_status_dict[req_id] - return response_obj - - def process_response_dict(self, response_dict, **kwargs): - """ - Preprocess the response - - Args: - response_obj: response for engine, contain ids fields - - Returns: - Dict: response contain text fields - """ - stream = kwargs.get("stream", True) - if stream: - return self.process_response_obj_streaming(response_dict, **kwargs) - else: - return self.process_response_obj_normal( - response_dict, - **kwargs, - ) - - def text2ids(self, text, max_model_len, **kwargs): - """ - text to token ids - - Args: - text (str): text - - Returns: - List[int]: token ids list - """ - - add_special_tokens = kwargs.get("add_special_tokens") - if envs.FD_USE_HF_TOKENIZER: - tokens = self.tokenizer( - text, - return_tensors="np", - padding=True, - truncation=True, - ) - else: - text = [text] if isinstance(text, str) else text - - tokens = self.tokenizer( - text, - return_tensors="np", - padding=True, - truncation=True, - max_length=max_model_len, - add_special_tokens=add_special_tokens, - ) - - return tokens["input_ids"][0] - - def messages2ids(self, request, **kwargs): - """ - Convert multi-turn messages into ID sequences. - - Args: - messages (List[List[Dict[str, Any]]]): multi-turn messages. - - Returns: - List[int]: ID sequences - """ - message_dict = { - key: getattr(request, key, None) - for key in ["messages", "tools", "documents", "enable_thinking", "system"] - if getattr(request, key, None) is not None - } - if "add_generation_prompt" not in kwargs: - kwargs["add_generation_prompt"] = ( - request.add_generation_prompt if request.add_generation_prompt is not None else True - ) - spliced_message = self.tokenizer.apply_chat_template( - message_dict, - tokenize=False, - split_special_tokens=False, - add_special_tokens=False, - **kwargs, - ) - request.prompt_tokens = spliced_message - tokens = self.tokenizer.tokenize(spliced_message) - req_id = getattr(request, "request_id", None) - token_ids = self.tokenizer.convert_tokens_to_ids(tokens) - data_processor_logger.info(f"req_id:{req_id}, tokens:{tokens}, token_ids: {token_ids}") - return token_ids - - def ids2tokens(self, token_id, task_id): - """ - token ids to strings - - Args: - token_ids (List[int]): token ids - task_id (str): task id - - Returns: - List[str]: strings - """ - if envs.FD_USE_HF_TOKENIZER: - if task_id not in self.decode_status: - # history token ids & history token strings & befer decode str - self.decode_status[task_id] = [[], [], ""] - - previous_token_ids = self.decode_status[task_id][0] - decode_str = self.tokenizer.batch_decode( - [previous_token_ids + token_id], - skip_special_tokens=True, - clean_up_tokenization_spaces=False, - ) - if isinstance(decode_str, list) and len(decode_str): - new_str = decode_str[0].replace(self.decode_status[task_id][2], "", 1) - self.decode_status[task_id][1].append(new_str) - self.decode_status[task_id][2] = decode_str[0] - else: - new_str = "" - self.decode_status[task_id][0] += token_id - return new_str - else: - if task_id not in self.decode_status: - # prefix offset & read offset & history token ids & history token strings - self.decode_status[task_id] = [0, 0, [], ""] - - prefix_offset = self.decode_status[task_id][0] - read_offset = self.decode_status[task_id][1] - previous_token_ids = self.decode_status[task_id][2] - previous_texts = self.decode_status[task_id][3] - decode_str, prefix_offset, read_offset = self.tokenizer.decode_token( - previous_token_ids + token_id, prefix_offset, read_offset - ) - self.decode_status[task_id][0] = prefix_offset - self.decode_status[task_id][1] = read_offset - self.decode_status[task_id][2] += token_id - self.decode_status[task_id][3] += decode_str - - return decode_str, previous_token_ids, previous_texts - - def _load_tokenizer(self): - """ - load tokenizer - - Returns: - tokenizer (AutoTokenizer) - """ - if envs.FD_USE_HF_TOKENIZER: - from transformers import AutoTokenizer - - return AutoTokenizer.from_pretrained(self.model_name_or_path, use_fast=False) - else: - from paddleformers.transformers import AutoTokenizer - - return AutoTokenizer.from_pretrained(self.model_name_or_path, padding_side="left", use_fast=True) - - def clear_request_status(self, task_id): - """ - clear request status - - Args: - task_id (str): task id - - Returns: - results_all (str): all token strings - """ - results_all = "" - if task_id in self.decode_status: - if envs.FD_USE_HF_TOKENIZER: - results_all = self.decode_status[task_id][2] - else: - results_all = "".join(self.decode_status[task_id][3]) - del self.decode_status[task_id] - return results_all - - def get_pad_id(self): - """ - get pad_token_id, if not pad_token_id, use eos_token - - Returns: - int: pad_token_id - """ - if isinstance(self.tokenizer, (LlamaTokenizer, Llama3Tokenizer)) and not self.tokenizer.pad_token_id: - return self.tokenizer.eos_token - return self.tokenizer.pad_token_id - - def pad_batch_data( - self, - insts, - pad_id=0, - return_seq_len=False, - return_array=True, - pad_style="right", - ): - """Pad the instances to the max sequence length in batch.""" - if len(insts) == 0: - padded_insts = np.array([[]], dtype=np.int64) if return_array else [[]] - if return_seq_len: - seq_len = np.array([], dtype=np.int64) if return_array else [] - return padded_insts, seq_len - return padded_insts - - max_len = max(map(len, insts)) - if pad_style == "left": - padded_insts = [[pad_id] * (max_len - len(inst)) + list(inst) for inst in insts] - else: - padded_insts = [list(inst) + [pad_id] * (max_len - len(inst)) for inst in insts] - if return_array: - padded_insts = np.array(padded_insts, dtype=np.int64).reshape([-1, max_len]) - - if return_seq_len: - seq_len = [len(inst) for inst in insts] - if return_array: - seq_len = np.array(seq_len, dtype=np.int64).reshape(-1, 1) - return padded_insts, seq_len - return padded_insts - - def update_stop_seq(self, stop_sequences): - """ - Update stop sequences from request. - """ - stop_seqs = [] - for seq in stop_sequences: - if seq != self.tokenizer.eos_token_id: - stop_seqs.append(self.tokenizer.convert_tokens_to_ids(self.tokenizer.tokenize(seq))) - stop_seqs, stop_seqs_len = self.pad_batch_data(stop_seqs, pad_id=-1, return_seq_len=True, return_array=False) - data_processor_logger.debug(f"processed stop_seqs: {stop_seqs}, {stop_seqs_len}") - return stop_seqs, stop_seqs_len - - def update_bad_words(self, bad_words, bad_words_token_ids): - """Support bad words""" - - token_ids = bad_words_token_ids - - if token_ids is None: - token_ids = [] - for bad_word in bad_words: - # To prohibit words both at the beginning - # and in the middle of text - # (related to add_prefix_space tokenizer parameter) - for add_prefix_space in [False, True]: - prefix = " " if add_prefix_space else "" - prompt = prefix + bad_word.lstrip() - prompt_token_ids = self.tokenizer.convert_tokens_to_ids(self.tokenizer.tokenize(prompt)) - - if len(prompt_token_ids) != 1: - if not add_prefix_space: - data_processor_logger.warning( - f"Skip bad_words: <{prompt}>." - f"Bad words should be a single token." - f"Got tokens: {prompt_token_ids}." - ) - continue - - if prompt_token_ids[0] > self.tokenizer.vocab_size: - if not add_prefix_space: - data_processor_logger.warning( - f"Skip bad_words: <{prompt}>." - f"All token id values should be satisfying:" - f" 0 <= token_id < {self.tokenizer.vocab_size}." - f"Got token: {prompt_token_ids}." - ) - continue - - if prompt_token_ids not in token_ids: - token_ids.extend(prompt_token_ids) - return token_ids diff --git a/fastdeploy/inter_communicator/zmq_server.py b/fastdeploy/inter_communicator/zmq_server.py index 7073edb48a5..c88fe96eb08 100644 --- a/fastdeploy/inter_communicator/zmq_server.py +++ b/fastdeploy/inter_communicator/zmq_server.py @@ -169,10 +169,7 @@ def pack_aggregated_data(self, data): if len(data) > 1: for response in data[1:]: result.add(response) - if not envs.ENABLE_V1_DATA_PROCESSOR: - result = ForkingPickler.dumps([result.to_dict()]) - else: - result = ForkingPickler.dumps([result]) + result = ForkingPickler.dumps([result.to_dict()]) return result def receive_json_once(self, block=False): @@ -303,10 +300,7 @@ def _send_response_per_query(self, req_id, data): if self.aggregate_send: result = self.pack_aggregated_data(new_data) else: - if not envs.ENABLE_V1_DATA_PROCESSOR: - result = ForkingPickler.dumps([response.to_dict() for response in new_data]) - else: - result = ForkingPickler.dumps(new_data) + result = ForkingPickler.dumps([response.to_dict() for response in new_data]) with self.response_token_lock: _zmq_metrics_stats = ZMQMetricsStats() @@ -349,13 +343,10 @@ def _send_batch_response(self, batch_data, worker_pid=None): metrics_address = self.address or self.worker_push_addresses.get(worker_pid, "unknown") try: - if not envs.ENABLE_V1_DATA_PROCESSOR: - result = msgpack.packb( - [[output.to_dict() for output in outputs] for outputs in batch_data], - default=_msgpack_default, - ) - else: - result = ForkingPickler.dumps(batch_data) + result = msgpack.packb( + [[output.to_dict() for output in outputs] for outputs in batch_data], + default=_msgpack_default, + ) result_len = len(result) # Only hold lock for the actual socket send diff --git a/tests/engine/test_common_engine.py b/tests/engine/test_common_engine.py index 5a6241c4433..69c6db2a753 100644 --- a/tests/engine/test_common_engine.py +++ b/tests/engine/test_common_engine.py @@ -1267,7 +1267,6 @@ def receive_json_once(self, block): with ( patch("fastdeploy.engine.common_engine.main_process_metrics", DummyMetrics()), - patch("fastdeploy.engine.common_engine.envs.ENABLE_V1_DATA_PROCESSOR", False), patch("fastdeploy.engine.common_engine.envs.ZMQ_SEND_BATCH_DATA", False), patch("fastdeploy.engine.common_engine.time.sleep", lambda *_: None), ): @@ -3277,7 +3276,6 @@ def __init__(self): with ( patch("fastdeploy.engine.common_engine.envs.ZMQ_SEND_BATCH_DATA", True), patch("fastdeploy.engine.common_engine.envs.FD_ENABLE_INTERNAL_ADAPTER", False), - patch("fastdeploy.engine.common_engine.envs.ENABLE_V1_DATA_PROCESSOR", False), patch("fastdeploy.engine.common_engine.main_process_metrics", DummyMetrics()), patch("fastdeploy.engine.common_engine.time.sleep", lambda *_: None), ): diff --git a/tests/entrypoints/openai/test_serving_chat.py b/tests/entrypoints/openai/test_serving_chat.py index 8af369c396a..1b33405503f 100644 --- a/tests/entrypoints/openai/test_serving_chat.py +++ b/tests/entrypoints/openai/test_serving_chat.py @@ -20,7 +20,6 @@ from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, Mock, patch -import numpy as np import paddle import fastdeploy.envs as envs @@ -307,51 +306,29 @@ async def test_create_chat_completion_request_id_and_v1_stream(self): self.chat_completion_handler.engine_client.format_and_add_data = AsyncMock( side_effect=ParameterError("param", "bad") ) - with patch.object(envs, "ENABLE_V1_DATA_PROCESSOR", False): - with patch("fastdeploy.entrypoints.openai.serving_chat.tracing.trace_req_start") as mock_trace: - resp = await self.chat_completion_handler.create_chat_completion( - ChatCompletionRequest( - messages=[{"role": "user", "content": "Hello"}], - request_id="abc", - stream=False, - ) + with patch("fastdeploy.entrypoints.openai.serving_chat.tracing.trace_req_start") as mock_trace: + resp = await self.chat_completion_handler.create_chat_completion( + ChatCompletionRequest( + messages=[{"role": "user", "content": "Hello"}], + request_id="abc", + stream=False, ) + ) self.assertEqual(resp.error.param, "param") self.assertIn("bad", resp.error.message) self.assertEqual(mock_trace.call_args.kwargs["rid"], "chatcmpl-abc") self.chat_completion_handler.engine_client.format_and_add_data = AsyncMock(side_effect=RuntimeError("boom")) - with patch.object(envs, "ENABLE_V1_DATA_PROCESSOR", False): - with patch("fastdeploy.entrypoints.openai.serving_chat.tracing.trace_req_start"): - resp = await self.chat_completion_handler.create_chat_completion( - ChatCompletionRequest( - messages=[{"role": "user", "content": "Hello"}], - request_id="err", - stream=False, - ) + with patch("fastdeploy.entrypoints.openai.serving_chat.tracing.trace_req_start"): + resp = await self.chat_completion_handler.create_chat_completion( + ChatCompletionRequest( + messages=[{"role": "user", "content": "Hello"}], + request_id="err", + stream=False, ) + ) self.assertIn("generator error", resp.error.message) - self.chat_completion_handler.engine_client.format_and_add_data = AsyncMock(return_value=np.array([1, 2])) - stream_mock = Mock(return_value="streamed") - with patch.object(envs, "ENABLE_V1_DATA_PROCESSOR", True): - with patch( - "fastdeploy.entrypoints.openai.serving_chat.Request.from_generic_request", - return_value={"metrics": {}, "prompt_tokens": "pt", "max_tokens": 3}, - ): - with patch("fastdeploy.entrypoints.openai.serving_chat.tracing.trace_req_start") as mock_trace: - with patch.object(self.chat_completion_handler, "chat_completion_stream_generator", stream_mock): - result = await self.chat_completion_handler.create_chat_completion( - ChatCompletionRequest( - messages=[{"role": "user", "content": "Hello"}], - user="user", - stream=True, - ) - ) - self.assertEqual(result, "streamed") - self.assertTrue(mock_trace.call_args.kwargs["rid"].startswith("chatcmpl-user-")) - self.assertEqual(stream_mock.call_args.args[3], [1, 2]) - async def test_create_chat_completion_full_and_waiting_errors(self): """Test full generator error and waiting error handling.""" self.chat_completion_handler.engine_client.is_master = True @@ -361,15 +338,14 @@ async def test_create_chat_completion_full_and_waiting_errors(self): self.chat_completion_handler.engine_client.semaphore.status = Mock(return_value="ok") self.chat_completion_handler.engine_client.format_and_add_data = AsyncMock(return_value=[1, 2]) - with patch.object(envs, "ENABLE_V1_DATA_PROCESSOR", False): - with patch.object( - self.chat_completion_handler, - "chat_completion_full_generator", - AsyncMock(side_effect=RuntimeError("boom")), - ): - resp = await self.chat_completion_handler.create_chat_completion( - ChatCompletionRequest(messages=[{"role": "user", "content": "Hello"}], stream=False) - ) + with patch.object( + self.chat_completion_handler, + "chat_completion_full_generator", + AsyncMock(side_effect=RuntimeError("boom")), + ): + resp = await self.chat_completion_handler.create_chat_completion( + ChatCompletionRequest(messages=[{"role": "user", "content": "Hello"}], stream=False) + ) self.assertIn("full generator error", resp.error.message) with patch( diff --git a/tests/entrypoints/test_serving_completion.py b/tests/entrypoints/test_serving_completion.py index b76d798afc5..9c2beb678df 100644 --- a/tests/entrypoints/test_serving_completion.py +++ b/tests/entrypoints/test_serving_completion.py @@ -20,7 +20,6 @@ import numpy as np import paddle -import fastdeploy.envs as envs import fastdeploy.metrics.trace as tracing from fastdeploy.entrypoints.openai.serving_completion import OpenAIServingCompletion from fastdeploy.utils import ErrorCode, ParameterError @@ -124,40 +123,21 @@ async def test_create_completion_branches(self): ec = _make_engine_client() ec.format_and_add_data = AsyncMock(side_effect=ParameterError("max_tokens", "bad")) serving = OpenAIServingCompletion(ec, None, "pid", None, -1) - with patch.object(envs, "ENABLE_V1_DATA_PROCESSOR", False): - res = await _assert_error(self, serving, _make_request(prompt_token_ids=[1, 2]), param="max_tokens") + res = await _assert_error(self, serving, _make_request(prompt_token_ids=[1, 2]), param="max_tokens") ec.semaphore.release.assert_called_once() ec = _make_engine_client() - ec.format_and_add_data = AsyncMock(side_effect=ValueError("bad")) - serving = OpenAIServingCompletion(ec, None, "pid", None, -1) - - def fake_from_generic_request(_, request_id): - return {"prompt": "hi", "request_id": request_id, "prompt_tokens": [1], "max_tokens": 2, "metrics": {}} - - with patch.object(envs, "ENABLE_V1_DATA_PROCESSOR", True): - with patch( - "fastdeploy.entrypoints.openai.serving_completion.Request.from_generic_request", - side_effect=fake_from_generic_request, - ): - await _assert_error(self, serving, _make_request(prompt="hi"), code=ErrorCode.INVALID_VALUE) - ec = _make_engine_client() ec.format_and_add_data = AsyncMock(return_value=np.array([1, 2])) serving = OpenAIServingCompletion(ec, None, "pid", None, -1) - with patch.object(envs, "ENABLE_V1_DATA_PROCESSOR", False): - with patch.object(serving, "completion_full_generator", AsyncMock(side_effect=RuntimeError("boom"))): - await _assert_error( - self, serving, _make_request(prompt="hi"), contains="completion_full_generator error" - ) + with patch.object(serving, "completion_full_generator", AsyncMock(side_effect=RuntimeError("boom"))): + await _assert_error(self, serving, _make_request(prompt="hi"), contains="completion_full_generator error") serving = OpenAIServingCompletion(_make_engine_client(), None, "pid", None, -1) - with patch.object(envs, "ENABLE_V1_DATA_PROCESSOR", False): - with patch.object(serving, "completion_stream_generator", return_value="streamed"): - res = await serving.create_completion(_make_request(request_id="req123", stream=True)) + with patch.object(serving, "completion_stream_generator", return_value="streamed"): + res = await serving.create_completion(_make_request(request_id="req123", stream=True)) self.assertEqual(res, "streamed") serving = OpenAIServingCompletion(_make_engine_client(), None, "pid", None, -1) - with patch.object(envs, "ENABLE_V1_DATA_PROCESSOR", False): - await _assert_error( - self, serving, _StreamRaiser(**_make_request().__dict__), contains="create_completion error" - ) + await _assert_error( + self, serving, _StreamRaiser(**_make_request().__dict__), contains="create_completion error" + ) async def test_completion_full_generator_branches(self): ec = _make_engine_client() diff --git a/tests/input/test_preprocess.py b/tests/input/test_preprocess.py index b4659261a8e..4196b729fbb 100644 --- a/tests/input/test_preprocess.py +++ b/tests/input/test_preprocess.py @@ -63,10 +63,8 @@ def test_create_processor_text_normal_path(self): mock_dp = MagicMock() with ( patch.dict("sys.modules", {"fastdeploy.plugins": None, "fastdeploy.plugins.input_processor": None}), - patch("fastdeploy.input.preprocess.envs") as mock_envs, patch("fastdeploy.input.text_processor.DataProcessor", return_value=mock_dp), ): - mock_envs.ENABLE_V1_DATA_PROCESSOR = False pp.create_processor() self.assertIs(pp.processor, mock_dp) diff --git a/tests/input/v1/test_ernie4_5_processor.py b/tests/input/v1/test_ernie4_5_processor.py deleted file mode 100644 index 13cfdb10747..00000000000 --- a/tests/input/v1/test_ernie4_5_processor.py +++ /dev/null @@ -1,448 +0,0 @@ -""" -# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -import unittest -from unittest.mock import MagicMock, patch - -import numpy as np - -from fastdeploy.engine.request import Request, RequestOutput - -MODULE_PATH = "fastdeploy.input.v1.ernie4_5_processor" - -from fastdeploy.input.v1.ernie4_5_processor import _SAMPLING_EPS, Ernie4_5Processor - - -class MockTokenizer: - """A simple mock tokenizer used to simulate tokenization behavior in unit tests.""" - - def __init__(self): - self.bos_token = "" - self.bos_token_id = 101 - self.eos_token = "" - self.eos_token_id = 102 - self.pad_token_id = 0 - self.vocab_size = 200 - # Non-None value indicates chat_template support - self.chat_template = "dummy" - - def tokenize(self, text): - """Return multi-token output for 'multi*' to test branching; otherwise return single-token.""" - if text.startswith("multi"): - return ["multi", "word"] - return [text] - - def convert_tokens_to_ids(self, tokens): - """Map tokens to synthetic IDs for branch coverage.""" - mapping = { - "bad": 5, - " bad": 6, - "multi": 7, - "word": 8, - "oov": 250, - " oov": 251, - "hello": 9, - "REASON": 42, - } - return [mapping.get(t, 1) for t in tokens] - - def decode(self, token_ids, **kwargs): - """Simple decode implementation returning a space-separated string.""" - return " ".join(str(t) for t in token_ids) - - def decode_token(self, token_ids, prefix_offset, read_offset): - """Incremental decode used to test streaming behavior.""" - new_tokens = token_ids[read_offset:] - decode_str = " ".join(str(t) for t in new_tokens) - new_read_offset = len(token_ids) - return decode_str, prefix_offset, new_read_offset - - def apply_chat_template(self, request_or_messages, tokenize, split_special_tokens, add_special_tokens, **kwargs): - """Minimal chat template implementation used by messages2ids.""" - if isinstance(request_or_messages, dict) and "messages" in request_or_messages: - return " | ".join(m["content"] for m in request_or_messages["messages"]) - return str(request_or_messages) - - -class ErnieX1ReasoningParser: - """Mock reasoning parser to trigger reasoning-related branches during streaming.""" - - def __init__(self, tokenizer): - self.tokenizer = tokenizer - - def extract_reasoning_content(self, full_text, response_dict, model_status): - """Extract reasoning content for non-streaming responses.""" - - class ReasoningContent: - def __init__(self): - self.reasoning_content = "mock_reasoning" - self.content = "mock_content" - - return ReasoningContent() - - def extract_reasoning_content_streaming( - self, - previous_texts, - full_text, - delta_text, - previous_token_ids, - all_token_ids, - delta_token_ids, - model_status, - ): - """Return a simple object with reasoning_content to cover reasoning branch.""" - - class ReasoningDelta: - def __init__(self, content): - self.reasoning_content = content - - return ReasoningDelta(delta_text) - - -class MockToolParser: - """Mock tool parser to cover tool-related branches in both normal and streaming responses.""" - - def __init__(self, tokenizer): - self.tokenizer = tokenizer - - class ToolDelta: - """Simple container representing detected tool calls.""" - - def __init__(self): - self.tool_calls = [{"name": "fake_tool"}] - self.tools_called = True - self.content = "tool_content" - - def extract_tool_calls(self, full_text, response_dict): - """Used in process_response and process_response_obj_normal.""" - return MockToolParser.ToolDelta() - - def extract_tool_calls_streaming( - self, - previous_texts, - full_text, - delta_text, - previous_token_ids, - all_token_ids, - delta_token_ids, - response_dict, - ): - """Used in process_response_obj_streaming.""" - return MockToolParser.ToolDelta() - - -class TestErnie4_5Processor(unittest.TestCase): - """Unit tests for Ernie4_5Processor focusing on preprocessing and postprocessing logic.""" - - def setUp(self): - """Patch external dependencies: tokenizer, generation config, eos token resolution.""" - self.gen_patcher = patch(f"{MODULE_PATH}.GenerationConfig.from_pretrained", return_value=MagicMock()) - self.tokenizer_patcher = patch( - f"{MODULE_PATH}.Ernie4_5Tokenizer.from_pretrained", side_effect=lambda path: MockTokenizer() - ) - self.eos_patcher = patch( - "paddleformers.cli.utils.llm_utils.get_eos_token_id", - side_effect=lambda tokenizer, cfg: [tokenizer.eos_token_id], - ) - - self.gen_patcher.start() - self.tokenizer_patcher.start() - self.eos_patcher.start() - - def tearDown(self): - """Stop all patches after each test.""" - self.gen_patcher.stop() - self.tokenizer_patcher.stop() - self.eos_patcher.stop() - - def _make_processor(self, reasoning=False, tool=False): - """Helper for creating a processor with optional reasoning/tool parser support.""" - reasoning_cls = ErnieX1ReasoningParser if reasoning else None - tool_cls = MockToolParser if tool else None - proc = Ernie4_5Processor("dummy-model", reasoning_parser_obj=reasoning_cls, tool_parser_obj=tool_cls) - proc._apply_default_parameters = lambda req: req - proc.model_status_dict = {"req-1": "think_start"} - return proc - - def test_update_bad_words(self): - """Verify filtering, multi-token skipping, and OOV behavior in update_bad_words.""" - proc = self._make_processor() - - bad_words = ["bad", "multi", "oov"] - token_ids = proc.update_bad_words(bad_words, bad_words_token_ids=None) - - self.assertEqual(token_ids, [5, 6, 1]) - - def test_process_request_dict_with_prompt_string(self): - """Test prompt-based tokenization, truncation, and temperature/top_p correction.""" - proc = self._make_processor() - req = { - "request_id": "test_0", - "prompt": "hello", - "temperature": 0.0, - "top_p": 0.0, - } - req = Request.from_dict(req) - - processed = proc.process_request_dict(req, max_model_len=10) - - self.assertTrue(hasattr(processed, "eos_token_ids")) - self.assertEqual(processed.eos_token_ids, [proc.tokenizer.eos_token_id]) - - expected_ids = proc.tokenizer.convert_tokens_to_ids(proc.tokenizer.tokenize("hello")) - self.assertEqual(processed.prompt_token_ids, expected_ids) - - self.assertEqual(processed.sampling_params.max_tokens, max(1, 10 - len(expected_ids))) - self.assertEqual(processed.sampling_params.temperature, 1) - self.assertEqual(processed.sampling_params.top_k, 1) - self.assertAlmostEqual(processed.sampling_params.top_p, _SAMPLING_EPS) - self.assertEqual(processed.prompt_tokens, "hello") - - def test_pad_batch_data_right_and_left_and_empty(self): - """Test left/right padding and empty input behavior.""" - proc = self._make_processor() - - insts = [[1, 2], [3]] - - padded, seq_len = proc.pad_batch_data( - insts, pad_id=0, return_seq_len=True, return_array=True, pad_style="right" - ) - np.testing.assert_array_equal(padded, np.array([[1, 2], [3, 0]], dtype=np.int64)) - np.testing.assert_array_equal(seq_len, np.array([[2], [1]], dtype=np.int64)) - - padded_left, seq_len_left = proc.pad_batch_data( - insts, pad_id=0, return_seq_len=True, return_array=True, pad_style="left" - ) - np.testing.assert_array_equal(padded_left, np.array([[1, 2], [0, 3]], dtype=np.int64)) - np.testing.assert_array_equal(seq_len_left, np.array([[2], [1]], dtype=np.int64)) - - padded_empty, seq_len_empty = proc.pad_batch_data( - [], pad_id=0, return_seq_len=True, return_array=True, pad_style="right" - ) - np.testing.assert_array_equal(padded_empty, np.array([[]], dtype=np.int64)) - np.testing.assert_array_equal(seq_len_empty, np.array([], dtype=np.int64)) - - def test_process_response_obj_streaming_with_reasoning_and_tool(self): - """Ensure streaming mode handles reasoning and tool-call parsing correctly.""" - proc = self._make_processor(reasoning=True, tool=True) - - response = { - "finished": True, - "request_id": "req-1", - "outputs": {"token_ids": [10, 11]}, - } - response = RequestOutput.from_dict(response) - - result = proc.process_response_obj_streaming(response, enable_thinking=False, include_stop_str_in_output=False) - - outputs = result.outputs - - self.assertTrue(hasattr(outputs, "completion_tokens")) - self.assertTrue(hasattr(outputs, "text")) - self.assertEqual(outputs.completion_tokens, outputs.reasoning_content) - - self.assertTrue(hasattr(outputs, "reasoning_token_num")) - self.assertGreaterEqual(outputs.reasoning_token_num, 0) - - self.assertTrue(hasattr(outputs, "delta_message")) - delta_msg = outputs.delta_message - self.assertTrue(hasattr(delta_msg, "tool_calls")) - - self.assertNotIn("req-1", proc.decode_status) - self.assertNotIn("req-1", proc.tool_parser_dict) - - def test_update_stop_seq(self): - """Test stop sequence tokenization and padding.""" - proc = self._make_processor() - - stop_seqs, stop_lens = proc.update_stop_seq("stop") - self.assertIsInstance(stop_seqs, list) - self.assertIsInstance(stop_lens, list) - - stop_seqs2, stop_lens2 = proc.update_stop_seq(["stop", "hello"]) - self.assertEqual(len(stop_seqs2), 2) - self.assertEqual(len(stop_lens2), 2) - - def test_process_request_chat_template_kwargs(self): - """Test chat_template_kwargs application inside process_request_dict.""" - - proc = self._make_processor() - - request = { - "request_id": "test_0", - "messages": [{"role": "user", "content": "hello"}], - "temperature": 0.5, - "top_p": 0.5, - } - request = Request.from_dict(request) - - processed = proc.process_request_dict(request, max_model_len=20, chat_template_kwargs={"extra": "VALUE"}) - - self.assertEqual(processed.eos_token_ids, [proc.tokenizer.eos_token_id]) - - expected_ids = proc.tokenizer.convert_tokens_to_ids(proc.tokenizer.tokenize("hello")) - self.assertIsNotNone(processed.prompt_token_ids) - self.assertEqual(processed.prompt_token_ids, expected_ids) - - self.assertTrue(hasattr(processed.sampling_params, "max_tokens")) - self.assertEqual(processed.sampling_params.max_tokens, max(1, 20 - len(expected_ids))) - - def test_process_request_dict_chat_template_kwargs(self): - """Test chat_template_kwargs insertion in process_request_dict.""" - proc = self._make_processor() - - req = { - "request_id": "test_0", - "messages": [{"role": "user", "content": "hey"}], - "chat_template_kwargs": {"A": "B"}, - "temperature": 0.5, - "top_p": 0.5, - } - req = Request.from_dict(req) - req.chat_template_kwargs = {"A": "B"} - - result = proc.process_request_dict(req, max_model_len=30) - - self.assertTrue(hasattr(result, "prompt_token_ids")) - self.assertEqual(getattr(result, "A"), "B") - - def test_init_generation_config_exception(self): - """Test fallback behavior when GenerationConfig loading fails.""" - with patch(f"{MODULE_PATH}.GenerationConfig.from_pretrained", side_effect=Exception("fail")): - proc = self._make_processor() - self.assertIsNone(proc.generation_config) - - # def test_process_response_with_tool_parser(self): - # """Verify tool_call extraction in process_response.""" - # proc = self._make_processor(tool=True) - - # class RespObj: - # """Mock response carrying token_ids and index for testing.""" - - # def __init__(self): - # self.request_id = "reqx" - # self.outputs = MagicMock() - # self.outputs.token_ids = [9, proc.tokenizer.eos_token_id] - # self.outputs.index = 0 - - # resp = RespObj() - # result = proc.process_response(resp) - - # self.assertTrue(hasattr(result.outputs, "tool_calls")) - # self.assertEqual(result.outputs.tool_calls[0]["name"], "fake_tool") - - def test_process_response_obj_normal_with_tool(self): - """Verify tool_call extraction in normal (non-streaming) response mode.""" - proc = self._make_processor(tool=True) - - resp = { - "finished": True, - "request_id": "task-99", - "outputs": {"token_ids": [10, 11], "text": ""}, - } - resp = RequestOutput.from_dict(resp) - - result = proc.process_response_obj_normal(resp, enable_thinking=False, include_stop_str_in_output=False) - - self.assertTrue(hasattr(result.outputs, "tool_calls")) - self.assertEqual(result.outputs.tool_calls[0]["name"], "fake_tool") - - def test_process_request_greedy_sets_top_k(self): - """process_request with temperature=0 should set top_k=1 for greedy decoding.""" - proc = self._make_processor() - proc.messages2ids = MagicMock(return_value=[9]) - - request = Request.from_dict( - { - "request_id": "test_greedy", - "prompt": "hello", - "temperature": 0.0, - "top_p": 0.5, - } - ) - result = proc.process_request(request, max_model_len=10) - self.assertEqual(result.get("temperature"), 1) - self.assertEqual(result.get("top_k"), 1) - - def test_process_request(self): - """Test process_request method with various input types.""" - proc = self._make_processor() - - # Test with prompt string - request = Request.from_dict( - { - "request_id": "test_1", - "prompt": "hello", - "temperature": 0.5, - "top_p": 0.5, - } - ) - result = proc.process_request(request, max_model_len=10) - self.assertEqual(result.prompt_token_ids, [9]) - - # Test with prompt token ids - request = Request.from_dict( - { - "request_id": "test_2", - "prompt_token_ids": [1, 2, 3], - "temperature": 0.5, - "top_p": 0.5, - } - ) - result = proc.process_request(request, max_model_len=5) - self.assertEqual(result.prompt_token_ids, [1, 2, 3]) - - # Test with messages - mock the messages2ids method to avoid dict attribute error - proc.messages2ids = MagicMock(return_value=[9]) - request = Request.from_dict( - { - "request_id": "test_3", - "messages": [{"role": "user", "content": "hello"}], - "temperature": 0.5, - "top_p": 0.5, - } - ) - result = proc.process_request(request, max_model_len=10) - self.assertEqual(result.prompt_token_ids, [9]) - proc.messages2ids.assert_called_once() - - def test_process_response(self): - """Test process_response method with various scenarios.""" - # Test without reasoning parser to avoid model_status_dict dependency - proc = self._make_processor(reasoning=False) - - # Test basic response - response = RequestOutput.from_dict( - {"request_id": "test_4", "outputs": {"token_ids": [10, 11, proc.tokenizer.eos_token_id], "index": 0}} - ) - result = proc.process_response(response) - self.assertEqual(result.outputs.text, "10 11") - - # Test another response to ensure consistency - response = RequestOutput.from_dict( - {"request_id": "test_5", "outputs": {"token_ids": [20, 21, proc.tokenizer.eos_token_id], "index": 0}} - ) - result = proc.process_response(response) - self.assertEqual(result.outputs.text, "20 21") - - # Test response without eos_token at the end - response = RequestOutput.from_dict({"request_id": "test_6", "outputs": {"token_ids": [30, 31], "index": 0}}) - result = proc.process_response(response) - self.assertEqual(result.outputs.text, "30 31") - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/input/v1/test_ernie_processor.py b/tests/input/v1/test_ernie_processor.py deleted file mode 100644 index 437e4029a5d..00000000000 --- a/tests/input/v1/test_ernie_processor.py +++ /dev/null @@ -1,162 +0,0 @@ -""" -# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -import unittest -from unittest.mock import MagicMock, patch - -from fastdeploy.engine.request import Request, RequestOutput -from fastdeploy.input.v1.ernie4_5_processor import Ernie4_5Processor - - -class MockReasoningParser: - def get_model_status(self, prompt_token_ids): - return "think_start" - - -class TestErnie4_5ProcessorProcessResponseDictStreaming(unittest.TestCase): - def setUp(self): - # 创建 Ernie4_5Processor 实例的模拟对象 - with patch.object(Ernie4_5Processor, "__init__", return_value=None) as mock_init: - self.processor = Ernie4_5Processor("model_path") - mock_init.side_effect = lambda *args, **kwargs: print(f"__init__ called with {args}, {kwargs}") - - # 设置必要的属性 - self.processor.tokenizer = MagicMock() - self.processor.tokenizer.eos_token_id = 1 - self.processor.decode_status = {"test": []} - self.processor.reasoning_end_dict = {} - self.processor.tool_parser_dict = {} - self.processor.generation_config = MagicMock() - self.processor.eos_token_ids = [1] - self.processor.reasoning_parser = MockReasoningParser() - self.processor.model_status_dict = {"request-id_0": "think_start", "test": "think_start"} - - # 模拟 ids2tokens 方法 - def mock_ids2tokens(token_ids, task_id): - self.processor.decode_status[task_id] = "mock_decode_status" - return "delta_text", [2, 3], "previous_texts" - - self.processor.ids2tokens = mock_ids2tokens - - def mock_messages2ids(request, **kwargs): - if "chat_template" in kwargs: - return [1] - else: - return [0] - - def mock_apply_default_parameters(request): - return request - - self.processor.messages2ids = mock_messages2ids - self.processor._apply_default_parameters = mock_apply_default_parameters - - # 模拟推理解析器 - self.mock_reasoning_parser = MagicMock() - self.mock_reasoning_parser.__class__.__name__ = "ErnieX1ReasoningParser" - # self.mock_reasoning_parser.extract_reasoning_content_streaming.return_value = ("reasoning", "text") - self.processor.reasoning_parser = self.mock_reasoning_parser - - # 模拟工具解析器 - self.mock_tool_parser = MagicMock() - self.mock_tool_parser.extract_tool_calls_streaming.return_value = None - self.mock_tool_parser_obj = MagicMock() - self.mock_tool_parser_obj.return_value = self.mock_tool_parser - self.processor.tool_parser_obj = self.mock_tool_parser_obj - - def test_process_response_obj_streaming_normal_case(self): - """测试正常情况下的流式响应处理""" - # 准备输入 - response_dict = {"finished": False, "request_id": "test", "outputs": {"token_ids": [4, 5]}} - kwargs = {"enable_thinking": True} - response = RequestOutput.from_dict(response_dict) - - # 调用方法 - result = self.processor.process_response_obj_streaming(response, **kwargs) - - # 验证结果 - self.assertEqual(result.outputs.completion_tokens, "delta_text") - - def test_process_request_dict(self): - request_dict = { - "request_id": "123", - "messages": [{"role": "user", "content": "Hello!"}], - "chat_template_kwargs": {"chat_template": "Hello!"}, - "eos_token_ids": [1], - "temperature": 1, - "top_p": 1, - } - request = Request.from_dict(request_dict) - request.chat_template_kwargs = {"chat_template": "Hello!"} - result = self.processor.process_request_dict(request, 100) - self.assertEqual(result.prompt_token_ids, [1]) - - def test_process_response_obj_normal(self): - mock_tokens = ["reasoning", "token", "list"] - self.processor.tokenizer.tokenize = MagicMock(return_value=mock_tokens) - self.processor.reasoning_parser.extract_reasoning_content = MagicMock( - return_value=("Mock reasoning content", "Mock final text") - ) - - self.processor.tool_parser_obj = None - - response_dict = { - "request_id": "request-id_0", - "outputs": {"token_ids": [2, 3, 4, 5, 1], "text": "Initial text", "top_logprobs": []}, - # "finish_reason": "stop", - "finished": True, - } - response = RequestOutput.from_dict(response_dict) - kwargs = {"enable_thinking": True} - - with patch("fastdeploy.input.ernie4_5_processor.data_processor_logger"): - result = self.processor.process_response_obj_normal(response, **kwargs) - - self.mock_reasoning_parser.extract_reasoning_content.assert_called_once() - self.assertEqual(result.outputs.reasoning_content, "Mock reasoning content") - self.assertEqual(result.outputs.reasoning_token_num, len(mock_tokens)) - self.assertEqual(result.outputs.text, "Mock final text") - self.assertTrue(hasattr(result.outputs, "completion_tokens")) - - def test_think_status(self): - """测试 思考机制""" - request = { - "prompt": "hello", - "request_id": "test_1", - "prompt_token_ids": [1, 2, 3], - "temperature": 0.7, - "top_p": 0.9, - } - request = Request.from_dict(request) - self.processor.reasoning_parser = MagicMock() - self.processor.reasoning_parser.get_model_status.return_value = "think_start" - self.processor.model_status_dict = {} - self.processor.process_request_dict(request, max_model_len=512) - self.assertEqual(request.enable_thinking, True) - - request = { - "prompt": "hello", - "request_id": "test", - "prompt_token_ids": [1, 2, 3], - "temperature": 0.7, - "top_p": 0.9, - } - request = Request.from_dict(request) - self.processor.process_request_dict(request, max_model_len=512) - self.assertEqual(request.enable_thinking, True) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/input/v1/test_ernie_vl_processor.py b/tests/input/v1/test_ernie_vl_processor.py deleted file mode 100644 index 132b423f3cb..00000000000 --- a/tests/input/v1/test_ernie_vl_processor.py +++ /dev/null @@ -1,1460 +0,0 @@ -""" -# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -import unittest -from types import SimpleNamespace -from unittest.mock import MagicMock, patch - -import numpy as np -from PIL import Image - -from fastdeploy.engine.request import CompletionOutput, Request, RequestOutput -from fastdeploy.engine.sampling_params import SamplingParams -from fastdeploy.input.ernie4_5_tokenizer import Ernie4_5Tokenizer -from fastdeploy.input.utils import IDS_TYPE_FLAG -from fastdeploy.input.v1.ernie4_5_vl_processor import Ernie4_5_VLProcessor -from fastdeploy.input.v1.ernie4_5_vl_processor.image_preprocessor.image_preprocessor_adaptive import ( - AdaptiveImageProcessor, -) -from fastdeploy.input.v1.ernie4_5_vl_processor.process import DataProcessor - - -class MockReasoningParser: - def get_model_status(self, prompt_token_ids): - return "think_start" - - -class TestErnie4_5VLProcessorProcessResponseDictStreaming(unittest.TestCase): - def setUp(self): - # Create mock object for Ernie4_5Processor instance - with patch.object(Ernie4_5_VLProcessor, "__init__", return_value=None) as mock_init: - self.processor = Ernie4_5_VLProcessor("model_path") - mock_init.side_effect = lambda *args, **kwargs: print(f"__init__ called with {args}, {kwargs}") - - # Set necessary attributes - self.processor.tokenizer = MagicMock() - self.processor.tokenizer.eos_token_id = 1 - self.processor.decode_status = {"test": []} - self.processor.reasoning_end_dict = {} - self.processor.tool_parser_dict = {} - self.processor.generation_config = MagicMock() - self.processor.eos_token_ids = [1] - self.processor.reasoning_parser = MockReasoningParser() - self.processor.model_status_dict = {"test": "think_start"} - self.processor.ernie4_5_processor = MagicMock() - - # Mock ids2tokens method - def mock_ids2tokens(token_ids, task_id): - return "delta_text", [2, 3], "previous_texts" - - self.processor.ids2tokens = mock_ids2tokens - - def mock_request2ids(request, **kwargs): - return {"input_ids": np.array([1, 2, 3]), "prompt_token_ids": [0]} - - def mock_check_mm_limits(item): - pass - - def mock_apply_default_parameters(request): - return request - - def mock_pack_outputs(outputs): - # Ensure input_ids is numpy array if it exists - result = outputs.copy() if isinstance(outputs, dict) else outputs - if isinstance(result, dict): - if "input_ids" in result and isinstance(result["input_ids"], list): - result["input_ids"] = np.array(result["input_ids"]) - if "token_type_ids" in result and isinstance(result["token_type_ids"], list): - result["token_type_ids"] = np.array(result["token_type_ids"]) - if "position_ids" in result and isinstance(result["position_ids"], list): - result["position_ids"] = np.array(result["position_ids"]) - return result - - def mock_prompt_token_ids2outputs(request): - return { - "input_ids": np.array([1, 1, 1]), - "token_type_ids": np.array([0, 0, 0]), - "position_ids": np.array([[0, 0, 0], [1, 1, 1], [2, 2, 2]]), - "images": [], - "grid_thw": [], - "image_type_ids": [], - "cur_position": 3, - } - - self.processor._apply_default_parameters = mock_apply_default_parameters - self.processor._check_mm_limits = mock_check_mm_limits - self.processor.ernie4_5_processor.request2ids = mock_request2ids - self.processor.ernie4_5_processor.prompt_token_ids2outputs = mock_prompt_token_ids2outputs - self.processor.pack_outputs = mock_pack_outputs - - # Mock reasoning parser - self.mock_reasoning_parser = MagicMock() - self.mock_reasoning_parser.extract_reasoning_content_streaming.return_value = None - self.processor.reasoning_parser = self.mock_reasoning_parser - - # Mock tool parser - self.mock_tool_parser = MagicMock() - self.mock_tool_parser.extract_tool_calls_streaming.return_value = None - self.mock_tool_parser_obj = MagicMock() - self.mock_tool_parser_obj.return_value = self.mock_tool_parser - self.processor.tool_parser_obj = self.mock_tool_parser_obj - - def test_think_status(self): - """测试 思考机制""" - request = { - "prompt": "hello", - "request_id": "test_1", - "prompt_token_ids": [1, 2, 3], - } - request = Request.from_dict(request) - self.processor.reasoning_parser = MagicMock() - self.processor.reasoning_parser.get_model_status.return_value = "think_start" - self.processor.model_status_dict = {} - self.processor.process_request_dict(request, max_model_len=512) - self.assertEqual(request.enable_thinking, True) - - request = { - "prompt": "hello", - "request_id": "test", - "prompt_token_ids": [1, 2, 3], - } - request = Request.from_dict(request) - self.processor.process_request_dict(request, max_model_len=512) - self.assertEqual(request.enable_thinking, True) - - def test_init(self): - """Test __init__ method""" - with patch("fastdeploy.input.v1.ernie4_5_vl_processor.ernie4_5_vl_processor.data_processor_logger"): - mock_dp = MagicMock() - mock_dp.image_patch_id = 1001 - mock_dp.spatial_conv_size = 14 - mock_dp.tokenizer = MagicMock() - mock_dp.tokenizer.pad_token_id = 0 - mock_dp.eval = MagicMock() - - with patch( - "fastdeploy.input.v1.ernie4_5_vl_processor.ernie4_5_vl_processor.DataProcessor" - ) as mock_dp_class: - mock_dp_class.return_value = mock_dp - with patch( - "fastdeploy.input.v1.ernie4_5_vl_processor.ernie4_5_vl_processor.GenerationConfig" - ) as mock_gen_config: - mock_gen_config.from_pretrained.return_value = MagicMock() - with patch("paddleformers.cli.utils.llm_utils.get_eos_token_id") as mock_get_eos: - mock_get_eos.return_value = [1, 2] - - # Test normal initialization - mock_reasoning_parser_class = MagicMock() - processor = Ernie4_5_VLProcessor( - "model_path", - limit_mm_per_prompt={"image": 2, "video": 1}, - mm_processor_kwargs={"spatial_conv_size": 14}, - reasoning_parser_obj=lambda tokenizer: mock_reasoning_parser_class, - tool_parser_obj=MagicMock(), - enable_processor_cache=True, - ) - - self.assertEqual(processor.image_patch_id, 1001) - self.assertEqual(processor.spatial_conv_size, 14) - self.assertIsNotNone(processor.tokenizer) - self.assertIsNotNone(processor.generation_config) - self.assertEqual(processor.eos_token_ids, [1, 2]) - self.assertEqual(processor.limit_mm_per_prompt["image"], 2) - self.assertEqual(processor.limit_mm_per_prompt["video"], 1) - mock_dp.eval.assert_called_once() - - # Test with generation config exception - mock_gen_config.from_pretrained.side_effect = Exception("Config not found") - processor2 = Ernie4_5_VLProcessor("model_path") - self.assertIsNone(processor2.generation_config) - - # Test with reasoning_parser_obj - mock_reasoning_parser = MagicMock() - processor3 = Ernie4_5_VLProcessor( - "model_path", reasoning_parser_obj=lambda tokenizer: mock_reasoning_parser - ) - self.assertIsNotNone(processor3.reasoning_parser) - - def test_parse_processor_kwargs(self): - """Test _parse_processor_kwargs with various inputs""" - with patch.object(Ernie4_5_VLProcessor, "__init__", return_value=None): - processor = Ernie4_5_VLProcessor("model_path") - processor._parse_processor_kwargs = Ernie4_5_VLProcessor._parse_processor_kwargs.__get__( - processor, Ernie4_5_VLProcessor - ) - - # Test with valid kwargs - valid_kwargs = { - "spatial_conv_size": 14, - "temporal_conv_size": 2, - "image_min_pixels": 1000, - "image_max_pixels": 10000, - } - result = processor._parse_processor_kwargs(valid_kwargs) - self.assertEqual(result, valid_kwargs) - - # Test with invalid type (implementation catches exception and returns empty dict) - invalid_kwargs = {"spatial_conv_size": "invalid"} # Should be int - result = Ernie4_5_VLProcessor._parse_processor_kwargs(processor, invalid_kwargs) - self.assertEqual(result, {}) - - # Test with non-dict input (implementation catches exception and returns empty dict) - result = Ernie4_5_VLProcessor._parse_processor_kwargs(processor, "not a dict") - self.assertEqual(result, {}) - - # Test exception handling with None - with patch("fastdeploy.input.v1.ernie4_5_vl_processor.ernie4_5_vl_processor.data_processor_logger"): - result = processor._parse_processor_kwargs(None) - self.assertEqual(result, {}) - - def test_parse_limits(self): - """Test _parse_limits with various inputs""" - with patch.object(Ernie4_5_VLProcessor, "__init__", return_value=None): - processor = Ernie4_5_VLProcessor("model_path") - processor._parse_limits = Ernie4_5_VLProcessor._parse_limits.__get__(processor, Ernie4_5_VLProcessor) - - # Test with valid limits - valid_limits = {"image": 5, "video": 3} - result = processor._parse_limits(valid_limits) - self.assertEqual(result["image"], 5) - self.assertEqual(result["video"], 3) - self.assertEqual(result["audio"], 1) # Default value - - # Test with empty input (None) - result = processor._parse_limits(None) - self.assertEqual(result["image"], 1) - self.assertEqual(result["video"], 1) - self.assertEqual(result["audio"], 1) - - # Test with invalid type (implementation catches exception and returns default limits) - result = Ernie4_5_VLProcessor._parse_limits(processor, "not a dict") - self.assertEqual(result["image"], 1) - self.assertEqual(result["video"], 1) - self.assertEqual(result["audio"], 1) - - def test_check_mm_limits(self): - """Test _check_mm_limits with various inputs""" - with patch.object(Ernie4_5_VLProcessor, "__init__", return_value=None): - processor = Ernie4_5_VLProcessor("model_path") - processor._check_mm_limits = Ernie4_5_VLProcessor._check_mm_limits.__get__(processor, Ernie4_5_VLProcessor) - - # Test with dict input (should not raise) - processor.limit_mm_per_prompt = {"image": 2, "video": 1} - mm_data = {"image": [1, 2], "video": [1]} - processor._check_mm_limits(mm_data) - - # Test with messages input (should not raise) - messages = [ - {"role": "user", "content": [{"type": "image", "data": "img1"}]}, - {"role": "user", "content": [{"type": "video", "data": "vid1"}]}, - ] - processor._check_mm_limits(messages) - - # Test when limit is exceeded (should raise ValueError) - processor.limit_mm_per_prompt = {"image": 1, "video": 1} - mm_data = {"image": [1, 2, 3], "video": []} # 3 images, limit is 1 - with self.assertRaises(ValueError) as context: - processor._check_mm_limits(mm_data) - self.assertIn("Too many image items", str(context.exception)) - - def test_process_request_dict(self): - """Test process_request_dict method""" - # from fastdeploy.engine.request import Request - - # Mock the process_request_dict method - self.processor.process_request_dict = MagicMock() - - # Create a mock Request object - mock_request = MagicMock(spec=Request) - mock_request.to_dict.return_value = {"messages": [{"role": "user", "content": "Hello"}]} - - # Mock Request.from_dict to return a mock request - with patch.object(Request, "from_dict") as mock_from_dict: - mock_result_request = MagicMock(spec=Request) - mock_from_dict.return_value = mock_result_request - - self.processor.process_request_dict(mock_request, max_model_len=100, chat_template_kwargs={"key": "value"}) - - # Verify process_request_dict was called - self.processor.process_request_dict.assert_called_once() - - def test_get_pad_id(self): - """Test get_pad_id method""" - with patch.object(Ernie4_5_VLProcessor, "__init__", return_value=None): - processor = Ernie4_5_VLProcessor("model_path") - processor.tokenizer = MagicMock() - processor.tokenizer.pad_token_id = 100 - processor.get_pad_id = Ernie4_5_VLProcessor.get_pad_id.__get__(processor, Ernie4_5_VLProcessor) - - result = processor.get_pad_id() - self.assertEqual(result, 100) - - def test_load_tokenizer(self): - """Test _load_tokenizer method""" - with patch.object(Ernie4_5_VLProcessor, "__init__", return_value=None): - processor = Ernie4_5_VLProcessor("model_path") - mock_tokenizer = MagicMock() - processor.ernie4_5_processor = MagicMock() - processor.ernie4_5_processor.tokenizer = mock_tokenizer - processor._load_tokenizer = Ernie4_5_VLProcessor._load_tokenizer.__get__(processor, Ernie4_5_VLProcessor) - - processor._load_tokenizer() - self.assertEqual(processor.tokenizer, mock_tokenizer) - - def test_append_completion_tokens(self): - """Test append_completion_tokens method""" - with patch.object(Ernie4_5_VLProcessor, "__init__", return_value=None): - processor = Ernie4_5_VLProcessor("model_path") - processor.append_completion_tokens = Ernie4_5_VLProcessor.append_completion_tokens.__get__( - processor, Ernie4_5_VLProcessor - ) - - multimodal_inputs = { - "input_ids": [1, 2, 3], - "token_type_ids": [0, 0, 0], - "position_ids": [[0, 0, 0], [1, 1, 1], [2, 2, 2]], - "cur_position": 3, - } - completion_token_ids = [10, 11, 12] - - processor.append_completion_tokens(multimodal_inputs, completion_token_ids) - - self.assertEqual(multimodal_inputs["input_ids"], [1, 2, 3, 10, 11, 12]) - self.assertEqual(multimodal_inputs["token_type_ids"], [0, 0, 0, 0, 0, 0]) - self.assertEqual(len(multimodal_inputs["position_ids"]), 6) - self.assertEqual(multimodal_inputs["cur_position"], 6) - - def test_pack_outputs(self): - """Test pack_outputs with and without images""" - with patch.object(Ernie4_5_VLProcessor, "__init__", return_value=None): - processor = Ernie4_5_VLProcessor("model_path") - processor.image_patch_id = 1001 - processor.ernie4_5_processor = SimpleNamespace(mm_num_tokens=lambda **kwargs: 123) - processor.pack_outputs = Ernie4_5_VLProcessor.pack_outputs.__get__(processor, Ernie4_5_VLProcessor) - # Test with images - outs_with_images = { - "input_ids": [1, 2, 3], - "token_type_ids": [0, 0, 0], - "position_ids": [[0, 0, 0], [1, 1, 1], [2, 2, 2]], - "images": [np.array([[1, 2], [3, 4]])], - "grid_thw": [np.array([[1, 2, 2]])], - "image_type_ids": [0], - } - - result = processor.pack_outputs(outs_with_images) - self.assertIsNotNone(result["images"]) - self.assertIsNotNone(result["grid_thw"]) - self.assertIsNotNone(result["image_type_ids"]) - self.assertEqual(result["image_patch_id"], 1001) - self.assertIsInstance(result["input_ids"], np.ndarray) - self.assertIsInstance(result["token_type_ids"], np.ndarray) - self.assertIsInstance(result["position_ids"], np.ndarray) - - # Test without images - outs_without_images = { - "input_ids": [1, 2, 3], - "token_type_ids": [0, 0, 0], - "position_ids": [[0, 0, 0], [1, 1, 1], [2, 2, 2]], - "images": [], - "grid_thw": [], - "image_type_ids": [], - } - - result = processor.pack_outputs(outs_without_images) - self.assertIsNone(result["images"]) - self.assertIsNone(result["grid_thw"]) - self.assertIsNone(result["image_type_ids"]) - - def test_process_response_dict(self): - """Test process_response_dict with different parameters""" - with patch.object(Ernie4_5_VLProcessor, "__init__", return_value=None): - processor = Ernie4_5_VLProcessor("model_path") - processor.process_response_dict = Ernie4_5_VLProcessor.process_response_dict.__get__( - processor, Ernie4_5_VLProcessor - ) - - response = RequestOutput( - request_id="test_0", - outputs=CompletionOutput(text="response", index=0, send_idx=0, token_ids=[1, 2, 3]), - ) - # Test with stream=True - processor.process_response_obj_streaming = MagicMock(return_value=response) - response_obj = RequestOutput(request_id="test_0") - result = processor.process_response_dict(response_obj, stream=True) - processor.process_response_obj_streaming.assert_called_once() - self.assertEqual(result, response) - - # Test with stream=False - processor.process_response_obj_normal = MagicMock(return_value=response) - response_obj = RequestOutput(request_id="test_0") - result = processor.process_response_dict(response_obj, stream=False) - processor.process_response_obj_normal.assert_called_once() - self.assertEqual(result, response) - - def test_apply_default_parameters(self): - """Test _apply_default_parameters with dict and object request""" - with patch.object(Ernie4_5_VLProcessor, "__init__", return_value=None): - processor = Ernie4_5_VLProcessor("model_path") - processor.generation_config = MagicMock() - processor.generation_config.top_p = 0.8 - processor.generation_config.temperature = 0.9 - processor._apply_default_parameters = Ernie4_5_VLProcessor._apply_default_parameters.__get__( - processor, Ernie4_5_VLProcessor - ) - - # Test with dict request - request = Request(request_id="test_0") - request.sampling_params = SamplingParams() - result = processor._apply_default_parameters(request) - self.assertEqual(result.sampling_params.top_p, 0.8) - self.assertEqual(result.sampling_params.temperature, 0.9) - - # Test with object request - class MockRequest: - def __init__(self): - self.sampling_params = SamplingParams() - self.sampling_params.top_p = None - self.sampling_params.temperature = None - - def get(self, key): - return getattr(self.sampling_params, key, None) - - def set(self, key, value): - setattr(self.sampling_params, key, value) - - request = MockRequest() - result = processor._apply_default_parameters(request) - self.assertEqual(result.sampling_params.top_p, 0.8) - - -class TestDataProcessorTargetMethods(unittest.TestCase): - def setUp(self): - self.mock_tokenizer = MagicMock(spec=Ernie4_5Tokenizer) - self.mock_tokenizer.ignored_index = -100 - self.mock_tokenizer.convert_tokens_to_ids.side_effect = self._mock_convert_tokens_to_ids - self.mock_tokenizer.chat_template = "mock_template" - self.mock_tokenizer.apply_chat_template.return_value = "User: Hello<|image@placeholder|>" - # Mock encode method for _add_text - self.mock_tokenizer.encode = MagicMock(return_value={"input_ids": [1, 2, 3]}) - - def mock_load_tokenizer(dp_instance): - dp_instance.tokenizer = self.mock_tokenizer - - with patch.object(DataProcessor, "_load_tokenizer", side_effect=mock_load_tokenizer, autospec=True): - with patch.object(AdaptiveImageProcessor, "from_pretrained") as mock_image_preprocessor: - mock_image_preprocessor.return_value = MagicMock() - self.data_processor = DataProcessor( - tokenizer_name="mock_tokenizer", - image_preprocessor_name="mock_image_preprocessor", - enable_processor_cache=False, - ) - self.data_processor.image_patch_id = 1001 - self.data_processor.image_start_id = 1002 - self.data_processor.image_end_id = 1003 - self.data_processor.video_start_id = 1004 - self.data_processor.video_end_id = 1005 - self.data_processor.role_prefixes = {"user": "User: ", "assistant": "Assistant: "} - self.data_processor.enable_processor_cache = False - # Note: extract_mm_items is not mocked by default, only when needed - self.data_processor.extract_mm_items = MagicMock(return_value=([], [], [], [], None, [], [])) - - def _restore_real_extract_mm_items(self): - """Helper method to restore real extract_mm_items method for testing""" - from fastdeploy.input.v1.ernie4_5_vl_processor.process import DataProcessor - - original_extract_mm_items = DataProcessor.extract_mm_items - self.data_processor.extract_mm_items = original_extract_mm_items.__get__(self.data_processor, DataProcessor) - - def _mock_convert_tokens_to_ids(self, token): - token_id_map = { - "<|begin_of_sentence|>": 101, - "<|end_of_sentence|>": 102, - "": 103, - "<|IMAGE_PLACEHOLDER|>": 1001, - "<|IMAGE_START|>": 1002, - "<|IMAGE_END|>": 1003, - "<|VIDEO_START|>": 1004, - "<|VIDEO_END|>": 1005, - } - return token_id_map.get(token, 999) - - def test_prompt_token_ids2outputs_only_prompt_token_ids(self): - test_prompt_token_ids = [101, 999, 998, 997, 102] - request = { - "request_id": "test_0", - "prompt_token_ids": test_prompt_token_ids, - } - request = Request.from_dict(request) - - outputs = self.data_processor.prompt_token_ids2outputs(request) - - prompt_len = len(test_prompt_token_ids) - - self.assertEqual( - outputs["input_ids"], - test_prompt_token_ids, - f"input_ids mismatch: actual {outputs['input_ids']}, expected {test_prompt_token_ids}", - ) - - self.assertEqual(outputs["token_type_ids"], [IDS_TYPE_FLAG["text"]] * prompt_len) - - expected_position_ids = [[i] * 3 for i in range(prompt_len)] - self.assertEqual(outputs["position_ids"], expected_position_ids) - - self.assertEqual(outputs["cur_position"], prompt_len) - - self.assertEqual(len(outputs["images"]), 0) - self.assertEqual(len(outputs["grid_thw"]), 0) - self.assertEqual(len(outputs["mm_positions"]), 0) - self.assertEqual(len(outputs["mm_hashes"]), 0) - self.assertEqual(outputs["video_cnt"], 0) - self.assertEqual(outputs["num_input_image_tokens"], 0) - self.assertEqual(outputs["num_input_video_tokens"], 0) - - def test_prompt_token_ids2outputs_with_messages_no_mm(self): - test_prompt_token_ids = [101, 999, 998, 997, 102] - request = { - "request_id": "test_0", - "prompt_token_ids": test_prompt_token_ids, - "messages": [{"role": "user", "content": "Hello World"}], - } - request = Request.from_dict(request) - - self.data_processor.extract_mm_items.return_value = ([], [], [], [], None, [], []) - - outputs = self.data_processor.prompt_token_ids2outputs(request) - - prompt_len = len(test_prompt_token_ids) - - self.assertEqual(outputs["input_ids"], test_prompt_token_ids) - - self.assertEqual(outputs["token_type_ids"], [IDS_TYPE_FLAG["text"]] * prompt_len) - - expected_position_ids = [[i] * 3 for i in range(prompt_len)] - self.assertEqual(outputs["position_ids"], expected_position_ids) - - self.assertEqual(outputs["cur_position"], prompt_len) - - self.assertEqual(len(outputs["images"]), 0) - self.assertEqual(outputs["video_cnt"], 0) - self.assertEqual(outputs["num_input_image_tokens"], 0) - - def test_prompt_token_ids2outputs_add_image(self): - test_prompt_token_ids = [101, 1002, 1001, 1001, 1003, 102] - mock_img = MagicMock() - mock_img.height = 224 - mock_img.width = 224 - mock_img.convert.return_value = mock_img - request = { - "request_id": "test_0", - "prompt_token_ids": test_prompt_token_ids, - "messages": [ - {"role": "user", "content": [{"type": "image_url", "image_url": mock_img, "uuid": "img_uuid"}]} - ], - } - request = Request.from_dict(request) - self.data_processor.extract_mm_items.return_value = ( - [mock_img], - [], - ["img_uuid"], - [], - None, - [], - [{"type": "image", "data": mock_img}], - ) - mock_resize = (None, (2, 4)) - self.data_processor.image_preprocessor.get_smarted_resize.return_value = mock_resize - mock_preprocess = {"pixel_values": np.random.randn(1, 16, 16, 3), "image_grid_thw": np.array([[2, 4]])} - self.data_processor.image_preprocessor.preprocess.return_value = mock_preprocess - # self.data_processor._compute_3d_positions = MagicMock(return_value=[[i]*3 for i in range(4)]) - outputs = self.data_processor.prompt_token_ids2outputs(request) - self.assertEqual(outputs["input_ids"], [101, 1002, 1001, 1001, 1003, 102]) - self.assertEqual( - outputs["token_type_ids"], - [ - IDS_TYPE_FLAG["text"], - IDS_TYPE_FLAG["text"], - IDS_TYPE_FLAG["image"], - IDS_TYPE_FLAG["image"], - IDS_TYPE_FLAG["text"], - IDS_TYPE_FLAG["text"], - ], - ) - self.assertEqual(len(outputs["position_ids"]), 6) - self.assertEqual(outputs["cur_position"], 6) - self.assertEqual(len(outputs["images"]), 1) - self.assertIsNotNone(outputs["images"][0]) - self.assertEqual(outputs["num_input_image_tokens"], 2) - self.assertEqual(len(outputs["mm_positions"]), 1) - self.assertEqual(len(outputs["mm_hashes"]), 1) - self.assertEqual(len(outputs["grid_thw"]), 1) - self.assertEqual(len(outputs["image_type_ids"]), 1) - - def test_prompt_token_ids2outputs_add_processed_image(self): - test_prompt_token_ids = [101, 1002, 1001, 1001, 1003, 102] - mock_img_data = np.random.randn(8, 28, 28) - mock_img_cache = (mock_img_data, {"thw": (1, 8, 8)}) - request = { - "request_id": "test_0", - "prompt_token_ids": test_prompt_token_ids, - "messages": [ - {"role": "user", "content": [{"type": "image_url", "image_url": mock_img_cache, "uuid": "img_uuid"}]} - ], - } - request = Request.from_dict(request) - self.data_processor.extract_mm_items.return_value = ( - [mock_img_cache], - [], - ["img_uuid"], - [], - None, - [], - [{"type": "image", "data": mock_img_cache}], - ) - outputs = self.data_processor.prompt_token_ids2outputs(request) - self.assertEqual(outputs["input_ids"], [101, 1002, 1001, 1001, 1003, 102]) - self.assertEqual( - outputs["token_type_ids"], - [ - IDS_TYPE_FLAG["text"], - IDS_TYPE_FLAG["text"], - IDS_TYPE_FLAG["image"], - IDS_TYPE_FLAG["image"], - IDS_TYPE_FLAG["text"], - IDS_TYPE_FLAG["text"], - ], - ) - self.assertEqual(len(outputs["position_ids"]), 20) - self.assertEqual(outputs["cur_position"], 8) - self.assertEqual(len(outputs["images"]), 1) - self.assertIsNotNone(outputs["images"][0]) - self.assertEqual(len(outputs["mm_positions"]), 1) - self.assertEqual(outputs["mm_hashes"][0], "img_uuid") - self.assertEqual(len(outputs["grid_thw"]), 1) - self.assertEqual(len(outputs["image_type_ids"]), 1) - - def test_prompt_token_ids2outputs_add_video(self): - test_prompt_token_ids = [101, 1004, 1001, 1001, 1001, 1001, 1005, 102] - mock_frame1 = MagicMock() - mock_frame1.height = 224 - mock_frame1.width = 224 - mock_frame1.convert.return_value = mock_frame1 - mock_frame2 = MagicMock() - mock_frame2.height = 224 - mock_frame2.width = 224 - mock_frame2.convert.return_value = mock_frame2 - frames = [mock_frame1, mock_frame2] - request = { - "request_id": "test_0", - "prompt_token_ids": test_prompt_token_ids, - "messages": [ - {"role": "user", "content": [{"type": "video_url", "video_url": frames, "uuid": "vid_uuid"}]} - ], - } - request = Request.from_dict(request) - self.data_processor.extract_mm_items.return_value = ( - [], - [frames], - [], - ["vid_uuid"], - None, - [], - [{"type": "video", "data": frames}], - ) - self.data_processor._load_and_process_video = MagicMock(return_value=frames) - patches_h, patches_w = 4, 4 - self.data_processor.image_preprocessor.get_smarted_resize.return_value = (None, (patches_h, patches_w)) - mock_preprocess = { - "pixel_values_videos": np.random.randn(2, patches_h, patches_w, 3), - "video_grid_thw": np.array([[patches_h, patches_w]] * 2), - } - self.data_processor.image_preprocessor.preprocess.return_value = mock_preprocess - outputs = self.data_processor.prompt_token_ids2outputs(request) - self.assertEqual(outputs["input_ids"], [101, 1004, 1001, 1001, 1001, 1001, 1005, 102]) - self.assertEqual( - outputs["token_type_ids"], - [ - IDS_TYPE_FLAG["text"], - IDS_TYPE_FLAG["text"], - IDS_TYPE_FLAG["video"], - IDS_TYPE_FLAG["video"], - IDS_TYPE_FLAG["video"], - IDS_TYPE_FLAG["video"], - IDS_TYPE_FLAG["text"], - IDS_TYPE_FLAG["text"], - ], - ) - self.assertEqual(len(outputs["position_ids"]), 8) - self.assertEqual(outputs["cur_position"], 6) - self.assertEqual(len(outputs["images"]), 1) - self.assertIsNotNone(outputs["images"][0]) - self.assertEqual(len(outputs["mm_positions"]), 1) - self.assertEqual(outputs["mm_hashes"][0], "vid_uuid") - self.assertEqual(len(outputs["grid_thw"]), 1) - self.assertEqual(len(outputs["image_type_ids"]), 2) - self.assertEqual(outputs["num_input_video_tokens"], 4) - - def test_prompt_token_ids2outputs_add_processed_video(self): - test_prompt_token_ids = [101, 1004, 1001, 1001, 1001, 1001, 1005, 102] - t, h, w = 2, 4, 4 - spatial_conv_size = self.data_processor.spatial_conv_size - temporal_conv_size = self.data_processor.temporal_conv_size - token_per_frame = (h // spatial_conv_size) * (w // spatial_conv_size) - num_tokens = (t // temporal_conv_size) * token_per_frame - mock_frames_data = np.random.randn(num_tokens * spatial_conv_size**2 * temporal_conv_size, 28, 28) - mock_frames_cache = (mock_frames_data, {"thw": (t, h, w)}) - request = { - "request_id": "test_0", - "prompt_token_ids": test_prompt_token_ids, - "messages": [ - {"role": "user", "content": [{"type": "video", "data": mock_frames_cache, "uuid": "vid_uuid"}]} - ], - } - request = Request.from_dict(request) - self.data_processor.extract_mm_items.return_value = ( - [], - [mock_frames_cache], - [], - ["vid_uuid"], - None, - [], - [{"type": "video", "data": mock_frames_cache}], - ) - outputs = self.data_processor.prompt_token_ids2outputs(request) - self.assertEqual(outputs["input_ids"], [101, 1004, 1001, 1001, 1001, 1001, 1005, 102]) - self.assertEqual( - outputs["token_type_ids"], - [ - IDS_TYPE_FLAG["text"], - IDS_TYPE_FLAG["text"], - IDS_TYPE_FLAG["video"], - IDS_TYPE_FLAG["video"], - IDS_TYPE_FLAG["video"], - IDS_TYPE_FLAG["video"], - IDS_TYPE_FLAG["text"], - IDS_TYPE_FLAG["text"], - ], - ) - self.assertEqual(len(outputs["position_ids"]), 8) - self.assertEqual(outputs["cur_position"], 6) - self.assertEqual(len(outputs["images"]), 1) - self.assertIsNotNone(outputs["images"][0]) - self.assertEqual(len(outputs["mm_positions"]), 1) - self.assertEqual(outputs["mm_hashes"][0], "vid_uuid") - self.assertEqual(len(outputs["grid_thw"]), 1) - self.assertEqual(len(outputs["image_type_ids"]), 2) - - def test_prompt_token_ids2outputs_add_image_token_len_mismatch(self): - test_prompt_token_ids = [101, 1002, 1001, 1001, 1001, 1003, 102] - mock_img = MagicMock() - mock_img.height = 224 - mock_img.width = 224 - mock_img.convert.return_value = mock_img - request = { - "request_id": "test_0", - "prompt_token_ids": test_prompt_token_ids, - "messages": [ - {"role": "user", "content": [{"type": "image_url", "image_url": mock_img, "uuid": "img_uuid"}]} - ], - } - request = Request.from_dict(request) - self.data_processor.extract_mm_items.return_value = ( - [mock_img], - [], - ["img_uuid"], - [], - None, - [], - [{"type": "image", "data": mock_img}], - ) - patches_h, patches_w = 8, 8 - self.data_processor.image_preprocessor.get_smarted_resize.return_value = (None, (patches_h, patches_w)) - mock_preprocess = { - "pixel_values": np.random.randn(1, patches_h, patches_w, 3), - "image_grid_thw": np.array([[patches_h, patches_w]]), - } - self.data_processor.image_preprocessor.preprocess.return_value = mock_preprocess - with self.assertRaises(ValueError) as ctx: - self.data_processor.prompt_token_ids2outputs(request) - self.assertIn("image tokens num not match the size", str(ctx.exception)) - - def test_prompt_token_ids2outputs_add_processed_image_token_len_mismatch(self): - test_prompt_token_ids = [101, 1002, 1001, 1001, 1003, 102] - spatial_conv_size = self.data_processor.spatial_conv_size - num_tokens = 4 - mock_img_data = np.random.randn(num_tokens * (spatial_conv_size**2), 28, 28) - mock_img_cache = (mock_img_data, {"thw": (1, 8, 8)}) - request = { - "request_id": "test_0", - "prompt_token_ids": test_prompt_token_ids, - "messages": [ - {"role": "user", "content": [{"type": "image_url", "image_url": mock_img_cache, "uuid": "img_uuid"}]} - ], - } - request = Request.from_dict(request) - self.data_processor.extract_mm_items.return_value = ( - [mock_img_cache], - [], - ["img_uuid"], - [], - None, - [], - [{"type": "image", "data": mock_img_cache}], - ) - with self.assertRaises(ValueError) as ctx: - self.data_processor.prompt_token_ids2outputs(request) - self.assertIn("image tokens num not match the size", str(ctx.exception)) - - def test_prompt_token_ids2outputs_add_video_token_len_mismatch(self): - test_prompt_token_ids = [101, 1004, 1001, 1001, 1005, 102] - mock_frame1 = MagicMock() - mock_frame1.height = 224 - mock_frame1.width = 224 - mock_frame1.convert.return_value = mock_frame1 - mock_frame2 = MagicMock() - mock_frame2.height = 224 - mock_frame2.width = 224 - mock_frame2.convert.return_value = mock_frame2 - frames = [mock_frame1, mock_frame2] - request = { - "request_id": "test_0", - "prompt_token_ids": test_prompt_token_ids, - "messages": [ - {"role": "user", "content": [{"type": "video_url", "video_url": frames, "uuid": "vid_uuid"}]} - ], - } - request = Request.from_dict(request) - self.data_processor.extract_mm_items.return_value = ( - [], - [frames], - [], - ["vid_uuid"], - None, - [], - [{"type": "video", "data": frames}], - ) - self.data_processor._load_and_process_video = MagicMock(return_value=frames) - patches_h, patches_w = 8, 8 - self.data_processor.image_preprocessor.get_smarted_resize.return_value = (None, (patches_h, patches_w)) - mock_preprocess = { - "pixel_values_videos": np.random.randn(2, patches_h, patches_w, 3), - "video_grid_thw": np.array([[patches_h, patches_w]] * 2), - } - self.data_processor.image_preprocessor.preprocess.return_value = mock_preprocess - with self.assertRaises(ValueError) as ctx: - self.data_processor.prompt_token_ids2outputs(request) - self.assertIn("video tokens num not match the size", str(ctx.exception)) - - def test_prompt_token_ids2outputs_add_processed_video_token_len_mismatch(self): - test_prompt_token_ids = [101, 1004, 1001, 1005, 102] - t, h, w = 2, 8, 8 - spatial_conv_size = self.data_processor.spatial_conv_size - temporal_conv_size = self.data_processor.temporal_conv_size - - num_tokens = 4 - mock_frames_data = np.random.randn(num_tokens * spatial_conv_size**2 * temporal_conv_size, 28, 28) - mock_frames_cache = (mock_frames_data, {"thw": (t, h, w)}) - request = { - "request_id": "test_0", - "prompt_token_ids": test_prompt_token_ids, - "messages": [ - {"role": "user", "content": [{"type": "video", "data": mock_frames_cache, "uuid": "vid_uuid"}]} - ], - } - request = Request.from_dict(request) - self.data_processor.extract_mm_items.return_value = ( - [], - [mock_frames_cache], - [], - ["vid_uuid"], - None, - [], - [{"type": "video", "data": mock_frames_cache}], - ) - with self.assertRaises(ValueError) as ctx: - self.data_processor.prompt_token_ids2outputs(request) - self.assertIn("video tokens num not match the size", str(ctx.exception)) - - def test_extract_mm_items(self): - """Test extract_mm_items with various scenarios: basic items, video, and missing data error""" - self._restore_real_extract_mm_items() - - # Test basic multimodal items (image + video) - request = { - "request_id": "test_0", - "messages": [ - { - "role": "user", - "content": [ - {"type": "text", "text": "Hello"}, - {"type": "image", "data": Image.new("RGB", (224, 224)), "uuid": "img1"}, - {"type": "video", "data": [Image.new("RGB", (224, 224))], "uuid": "vid1"}, - ], - } - ], - } - request = Request.from_dict(request) - with patch("fastdeploy.input.v1.ernie4_5_vl_processor.process.parse_chat_messages") as mock_parse: - mock_parse.return_value = request.messages - images, videos, image_uuid, video_uuid, dealer, missing_idx, mm_items = ( - self.data_processor.extract_mm_items(request) - ) - self.assertEqual(len(images), 1) - self.assertEqual(len(videos), 1) - self.assertEqual(image_uuid[0], "img1") - self.assertEqual(video_uuid[0], "vid1") - self.assertEqual(len(mm_items), 2) - - # Test missing data error when cache is disabled - self.data_processor.enable_processor_cache = False - request = { - "request_id": "test_0", - "messages": [{"role": "user", "content": [{"type": "image", "uuid": "img1"}]}], - } - request = Request.from_dict(request) - with patch("fastdeploy.input.v1.ernie4_5_vl_processor.process.parse_chat_messages") as mock_parse: - mock_parse.return_value = request.messages - with self.assertRaises(ValueError) as ctx: - self.data_processor.extract_mm_items(request) - self.assertIn("Missing items cannot be retrieved", str(ctx.exception)) - - -class TestDataProcessor(unittest.TestCase): - def setUp(self): - """Set up test environment""" - self.mock_tokenizer = MagicMock() - - def mock_convert_tokens_to_ids(x): - if isinstance(x, list): - return [hash(str(token)) % 10000 for token in x] - return hash(str(x)) % 10000 - - self.mock_tokenizer.convert_tokens_to_ids = MagicMock(side_effect=mock_convert_tokens_to_ids) - self.mock_tokenizer.encode = MagicMock(return_value={"input_ids": [1, 2, 3]}) - self.mock_tokenizer.decode = MagicMock(return_value="decoded_text") - self.mock_tokenizer.tokenize = MagicMock(return_value=["token1", "token2"]) - self.mock_tokenizer.ignored_index = -100 - self.mock_tokenizer.chat_template = MagicMock() - self.mock_tokenizer.apply_chat_template = MagicMock(return_value="formatted_prompt") - - self.mock_image_preprocessor = MagicMock() - self.mock_image_preprocessor.get_smarted_resize = MagicMock(return_value=((224, 224), (16, 16))) - self.mock_image_preprocessor.preprocess = MagicMock( - return_value={ - "pixel_values": np.random.rand(256, 3 * 14 * 14).astype(np.float32), - "image_grid_thw": np.array([[1, 16, 16]]), - } - ) - self.mock_image_preprocessor.from_pretrained = MagicMock(return_value=self.mock_image_preprocessor) - - with patch( - "fastdeploy.input.v1.ernie4_5_vl_processor.process.AdaptiveImageProcessor", - self.mock_image_preprocessor, - ): - with patch("fastdeploy.input.v1.ernie4_5_vl_processor.process.Ernie4_5Tokenizer") as mock_tokenizer_class: - mock_tokenizer_class.from_pretrained = MagicMock(return_value=self.mock_tokenizer) - mock_tokenizer_class.resource_files_names = {"vocab_file": "tokenizer.model"} - with patch("os.path.exists", return_value=True): - self.processor = DataProcessor( - tokenizer_name="test_model", - image_preprocessor_name="test_model", - ) - - def _create_outputs(self): - """Helper to create outputs dict""" - return { - "input_ids": [], - "token_type_ids": [], - "position_ids": [], - "images": [], - "grid_thw": [], - "image_type_ids": [], - "mm_positions": [], - "mm_hashes": [], - "cur_position": 0, - "num_input_image_tokens": 0, - "num_input_video_tokens": 0, - } - - def _mock_video_processing(self, mock_frames=None): - """Helper to mock video processing""" - if mock_frames is None: - mock_frames = [Image.new("RGB", (224, 224)) for _ in range(4)] - mock_read = patch("fastdeploy.input.v1.ernie4_5_vl_processor.process.read_video_decord") - mock_frames_read = patch("fastdeploy.input.v1.ernie4_5_vl_processor.process.read_frames_decord") - mock_render = patch("fastdeploy.input.v1.ernie4_5_vl_processor.process.render_frame_timestamp") - return mock_read, mock_frames_read, mock_render, mock_frames - - def _setup_video_mocks(self, mock_read, mock_frames_read, mock_render, mock_frames): - """Setup video processing mocks""" - mock_read.return_value = (None, {"duration": 2.0}, "test_path") - mock_frames_read.return_value = ( - [np.array(f) for f in mock_frames], - None, - [0.0, 0.5, 1.0, 1.5] if len(mock_frames) == 4 else [float(i) * 0.5 for i in range(len(mock_frames))], - ) - mock_render.side_effect = lambda img, ts: (Image.fromarray(img) if isinstance(img, np.ndarray) else img) - self.mock_image_preprocessor.preprocess.return_value = { - "pixel_values_videos": np.random.rand(len(mock_frames), 256, 3 * 14 * 14).astype(np.float32), - "video_grid_thw": np.array([[len(mock_frames), 16, 16]]), - } - - def test_train_and_eval(self): - """Test training and evaluation mode switching""" - self.assertTrue(self.processor.is_training) - self.processor.eval() - self.assertFalse(self.processor.is_training) - self.processor.train() - self.assertTrue(self.processor.is_training) - - def test_build_token_type_mapping(self): - """Test token type mapping construction""" - mapping = self.processor._build_token_type_mapping() - for token in [ - self.processor.IMG_START, - self.processor.IMG_END, - self.processor.VID_START, - self.processor.VID_END, - ]: - self.assertEqual(mapping[token], IDS_TYPE_FLAG["image"]) - self.assertEqual(mapping[self.processor.image_patch_id], IDS_TYPE_FLAG["image"]) - - def test_add_text_and_special_token(self): - """Test adding text and special tokens""" - outputs = self._create_outputs() - self.processor._add_text("hello", outputs) - self.assertEqual(len(outputs["input_ids"]), 3) - self.assertEqual(outputs["cur_position"], 3) - - outputs2 = self._create_outputs() - self.processor._add_text([1, 2, 3, 4, 5], outputs2) - self.assertEqual(len(outputs2["input_ids"]), 5) - - outputs3 = self._create_outputs() - self.processor._add_special_token("<|begin_of_sentence|>", outputs3) - self.processor._add_special_token(12345, outputs3) - self.assertEqual(len(outputs3["input_ids"]), 2) - - def test_compute_3d_positions(self): - """Test 3D position computation""" - pos_ids = self.processor._compute_3d_positions(t=2, h=16, w=16, start_idx=10) - self.assertIsInstance(pos_ids, list) - self.assertGreater(len(pos_ids), 0) - self.assertEqual(len(pos_ids[0]), 3) - - pos_ids2 = self.processor._compute_3d_positions(t=1, h=16, w=16, start_idx=0) - expected_len = 1 * (16 // self.processor.spatial_conv_size) ** 2 - self.assertEqual(len(pos_ids2), expected_len) - - def test_set_video_frame_args_comprehensive(self): - """Test _set_video_frame_args with various scenarios""" - # Valid cases - result = self.processor._set_video_frame_args( - { - "target_frames": 32, - "fps": -1, - "min_frames": 16, - "max_frames": 64, - "frames_sample": "leading", - }, - {"duration": 10.0}, - ) - self.assertEqual(result["target_frames"], 32) - - result = self.processor._set_video_frame_args( - { - "target_frames": -1, - "fps": 2, - "min_frames": 16, - "max_frames": 64, - "frames_sample": "leading", - }, - {"duration": 10.0}, - ) - self.assertIsNotNone(result) - - # Error cases - with self.assertRaises(ValueError): - self.processor._set_video_frame_args( - { - "target_frames": -1, - "fps": -1, - "min_frames": 16, - "max_frames": 64, - "frames_sample": "leading", - }, - {"duration": 10.0}, - ) - with self.assertRaises(ValueError): - self.processor._set_video_frame_args( - { - "target_frames": 10, - "fps": 2, - "min_frames": 1, - "max_frames": 100, - "frames_sample": "leading", - }, - {"duration": 10.0}, - ) - with self.assertRaises(ValueError): - self.processor._set_video_frame_args( - { - "target_frames": 5, - "fps": -1, - "min_frames": 10, - "max_frames": 100, - "frames_sample": "leading", - }, - {"duration": 10.0}, - ) - with self.assertRaises(ValueError): - self.processor._set_video_frame_args( - { - "target_frames": 200, - "fps": -1, - "min_frames": 1, - "max_frames": 100, - "frames_sample": "leading", - }, - {"duration": 10.0}, - ) - with self.assertRaises(ValueError): - self.processor._set_video_frame_args( - { - "target_frames": -1, - "fps": 2, - "min_frames": 100, - "max_frames": 10, - "frames_sample": "leading", - }, - {"duration": 10.0}, - ) - - # Adjustment cases - result = self.processor._set_video_frame_args( - { - "target_frames": -1, - "fps": 1, - "min_frames": 10, - "max_frames": 100, - "frames_sample": "leading", - }, - {"duration": 1.0}, - ) - self.assertEqual(result["target_frames"], 10) - self.assertEqual(result["fps"], -1) - - result = self.processor._set_video_frame_args( - { - "target_frames": -1, - "fps": 10, - "min_frames": 1, - "max_frames": 100, - "frames_sample": "leading", - }, - {"duration": 100.0}, - ) - self.assertEqual(result["target_frames"], 100) - self.assertEqual(result["fps"], -1) - - def test_text2ids_comprehensive(self): - """Test text2ids with various scenarios""" - # Text only - outputs = self.processor.text2ids("Hello world") - self.assertIn("input_ids", outputs) - self.assertEqual(len(outputs["images"]), 0) - - # Empty text - outputs = self.processor.text2ids("") - self.assertEqual(len(outputs["input_ids"]), 0) - - # With image placeholder - mock_image = Image.new("RGB", (224, 224)) - outputs = self.processor.text2ids("Hello <|image@placeholder|> world", images=[mock_image]) - self.assertGreater(len(outputs["input_ids"]), 0) - self.assertGreater(len(outputs["images"]), 0) - - # With cached image - cached_image = ( - np.random.rand(256, 3 * 14 * 14).astype(np.float32), - {"thw": (1, 16, 16)}, - ) - outputs = self.processor.text2ids( - "Hello <|image@placeholder|> world", - images=[cached_image], - image_uuid=["uuid"], - ) - self.assertGreater(len(outputs["input_ids"]), 0) - - # Multiple images - outputs = self.processor.text2ids( - "Hello <|image@placeholder|> world <|image@placeholder|> end", - images=[mock_image, mock_image], - ) - self.assertEqual(len(outputs["images"]), 2) - - # With video placeholder - mock_read, mock_frames_read, mock_render, mock_frames = self._mock_video_processing() - with mock_read as mr, mock_frames_read as mfr, mock_render as mren: - mr.return_value = (None, {"duration": 2.0}, "test_path") - mfr.return_value = ( - [np.array(f) for f in mock_frames], - None, - [0.0, 0.5, 1.0, 1.5], - ) - mren.side_effect = lambda img, ts: (Image.fromarray(img) if isinstance(img, np.ndarray) else img) - self.mock_image_preprocessor.preprocess.return_value = { - "pixel_values_videos": np.random.rand(4, 256, 3 * 14 * 14).astype(np.float32), - "video_grid_thw": np.array([[4, 16, 16]]), - } - outputs = self.processor.text2ids("Hello <|video@placeholder|> world", videos=["test_video.mp4"]) - self.assertGreater(len(outputs["input_ids"]), 0) - - # Cached video - cached_video = ( - np.random.rand(256, 3 * 14 * 14).astype(np.float32), - {"thw": (4, 16, 16)}, - ) - outputs = self.processor.text2ids( - "Hello <|video@placeholder|> world", - videos=[cached_video], - video_uuid=["uuid"], - ) - self.assertGreater(len(outputs["input_ids"]), 0) - - # Video dict format - mock_read, mock_frames_read, mock_render, mock_frames = self._mock_video_processing() - with mock_read as mr, mock_frames_read as mfr, mock_render as mren: - mr.return_value = (None, {"duration": 2.0}, "test_path") - mfr.return_value = ( - [np.array(f) for f in mock_frames], - None, - [0.0, 0.5, 1.0, 1.5], - ) - mren.side_effect = lambda img, ts: (Image.fromarray(img) if isinstance(img, np.ndarray) else img) - self.mock_image_preprocessor.preprocess.return_value = { - "pixel_values_videos": np.random.rand(4, 256, 3 * 14 * 14).astype(np.float32), - "video_grid_thw": np.array([[4, 16, 16]]), - } - outputs = self.processor.text2ids( - "Hello <|video@placeholder|> world", - videos=[{"video": "test.mp4", "fps": 2}], - ) - self.assertGreater(len(outputs["input_ids"]), 0) - - # Image and video together - mock_read, mock_frames_read, mock_render, mock_frames = self._mock_video_processing() - with mock_read as mr, mock_frames_read as mfr, mock_render as mren: - mr.return_value = (None, {"duration": 2.0}, "test_path") - mfr.return_value = ( - [np.array(f) for f in mock_frames], - None, - [0.0, 0.5, 1.0, 1.5], - ) - mren.side_effect = lambda img, ts: (Image.fromarray(img) if isinstance(img, np.ndarray) else img) - self.mock_image_preprocessor.preprocess.side_effect = [ - { - "pixel_values": np.random.rand(256, 3 * 14 * 14).astype(np.float32), - "image_grid_thw": np.array([[1, 16, 16]]), - }, - { - "pixel_values_videos": np.random.rand(4, 256, 3 * 14 * 14).astype(np.float32), - "video_grid_thw": np.array([[4, 16, 16]]), - }, - ] - outputs = self.processor.text2ids( - "Hello <|image@placeholder|> world <|video@placeholder|> end", - images=[mock_image], - videos=["test_video.mp4"], - ) - self.assertGreater(len(outputs["input_ids"]), 0) - self.mock_image_preprocessor.preprocess.side_effect = None - - def test_request2ids_comprehensive(self): - """Test request2ids with various scenarios""" - self.processor.is_training = False - - # Basic request with multimodal content - covers both text and image branches in one call - mock_image = Image.new("RGB", (224, 224)) - request = { - "request_id": "test_0", - "messages": [ - { - "role": "user", - "content": [ - {"type": "text", "text": "What's in this image?"}, - {"type": "image", "data": mock_image, "uuid": "img1"}, - ], - } - ], - "add_generation_prompt": True, - } - request = Request.from_dict(request) - with patch("fastdeploy.input.v1.ernie4_5_vl_processor.process.parse_chat_messages") as mock_parse: - mock_parse.return_value = request.messages - outputs = self.processor.request2ids(request) - self.assertIn("input_ids", outputs) - - # Error case: missing chat_template - self.processor.tokenizer.chat_template = None - with patch("fastdeploy.input.v1.ernie4_5_vl_processor.process.parse_chat_messages") as mock_parse: - mock_parse.return_value = [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}] - with self.assertRaises(ValueError): - self.processor.request2ids(request) - self.processor.tokenizer.chat_template = MagicMock() - - # Error case: unsupported role - request = { - "request_id": "test_0", - "messages": [{"role": "invalid_role", "content": "Hello"}], - "add_generation_prompt": True, - } - request = Request.from_dict(request) - with patch("fastdeploy.input.v1.ernie4_5_vl_processor.process.parse_chat_messages") as mock_parse: - mock_parse.return_value = [{"role": "invalid_role", "content": [{"type": "text", "text": "Hello"}]}] - with self.assertRaises(AssertionError): - self.processor.request2ids(request) - - # Error case: missing cache when cache is disabled - self.processor.enable_processor_cache = False - request = { - "request_id": "test_0", - "messages": [{"role": "user", "content": [{"type": "image", "uuid": "img1"}]}], - } - request = Request.from_dict(request) - with patch("fastdeploy.input.v1.ernie4_5_vl_processor.process.parse_chat_messages") as mock_parse: - mock_parse.return_value = request.messages - with self.assertRaises(ValueError): - self.processor.request2ids(request) - - def test_extract_labels(self): - """Test label extraction""" - outputs = {"input_ids": [1, 2, 3, self.processor.sep_token_id, 4, 5], "labels": []} - self.processor.is_training = True - self.processor._extract_labels(outputs, ["target text"]) - self.assertEqual(len(outputs["labels"]), len(outputs["input_ids"])) - - # Multiple targets - outputs2 = { - "input_ids": [1, 2, 3, self.processor.sep_token_id, 4, 5, self.processor.sep_token_id, 6, 7], - "labels": [], - } - self.processor._extract_labels(outputs2, ["target1", "target2"]) - self.assertEqual(len(outputs2["labels"]), len(outputs2["input_ids"])) - - # Error case - outputs3 = {"input_ids": [1, 2, 3, self.processor.sep_token_id], "labels": []} - with self.assertRaises(AssertionError): - self.processor._extract_labels(outputs3, ["target1", "target2"]) - - def test_fancy_print(self): - """Test fancy_print function""" - from fastdeploy.input.v1.ernie4_5_vl_processor.process import fancy_print - - test_cases = [ - ([1, 2, 3, self.processor.image_patch_id, 4, 5], self.processor.image_patch_id, None), - ( - [ - 1, - 2, - self.processor.image_patch_id, - self.processor.image_patch_id, - self.processor.image_patch_id, - 4, - 5, - ], - self.processor.image_patch_id, - "<|IMAGE@", - ), - ([1, 2, 3, 4, 5], self.processor.image_patch_id, None), - ] - for input_ids, image_patch_id, expected_contains in test_cases: - result = fancy_print(input_ids, self.mock_tokenizer, image_patch_id) - self.assertIsInstance(result, str) - if expected_contains: - self.assertIn(expected_contains, result) - - def test_processor_cache_operations(self): - """Test processor cache get/update and request2ids with cache""" - # Test get_processor_cache - mock_socket = MagicMock() - mock_socket.recv_multipart = MagicMock(return_value=(b"", b"pickled_data")) - with patch("fastdeploy.input.v1.ernie4_5_vl_processor.process.pickle") as mock_pickle: - mock_pickle.loads = MagicMock(return_value=[{"data": "cached_item"}]) - result = self.processor.get_processor_cache(mock_socket, ["hash1", "hash2"]) - self.assertEqual(len(result), 1) - - # Test update_processor_cache - mock_socket2 = MagicMock() - with patch("fastdeploy.input.v1.ernie4_5_vl_processor.process.pickle"): - self.processor.update_processor_cache( - mock_socket2, - ["hash1"], - [(np.array([1, 2, 3]), {"meta": "data"})], - ) - mock_socket2.send_multipart.assert_called_once() - - # Test request2ids with processor cache update - self.processor.is_training = False - self.processor.enable_processor_cache = True - mock_image = Image.new("RGB", (224, 224)) - request = { - "request_id": "test_0", - "messages": [ - { - "role": "user", - "content": [ - {"type": "text", "text": "Hello"}, - {"type": "image", "data": mock_image, "uuid": "img1"}, - ], - } - ], - "add_generation_prompt": True, - } - request = Request.from_dict(request) - with patch("fastdeploy.input.v1.ernie4_5_vl_processor.process.zmq") as mock_zmq: - mock_context = MagicMock() - mock_socket = MagicMock() - mock_socket.recv_multipart = MagicMock(return_value=(b"", b"pickled_data")) - mock_context.socket.return_value = mock_socket - mock_zmq.Context.return_value = mock_context - with patch("fastdeploy.input.v1.ernie4_5_vl_processor.process.parse_chat_messages") as mock_parse: - mock_parse.return_value = request.messages - with patch("fastdeploy.input.v1.ernie4_5_vl_processor.process.pickle") as mock_pickle: - mock_pickle.loads = MagicMock(return_value=[]) - with patch.object(self.processor, "text2ids") as mock_text2ids: - mock_text2ids.return_value = { - "input_ids": [1, 2, 3], - "token_type_ids": [0] * 3, - "position_ids": [[i] * 3 for i in range(3)], - "images": [np.random.rand(256, 3 * 14 * 14).astype(np.float32)], - "grid_thw": [np.array([[1, 16, 16]])], - "image_type_ids": [0], - "cur_position": 3, - "video_cnt": 0, - "num_input_image_tokens": 0, - "num_input_video_tokens": 0, - "mm_positions": [], - "mm_hashes": ["hash1"], - } - with patch.object(self.processor, "update_processor_cache") as mock_update: - self.processor.request2ids(request) - mock_update.assert_called_once() - self.processor.enable_processor_cache = False - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/input/v1/test_image_preprocessor_adaptive.py b/tests/input/v1/test_image_preprocessor_adaptive.py deleted file mode 100644 index 5a15244d1fd..00000000000 --- a/tests/input/v1/test_image_preprocessor_adaptive.py +++ /dev/null @@ -1,499 +0,0 @@ -""" -# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -import unittest -from unittest.mock import patch - -import numpy as np -from PIL import Image - -from fastdeploy.input.v1.ernie4_5_vl_processor.image_preprocessor.image_preprocessor_adaptive import ( - AdaptiveImageProcessor, - ceil_by_factor, - floor_by_factor, - is_scaled_image, - make_batched_images, - make_batched_videos, - round_by_factor, - smart_resize, -) - - -class TestImagePreprocessorAdaptive(unittest.TestCase): - def setUp(self): - """Set up test environment""" - self.processor = AdaptiveImageProcessor( - min_pixels=56 * 56, - max_pixels=28 * 28 * 1280, - patch_size=14, - temporal_conv_size=2, - merge_size=2, - ) - - def test_init(self): - """Test initialization""" - self.assertEqual(self.processor.min_pixels, 56 * 56) - self.assertEqual(self.processor.max_pixels, 28 * 28 * 1280) - self.assertEqual(self.processor.patch_size, 14) - self.assertEqual(self.processor.temporal_conv_size, 2) - self.assertEqual(self.processor.merge_size, 2) - - def test_set_pixels(self): - """Test setting pixels with valid and invalid values (lines 205-214)""" - # Test setting only min_pixels - self.processor.set_pixels(min_pixels=100, msg="test") - self.assertEqual(self.processor.min_pixels, 100) - self.assertEqual(self.processor.size["min_pixels"], 100) - - # Test setting only max_pixels - self.processor.set_pixels(max_pixels=200, msg="test") - self.assertEqual(self.processor.max_pixels, 200) - self.assertEqual(self.processor.size["max_pixels"], 200) - - # Test setting both - self.processor.set_pixels(min_pixels=150, max_pixels=250, msg="test") - self.assertEqual(self.processor.min_pixels, 150) - self.assertEqual(self.processor.max_pixels, 250) - self.assertEqual(self.processor.size["min_pixels"], 150) - self.assertEqual(self.processor.size["max_pixels"], 250) - - # Invalid cases - with self.assertRaises(AssertionError): - self.processor.set_pixels(min_pixels=-1) - with self.assertRaises(AssertionError): - self.processor.set_pixels(max_pixels=0) - - def test_get_smarted_resize(self): - """Test get_smarted_resize with default and custom pixels""" - height, width = 224, 224 - # Test with default pixels - (resized_h, resized_w), (patches_h, patches_w) = self.processor.get_smarted_resize(height, width) - self.assertIsInstance(resized_h, int) - self.assertIsInstance(resized_w, int) - self.assertIsInstance(patches_h, int) - self.assertIsInstance(patches_w, int) - # Test with custom pixels - (resized_h, resized_w), (_, _) = self.processor.get_smarted_resize( - height, width, min_pixels=100, max_pixels=10000 - ) - self.assertIsInstance(resized_h, int) - self.assertIsInstance(resized_w, int) - - def test_round_by_factor(self): - """Test round_by_factor with various cases""" - self.assertEqual(round_by_factor(100, 28), 112) # 100/28 ≈ 3.57, round(3.57) = 4, 4*28 = 112 - self.assertEqual(round_by_factor(50, 10), 50) - self.assertEqual(round_by_factor(55, 10), 60) - # Edge cases - self.assertEqual(round_by_factor(0, 14), 0) - self.assertEqual(round_by_factor(14, 14), 14) - self.assertEqual(round_by_factor(13, 14), 14) # Round up - self.assertEqual(round_by_factor(15, 14), 14) # Round down - - def test_ceil_by_factor(self): - """Test ceil_by_factor with various cases""" - self.assertEqual(ceil_by_factor(100, 28), 112) # ceil(100/28)*28 = ceil(3.57)*28 = 4*28 = 112 - self.assertEqual(ceil_by_factor(50, 10), 50) - self.assertEqual(ceil_by_factor(55, 10), 60) - # Edge cases - self.assertEqual(ceil_by_factor(0, 14), 0) - self.assertEqual(ceil_by_factor(14, 14), 14) - self.assertEqual(ceil_by_factor(13, 14), 14) # Ceil up - self.assertEqual(ceil_by_factor(15, 14), 28) # Ceil up to next multiple - - def test_floor_by_factor(self): - """Test floor_by_factor with various cases""" - self.assertEqual(floor_by_factor(100, 28), 84) # floor(100/28)*28 = floor(3.57)*28 = 3*28 = 84 - self.assertEqual(floor_by_factor(50, 10), 50) - self.assertEqual(floor_by_factor(55, 10), 50) - # Edge cases - self.assertEqual(floor_by_factor(0, 14), 0) - self.assertEqual(floor_by_factor(14, 14), 14) - self.assertEqual(floor_by_factor(13, 14), 0) # Floor down - self.assertEqual(floor_by_factor(15, 14), 14) # Floor down to multiple - self.assertEqual(floor_by_factor(28, 14), 28) # Exact multiple - - def test_smart_resize(self): - """Test smart_resize with various scenarios (lines 557-587)""" - # Basic functionality - height, width = 224, 224 - new_h, new_w = smart_resize(height, width, factor=28, min_pixels=56 * 56, max_pixels=28 * 28 * 1280) - self.assertIsInstance(new_h, int) - self.assertIsInstance(new_w, int) - self.assertEqual(new_h % 28, 0) - self.assertEqual(new_w % 28, 0) - - # High aspect ratio (height > width) - tests lines 557-563 - height, width = 10000, 10 # aspect ratio > 200 - new_h, new_w = smart_resize(height, width, factor=28, min_pixels=56 * 56, max_pixels=28 * 28 * 1280) - self.assertIsInstance(new_h, int) - self.assertIsInstance(new_w, int) - self.assertLessEqual(max(new_h, new_w) / min(new_h, new_w), 200) - - # High aspect ratio (width > height) - tests lines 562-563 - height, width = 10, 10000 - new_h, new_w = smart_resize(height, width, factor=28, min_pixels=56 * 56, max_pixels=28 * 28 * 1280) - self.assertIsInstance(new_h, int) - self.assertIsInstance(new_w, int) - self.assertLessEqual(max(new_h, new_w) / min(new_h, new_w), 200) - - # Too large - tests lines 575-578 - height, width = 10000, 10000 - new_h, new_w = smart_resize(height, width, factor=28, min_pixels=56 * 56, max_pixels=28 * 28 * 1280) - self.assertLessEqual(new_h * new_w, 28 * 28 * 1280) - - # Too small - tests lines 579-582 - height, width = 10, 10 - new_h, new_w = smart_resize(height, width, factor=28, min_pixels=56 * 56, max_pixels=28 * 28 * 1280) - self.assertGreaterEqual(new_h * new_w, 56 * 56) - - # Exceeds max_pixels with custom parameters - height, width = 10000, 10000 - max_pixels = 10000 - min_pixels = 1000 - new_h, new_w = smart_resize(height, width, factor=14, min_pixels=min_pixels, max_pixels=max_pixels) - self.assertLessEqual(new_h * new_w, max_pixels) - self.assertGreaterEqual(new_h * new_w, min_pixels) - - # Below min_pixels with custom parameters - height, width = 10, 10 - min_pixels = 10000 - max_pixels = 100000 - new_h, new_w = smart_resize(height, width, factor=14, min_pixels=min_pixels, max_pixels=max_pixels) - self.assertGreaterEqual(new_h * new_w, min_pixels) - self.assertLessEqual(new_h * new_w, max_pixels) - - # Invalid result (extreme parameters) - tests lines 584-585 - with self.assertRaises(ValueError): - smart_resize(1, 1, factor=100000, min_pixels=100, max_pixels=1000) - - def test_is_scaled_image(self): - """Test is_scaled_image with various image types""" - # uint8 image - image = np.array([[0, 255], [128, 200]], dtype=np.uint8) - self.assertFalse(is_scaled_image(image)) - image = np.random.rand(224, 224, 3).astype(np.uint8) * 255 - self.assertFalse(is_scaled_image(image)) - - # Scaled float image (values in [0, 1]) - image = np.array([[0.0, 0.5], [0.3, 1.0]], dtype=np.float32) - self.assertTrue(is_scaled_image(image)) - image = np.random.rand(224, 224, 3).astype(np.float32) * 0.5 - self.assertTrue(is_scaled_image(image)) - - # Unscaled float image (values > 1) - image = np.array([[0.0, 255.0], [128.0, 300.0]], dtype=np.float32) - self.assertFalse(is_scaled_image(image)) - image = np.random.rand(224, 224, 3).astype(np.float32) * 255 - self.assertFalse(is_scaled_image(image)) - - # Edge cases - image = np.array([[0.0, 1.0]], dtype=np.float32) - self.assertTrue(is_scaled_image(image)) - image = np.array([[0.0, 1.1]], dtype=np.float32) - self.assertFalse(is_scaled_image(image)) - image = np.array([[-0.1, 1.0]], dtype=np.float32) - self.assertFalse(is_scaled_image(image)) - - def test_make_batched_images(self): - """Test make_batched_images with various input types""" - # Single image - img = Image.new("RGB", (224, 224)) - result = make_batched_images(img) - self.assertEqual(len(result), 1) - self.assertEqual(result[0], img) - - # List of images - imgs = [Image.new("RGB", (224, 224)) for _ in range(3)] - result = make_batched_images(imgs) - self.assertEqual(len(result), 3) - self.assertEqual(result, imgs) - - # Nested list - imgs = [[Image.new("RGB", (224, 224)) for _ in range(2)] for _ in range(2)] - result = make_batched_images(imgs) - self.assertEqual(len(result), 4) # 2*2 = 4 - - # Invalid inputs - with self.assertRaises(ValueError) as context: - make_batched_images("invalid") - self.assertIn("Could not make batched images", str(context.exception)) - with self.assertRaises(ValueError) as context: - make_batched_images([[1, 2, 3], [4, 5, 6]]) - self.assertIn("Could not make batched images", str(context.exception)) - - def test_make_batched_videos(self): - """Test make_batched_videos with various input types""" - # List of images - imgs = [Image.new("RGB", (224, 224)) for _ in range(3)] - result = make_batched_videos(imgs) - self.assertEqual(len(result), 1) - self.assertEqual(len(result[0]), 3) - - # Single image in list - img = Image.new("RGB", (224, 224)) - result = make_batched_videos([img]) - self.assertEqual(len(result), 1) - self.assertEqual(len(result[0]), 1) - - # Nested list - imgs = [[Image.new("RGB", (224, 224)) for _ in range(2)] for _ in range(2)] - result = make_batched_videos(imgs) - self.assertEqual(len(result), 2) - self.assertEqual(len(result[0]), 2) - - # 4D array (single) - video = np.random.rand(3, 224, 224, 3).astype(np.uint8) - result = make_batched_videos(video) - self.assertEqual(len(result), 1) - self.assertIsInstance(result[0], list) - - # 4D array in list (lines 119-120) - videos = [np.random.rand(3, 224, 224, 3).astype(np.uint8)] - result = make_batched_videos(videos) - self.assertEqual(len(result), 1) - self.assertIsInstance(result[0], list) - - # Invalid input - with self.assertRaises(ValueError) as context: - make_batched_videos("invalid") - self.assertIn("Could not make batched video", str(context.exception)) - - def test_preprocess_images(self): - """Test preprocess handling images""" - img = Image.new("RGB", (224, 224)) - result = self.processor.preprocess(images=img) - self.assertIn("pixel_values", result) - self.assertIn("image_grid_thw", result) - # Verify pixel_values shape - pixel_values = result["pixel_values"] - self.assertIsInstance(pixel_values, np.ndarray) - - def test_preprocess_videos(self): - """Test preprocess handling videos""" - frames = [Image.new("RGB", (224, 224)) for _ in range(4)] - result = self.processor.preprocess(images=None, videos=frames) - self.assertIn("pixel_values_videos", result) - self.assertIn("video_grid_thw", result) - - def test_preprocess_invalid_images(self): - """Test preprocess handling invalid image""" - with self.assertRaises(ValueError): - self.processor.preprocess(images="invalid") - - def test_preprocess_with_predetermined_grid_thw(self): - """Test preprocess using predetermined_grid_thw""" - img = Image.new("RGB", (224, 224)) - # predetermined_grid_thw should be (h, w) format, not [1, h, w] - predetermined_grid_thw = [(16, 16)] # For single image, should be (h, w) tuple - result = self.processor.preprocess(images=img, predetermined_grid_thw=predetermined_grid_thw) - self.assertIn("pixel_values", result) - - def test_preprocess_flags(self): - """Test preprocess with various flags disabled""" - img = Image.new("RGB", (224, 224)) - # Test without resize - result = self.processor.preprocess(images=img, do_resize=False) - self.assertIn("pixel_values", result) - # Test without rescale - result = self.processor.preprocess(images=img, do_rescale=False) - self.assertIn("pixel_values", result) - # Test without normalize - result = self.processor.preprocess(images=img, do_normalize=False) - self.assertIn("pixel_values", result) - - def test_preprocess_custom_mean_std(self): - """Test preprocess using custom mean and std""" - img = Image.new("RGB", (224, 224)) - # Test with simple custom mean/std - result = self.processor.preprocess(images=img, image_mean=[0.5, 0.5, 0.5], image_std=[0.5, 0.5, 0.5]) - self.assertIn("pixel_values", result) - # Test with ImageNet-style mean/std - result = self.processor.preprocess( - images=img, image_mean=[0.485, 0.456, 0.406], image_std=[0.229, 0.224, 0.225] - ) - self.assertIn("pixel_values", result) - - def test_preprocess_do_convert_rgb(self): - """Test preprocess with do_convert_rgb=True (line 289)""" - img = Image.new("L", (224, 224)) # Grayscale image - result = self.processor.preprocess(images=img, do_convert_rgb=True) - self.assertIn("pixel_values", result) - - def test_preprocess_scaled_image_warning(self): - """Test warning for scaled image in preprocess (lines 294-298)""" - # Create a scaled image (values between 0-1) - img_array = np.random.rand(224, 224, 3).astype(np.float32) * 0.5 - # Use patch to capture warning - with patch( - "fastdeploy.input.v1.ernie4_5_vl_processor.image_preprocessor.image_preprocessor_adaptive.data_processor_logger" - ) as mock_logger: - # Directly call _preprocess, pass scaled image - self.processor._preprocess( - [img_array], # Pass scaled numpy array - do_rescale=True, - do_convert_rgb=False, - ) - # Verify warning is called when is_scaled_image returns True and do_rescale is True - mock_logger.warning.assert_called() - - def test_preprocess_invalid_images_check(self): - """Test invalid image check in preprocess (line 464)""" - # Test invalid image type - need to ensure valid_images returns False - # Use patch to make valid_images return False, but make_batched_images succeeds - with patch( - "fastdeploy.input.v1.ernie4_5_vl_processor.image_preprocessor.image_preprocessor_adaptive.valid_images" - ) as mock_valid: - mock_valid.return_value = False - valid_images_list = [Image.new("RGB", (224, 224))] # Valid image, but valid_images returns False - with self.assertRaises(ValueError) as context: - self.processor.preprocess(images=valid_images_list) - self.assertIn("Invalid image type", str(context.exception)) - - def test_preprocess_predetermined_grid_thw_multiple_images(self): - """Test preprocess with predetermined_grid_thw for multiple images (lines 307-310)""" - imgs = [Image.new("RGB", (224, 224)) for _ in range(2)] - predetermined_grid_thw = [(16, 16), (20, 20)] - result = self.processor.preprocess(images=imgs, predetermined_grid_thw=predetermined_grid_thw) - self.assertIn("pixel_values", result) - - def test_preprocess_predetermined_grid_thw_length_mismatch(self): - """Test preprocess with predetermined_grid_thw length mismatch (lines 307-310, 470)""" - imgs = [Image.new("RGB", (224, 224)) for _ in range(2)] - predetermined_grid_thw = [(16, 16)] # Length mismatch - only 1 element for 2 images - # The function raises IndexError when accessing predetermined_grid_thw[img_idx] with img_idx=1 - with self.assertRaises(IndexError): - self.processor.preprocess(images=imgs, predetermined_grid_thw=predetermined_grid_thw) - - def test_preprocess_with_input_data_format(self): - """Test preprocess with input_data_format parameter (lines 299-301)""" - img = Image.new("RGB", (224, 224)) - from paddleformers.transformers.image_utils import ChannelDimension - - # Test with FIRST - result = self.processor.preprocess(images=img, input_data_format=ChannelDimension.FIRST) - self.assertIn("pixel_values", result) - # Test with None - result = self.processor.preprocess(images=img, input_data_format=None) - self.assertIn("pixel_values", result) - - def test_preprocess_do_resize_with_predetermined_grid_thw(self): - """Test preprocess with do_resize=True and predetermined_grid_thw (lines 314-317)""" - img = Image.new("RGB", (224, 224)) - predetermined_grid_thw = [(16, 16)] - result = self.processor.preprocess(images=img, predetermined_grid_thw=predetermined_grid_thw, do_resize=True) - self.assertIn("pixel_values", result) - - def test_preprocess_videos_with_predetermined_grid_thw(self): - """Test preprocess videos with predetermined_grid_thw (lines 511)""" - frames = [Image.new("RGB", (224, 224)) for _ in range(4)] - predetermined_grid_thw = [(16, 16)] * 4 - result = self.processor.preprocess(images=None, videos=frames, predetermined_grid_thw=predetermined_grid_thw) - self.assertIn("pixel_values_videos", result) - - def test_preprocess_return_tensors(self): - """Test preprocess with return_tensors parameter (lines 396, 523)""" - img = Image.new("RGB", (224, 224)) - # Use string instead of TensorType enum which may not be available - result = self.processor.preprocess(images=img, return_tensors="np") - self.assertIn("pixel_values", result) - - def test_preprocess_do_rescale_false_with_scaled_image(self): - """Test preprocess with do_rescale=False and scaled image (line 335)""" - # Create a scaled image - img_array = np.random.rand(224, 224, 3).astype(np.float32) * 0.5 # Values in [0, 0.5] - img = Image.fromarray((img_array * 255).astype(np.uint8)) - result = self.processor.preprocess(images=img, do_rescale=False) - self.assertIn("pixel_values", result) - - def test_preprocess_custom_resample(self): - """Test preprocess with custom resample parameter (line 332)""" - img = Image.new("RGB", (224, 224)) - from PIL import Image as PILImage - - result = self.processor.preprocess(images=img, resample=PILImage.BILINEAR) - self.assertIn("pixel_values", result) - - def test_preprocess_custom_rescale_factor(self): - """Test preprocess with custom rescale_factor (line 336)""" - img = Image.new("RGB", (224, 224)) - result = self.processor.preprocess(images=img, rescale_factor=1.0 / 128.0) - self.assertIn("pixel_values", result) - - def test_preprocess_data_format(self): - """Test preprocess with different data_format values""" - img = Image.new("RGB", (224, 224)) - from paddleformers.transformers.image_utils import ChannelDimension - - # Test with FIRST - result = self.processor.preprocess(images=img, data_format=ChannelDimension.FIRST) - self.assertIn("pixel_values", result) - # Test with LAST - result = self.processor.preprocess(images=img, data_format=ChannelDimension.LAST) - self.assertIn("pixel_values", result) - - def test_preprocess_multiple_images_loop(self): - """Test preprocess loop with multiple images (lines 312-348, 468-488)""" - images = [Image.new("RGB", (224, 224)) for _ in range(3)] - result = self.processor.preprocess(images=images) - self.assertIn("pixel_values", result) - self.assertIn("image_grid_thw", result) - pixel_values = result["pixel_values"] - self.assertIsInstance(pixel_values, np.ndarray) - self.assertEqual(len(pixel_values.shape), 2) # Should be [grid_t * grid_h * grid_w, C * psz * psz] - - def test_preprocess_videos_loop(self): - """Test preprocess with videos in loop (lines 496-521)""" - # Test with multiple videos - videos = [ - [Image.new("RGB", (224, 224)) for _ in range(4)], - [Image.new("RGB", (224, 224)) for _ in range(4)], - ] - result = self.processor.preprocess(images=None, videos=videos) - self.assertIn("pixel_values_videos", result) - self.assertIn("video_grid_thw", result) - self.assertIsInstance(result["pixel_values_videos"], np.ndarray) - # Test with nested list format - videos = [[Image.new("RGB", (224, 224)) for _ in range(4)] for _ in range(2)] - result = self.processor.preprocess(images=None, videos=videos) - self.assertIn("pixel_values_videos", result) - self.assertIn("video_grid_thw", result) - self.assertIsInstance(result["pixel_values_videos"], np.ndarray) - - def test_preprocess_both_images_and_videos(self): - """Test preprocess with both images and videos (lines 458-523)""" - images = [Image.new("RGB", (224, 224))] - videos = [[Image.new("RGB", (224, 224)) for _ in range(4)]] - result = self.processor.preprocess(images=images, videos=videos) - # Due to implementation, only video results are returned when both are provided - self.assertIn("pixel_values_videos", result) - self.assertIn("video_grid_thw", result) - - def test_preprocess_invalid_images_check_list_input(self): - """Test preprocess with invalid images check (line 464) - - Note: The error is raised by make_batched_images before valid_images check, - so the error message is different. - """ - invalid_images = ["not an image", "also not an image"] - - with self.assertRaises(ValueError) as context: - self.processor.preprocess(images=invalid_images) - self.assertIn("Could not make batched images", str(context.exception)) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/input/v1/test_paddleocr_vl_processor.py b/tests/input/v1/test_paddleocr_vl_processor.py deleted file mode 100644 index 3c1e83b42ff..00000000000 --- a/tests/input/v1/test_paddleocr_vl_processor.py +++ /dev/null @@ -1,1182 +0,0 @@ -""" -# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -import pickle -import unittest -from unittest.mock import ANY, MagicMock, patch - -import numpy as np -import zmq -from PIL import Image - -from fastdeploy.engine.request import Request -from fastdeploy.input.v1.paddleocr_vl_processor.image_processor import ( - ImageProcessor, - smart_resize, -) -from fastdeploy.input.v1.paddleocr_vl_processor.paddleocr_vl_processor import ( - PaddleOCRVLProcessor, -) -from fastdeploy.input.v1.paddleocr_vl_processor.process import DataProcessor -from fastdeploy.input.v1.paddleocr_vl_processor.process_video import sample_frames - -MODULE_PATH = "fastdeploy.input.v1.paddleocr_vl_processor.process" - - -class TestProcessVideo(unittest.TestCase): - def setUp(self): - self.metadata = {"num_of_frame": 100, "fps": 25} - self.frame_factor = 4 - self.min_frames = 8 - self.max_frames = 32 - - def test_sample_with_num_frames(self): - """测试使用num_frames参数采样（来自用户的原始测试）""" - num_frames = 16 - indices = sample_frames( - frame_factor=self.frame_factor, - min_frames=self.min_frames, - max_frames=self.max_frames, - num_frames=num_frames, - fps=0, # 确保 fps 不>0 - metadata=self.metadata, - ) - self.assertEqual(len(indices), 16) - self.assertEqual(indices[0], 0) - self.assertEqual(indices[-1], 93) - np.testing.assert_array_equal(indices, np.arange(0, 100, 100 / 16).astype(np.int32)) - - def test_error_num_frames_exceeds_total(self): - """测试 num_frames 超过总帧数的异常（来自用户的原始测试）""" - with self.assertRaises(ValueError) as context: - sample_frames( - frame_factor=self.frame_factor, - min_frames=self.min_frames, - max_frames=self.max_frames, - num_frames=200, # 超过总帧数100 - fps=0, - metadata=self.metadata, - ) - self.assertIn("exceeds", str(context.exception)) - - def test_error_mutual_exclusion(self): - """新增：测试 num_frames 和 fps 互斥""" - with self.assertRaises(ValueError) as context: - sample_frames( - frame_factor=self.frame_factor, - min_frames=self.min_frames, - max_frames=self.max_frames, - num_frames=16, # > 0 - fps=10, # > 0 - metadata=self.metadata, - ) - self.assertIn("mutually exclusive", str(context.exception)) - - def test_error_fps_without_metadata(self): - """新增：测试 fps > 0 但 metadata 为 None""" - with self.assertRaises(TypeError) as context: - sample_frames( - frame_factor=self.frame_factor, - min_frames=self.min_frames, - max_frames=self.max_frames, - num_frames=0, - fps=10, - metadata=None, # 缺失 - ) - # 验证是预期的 TypeError - self.assertIn("'NoneType' object is not subscriptable", str(context.exception)) - - def test_num_frames_rounding(self): - """新增：测试 num_frames 向 frame_factor 舍入""" - num_frames = 17 # 不是 4 的倍数 - # 逻辑: round(17 / 4) * 4 = round(4.25) * 4 = 4 * 4 = 16 - indices = sample_frames( - frame_factor=self.frame_factor, - min_frames=self.min_frames, - max_frames=self.max_frames, - num_frames=num_frames, - fps=0, - metadata=self.metadata, - ) - # 应舍入到 16 - self.assertEqual(len(indices), 16) - - def test_sample_with_fps_basic(self): - """新增：测试使用 fps 采样（基本路径，被 max_frames 限制）""" - # 逻辑: num_frames_calc = 100 / 25 * 10 = 40 - # num_frames_clamped = min(max(40, 8), 32) = 32 - # num_frames_factored = floor(32 / 4) * 4 = 32 - indices = sample_frames( - frame_factor=self.frame_factor, - min_frames=self.min_frames, - max_frames=self.max_frames, - num_frames=0, - fps=10, - metadata=self.metadata, - ) - # 应被 max_frames=32 限制 - self.assertEqual(len(indices), 32) - self.assertEqual(indices[-1], 96) - - def test_sample_with_fps_hits_min_frames(self): - """新增：测试使用 fps 采样（被 min_frames 限制）""" - # 逻辑: num_frames_calc = 100 / 25 * 1 = 4 - # num_frames_clamped = min(max(4, 8), 32) = 8 - # num_frames_factored = floor(8 / 4) * 4 = 8 - indices = sample_frames( - frame_factor=self.frame_factor, - min_frames=self.min_frames, - max_frames=self.max_frames, - num_frames=0, - fps=1, - metadata=self.metadata, - ) - # 应被 min_frames=8 限制 - self.assertEqual(len(indices), 8) - self.assertEqual(indices[-1], 87) - - def test_sample_with_fps_hits_total_frames(self): - """新增：测试使用 fps 采样（被 total_num_frames 限制）""" - local_max_frames = 200 - - # 逻辑: num_frames_calc = 100 / 25 * 50 = 200 - # num_frames_clamped = min(min(max(200, 8), 200), 100) = 100 - # num_frames_factored = floor(100 / 4) * 4 = 100 - indices = sample_frames( - frame_factor=self.frame_factor, - min_frames=self.min_frames, - max_frames=local_max_frames, - num_frames=0, - fps=50, - metadata=self.metadata, - ) - # 应被 total_num_frames=100 限制 - self.assertEqual(len(indices), 100) - self.assertEqual(indices[-1], 99) # 采样所有帧 - - def test_no_sampling(self): - """新增：测试不采样（fps=0, num_frames=0）""" - indices = sample_frames( - frame_factor=self.frame_factor, - min_frames=self.min_frames, - max_frames=self.max_frames, - num_frames=0, - fps=0, - metadata=self.metadata, - ) - # 应返回所有帧 - self.assertEqual(len(indices), self.metadata["num_of_frame"]) - self.assertEqual(len(indices), 100) - self.assertEqual(indices[-1], 99) - np.testing.assert_array_equal(indices, np.arange(0, 100).astype(np.int32)) - - -class Test_DataProcessor(unittest.TestCase): - """ - 针对 process.py 中 DataProcessor 类的单元测试。 - """ - - def setUp(self): - - # 1. 手动启动 Patcher - patcher1 = patch(f"{MODULE_PATH}.AutoTokenizer.from_pretrained") - patcher2 = patch(f"{MODULE_PATH}.ImageProcessor.from_pretrained") - patcher_zmq_context = patch(f"{MODULE_PATH}.zmq.Context") - - self.mock_auto_tokenizer_constructor = patcher1.start() - self.mock_image_processor_constructor = patcher2.start() - self.mock_zmq_context_constructor = patcher_zmq_context.start() - - self.addCleanup(patcher1.stop) - self.addCleanup(patcher2.stop) - self.addCleanup(patcher_zmq_context.stop) - - # 2. 创建模拟对象 - self.mock_tokenizer = MagicMock() - self.mock_image_processor = MagicMock() - self.mock_zmq_context = MagicMock() - self.mock_zmq_socket = MagicMock() - - # 3. 配置 from_pretrained 和 zmq - self.mock_auto_tokenizer_constructor.return_value = self.mock_tokenizer - self.mock_image_processor_constructor.return_value = self.mock_image_processor - self.mock_zmq_context_constructor.return_value = self.mock_zmq_context - self.mock_zmq_context.socket.return_value = self.mock_zmq_socket - - # 4. 配置模拟对象的属性和方法 - self._configure_mocks() - - # 5. 实例化 DataProcessor (默认不启用 cache) - self.processor = DataProcessor(model_path="dummy_model_path") - self._configure_processor_ids() - - # 6. 准备测试用的虚拟数据 - self.dummy_image = Image.fromarray(np.uint8(np.random.rand(224, 224, 3) * 255)) - self.dummy_video_frames = np.uint8(np.random.rand(16, 224, 224, 3) * 255) - self.dummy_video_data = "path/to/dummy_video.mp4" - self.dummy_processed_image_cache = ( - np.random.rand(64, 3, 14, 14).astype(np.float32), - {"thw": (1, 8, 8), "fps": 0}, - ) - self.dummy_processed_video_cache = ( - np.random.rand(256, 3, 14, 14).astype(np.float32), - {"thw": (4, 8, 8), "fps": 30}, - ) - - def _configure_mocks(self): - def mock_convert_tokens_to_ids(tokens): - if tokens == "<|IMAGE_PLACEHOLDER|>": - return 100 - if tokens == "<|video_pad|>": - return 101 - if tokens == "<|IMAGE_START|>": - return 102 - if isinstance(tokens, list): - if tokens == ["Hello", "world"]: - return [983, 984] - if tokens == ["Prompt", "text"]: - return [606, 511] - if tokens == ["Prompt", "", "text"]: - return [606, 511] # 模拟 "Prompt text".split() - return [hash(t) % 1000 for t in tokens] - return hash(tokens) % 1000 - - self.mock_tokenizer.convert_tokens_to_ids.side_effect = mock_convert_tokens_to_ids - self.mock_tokenizer.tokenize.side_effect = lambda s: s.split() - self.mock_tokenizer.ignored_index = -100 - self.mock_tokenizer.chat_template = "dummy_template_string" - - self.mock_image_processor.merge_size = 2 - self.mock_image_processor.temporal_patch_size = 1 - - def _configure_processor_ids(self): - self.processor.image_token_id = 100 - self.processor.video_token_id = 101 - self.processor.image_patch_id = 100 - self.processor.vision_start_id = 102 - - def _get_init_outputs(self): - return { - "input_ids": [], - "token_type_ids": [], - "position_ids": [], - "images": [], - "grid_thw": [], - "image_type_ids": [], - "labels": [], - "cur_position": 0, - "video_cnt": 0, - "num_input_image_tokens": 0, - "num_input_video_tokens": 0, - "fps": [], - "mm_positions": [], - "mm_hashes": [], - "vit_seqlen": [], - "vit_position_ids": [], - } - - def test_init(self): - """测试 DataProcessor 的初始化""" - self.mock_auto_tokenizer_constructor.assert_called_with("dummy_model_path", padding_side="left", use_fast=True) - self.mock_image_processor_constructor.assert_called_with("dummy_model_path") - self.assertEqual(self.processor.image_token, "<|IMAGE_PLACEHOLDER|>") - self.assertEqual(self.processor.video_token_id, 101) - - def test_compute_text_positions(self): - """测试 _compute_text_positions 纯函数""" - pos_ids = self.processor._compute_text_positions(start_pos=5, num_tokens=3) - expected = np.array([[5, 6, 7], [5, 6, 7], [5, 6, 7]]) - np.testing.assert_array_equal(pos_ids, expected) - - def test_compute_vision_positions(self): - """测试 _compute_vision_positions 纯函数""" - pos_ids = self.processor._compute_vision_positions(start_pos=10, t=2, h=4, w=4, second_per_grid_t=1.0) - self.assertEqual(pos_ids.shape, (3, 8)) - expected_t = np.array([0, 0, 0, 0, 2, 2, 2, 2]) - expected_h = np.array([0, 0, 1, 1, 0, 0, 1, 1]) - expected_w = np.array([0, 1, 0, 1, 0, 1, 0, 1]) - expected = np.stack([expected_t, expected_h, expected_w]) + 10 - np.testing.assert_array_equal(pos_ids, expected) - - @patch(f"{MODULE_PATH}.IDS_TYPE_FLAG", {"text": 0, "image": 1, "video": 2}) - def test_add_text(self): - """测试 _add_text 辅助函数""" - outputs = self._get_init_outputs() - self.mock_tokenizer.tokenize.return_value = ["Hello", "world"] - self.mock_tokenizer.convert_tokens_to_ids.side_effect = None - self.mock_tokenizer.convert_tokens_to_ids.return_value = [10, 11] - - self.processor._add_text("Hello world", outputs) - - self.assertEqual(outputs["input_ids"], [10, 11]) - self.assertEqual(outputs["token_type_ids"], [0, 0]) - self.assertEqual(outputs["cur_position"], 2) - - @patch(f"{MODULE_PATH}.MultimodalHasher.hash_features", return_value="dummy_hash_123") - @patch(f"{MODULE_PATH}.IDS_TYPE_FLAG", {"text": 0, "image": 1, "video": 2}) - def test_add_image_autohash(self, mock_hasher): - """测试 _add_image 辅助函数 (自动哈希)""" - outputs = self._get_init_outputs() - outputs["cur_position"] = 5 - - num_patches_hw = 8 * 8 - num_tokens = 16 - mock_preprocess_return = { - "pixel_values": np.random.rand(num_patches_hw, 3, 14, 14), - "grid_thw": np.array([1, 8, 8]), - } - self.mock_image_processor.preprocess.return_value = mock_preprocess_return - - self.processor._add_image(self.dummy_image, outputs, uuid=None) - - self.assertEqual(len(outputs["input_ids"]), num_tokens) - self.assertEqual(outputs["num_input_image_tokens"], num_tokens) - mock_hasher.assert_called_once_with(mock_preprocess_return["pixel_values"]) - self.assertEqual(outputs["mm_hashes"][0], "dummy_hash_123") - self.assertEqual(outputs["cur_position"], 9) - - @patch(f"{MODULE_PATH}.MultimodalHasher.hash_features") - @patch(f"{MODULE_PATH}.IDS_TYPE_FLAG", {"text": 0, "image": 1, "video": 2}) - def test_add_video_with_uuid(self, mock_hasher): - """测试 _add_video 辅助函数 (使用 uuid)""" - outputs = self._get_init_outputs() - outputs["cur_position"] = 10 - meta = {"fps": 30} - - num_patches_total = 256 - num_tokens = 64 - - mock_preprocess_return = { - "pixel_values": np.random.rand(num_patches_total, 3, 14, 14), - "image_grid_thw": np.array([4, 8, 8]), - } - self.mock_image_processor.preprocess.return_value = mock_preprocess_return - - self.processor._add_video(self.dummy_video_frames, meta, outputs, uuid="custom_vid_uuid") - - self.assertEqual(len(outputs["input_ids"]), num_tokens) - self.assertEqual(outputs["token_type_ids"], [2] * num_tokens) - mock_hasher.assert_not_called() - self.assertEqual(outputs["mm_hashes"][0], "custom_vid_uuid") - self.assertEqual(outputs["image_type_ids"], [1, 1, 1, 1]) - - @patch.object(DataProcessor, "_add_text", MagicMock()) - @patch.object(DataProcessor, "_add_image", MagicMock()) - @patch.object(DataProcessor, "_add_video", MagicMock()) - @patch.object(DataProcessor, "_load_and_process_video") - def test_text2ids_parsing(self, mock_load_video): - """测试 text2ids 的解析和分支逻辑""" - mock_load_video.return_value = (self.dummy_video_frames, {"fps": 30}) - text = "Text1 <|IMAGE_PLACEHOLDER|> Text2 <|video_pad|> Text3" - images = [self.dummy_image] - videos = [self.dummy_video_data] - image_uuid = ["img_uuid_1"] - video_uuid = ["vid_uuid_1"] - - outputs = self.processor.text2ids(text, images, videos, image_uuid, video_uuid) - - self.processor._add_text.assert_any_call("Text1 ", outputs) - self.processor._add_image.assert_called_once_with(self.dummy_image, outputs, "img_uuid_1") - self.processor._add_video.assert_called_once_with(self.dummy_video_frames, {"fps": 30}, outputs, "vid_uuid_1") - - @patch(f"{MODULE_PATH}.parse_chat_messages") - @patch.object(DataProcessor, "text2ids", return_value="final_output") - def test_request2ids(self, mock_text2ids, mock_parse_chat): - """测试 request2ids 的 chat 模板逻辑""" - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": "Hello"}, - {"type": "image", "data": self.dummy_image, "uuid": "img1"}, - ], - } - ] - request = {"request_id": "test_0", "messages": messages, "add_generation_prompt": True} - request = Request.from_dict(request) - mock_parse_chat.return_value = messages - parsed_prompt = "User: Hello <|IMAGE_PLACEHOLDER|> Assistant:" - self.mock_tokenizer.apply_chat_template.return_value = parsed_prompt - - result = self.processor.request2ids(request) - - self.mock_tokenizer.apply_chat_template.assert_called_once() - mock_text2ids.assert_called_once_with(parsed_prompt, [self.dummy_image], [], ["img1"], []) - self.assertEqual(result, "final_output") - - @patch(f"{MODULE_PATH}.sample_frames") - @patch(f"{MODULE_PATH}.read_video_decord") - def test_load_and_process_video(self, mock_read_video, mock_sample_frames): - """测试 _load_and_process_video 的帧采样逻辑""" - mock_reader = MagicMock() - mock_reader.__getitem__.return_value.asnumpy.return_value = np.random.randint( - 0, 255, (100, 100, 3), dtype=np.uint8 - ) - mock_meta = {"num_of_frame": 100, "duration": 10.0, "fps": 10.0} - mock_read_video.return_value = (mock_reader, mock_meta, None) - mock_sample_frames.return_value = [0, 10, 20, 30, 40] - self.processor.fps = 1 - - frames, meta = self.processor._load_and_process_video("dummy_url", {"min_frames": 2, "max_frames": 10}) - - mock_sample_frames.assert_called_once_with( - frame_factor=ANY, - min_frames=2, - max_frames=10, - metadata=mock_meta, - fps=self.processor.fps, - num_frames=self.processor.target_frames, - ) - self.assertEqual(frames.shape, (5, 100, 100, 3)) - self.assertEqual(meta["fps"], 1) - - def test_init_with_external_tokenizer(self): - """新增：测试使用外部传入的 tokenizer 初始化""" - self.mock_auto_tokenizer_constructor.reset_mock() - - external_tokenizer = MagicMock() - processor = DataProcessor(model_path="dummy", tokenizer=external_tokenizer) - - self.mock_auto_tokenizer_constructor.assert_not_called() - self.assertIs(processor.tokenizer, external_tokenizer) - - def test_add_text_empty(self): - """新增：测试 _add_text 传入空字符串""" - outputs = self._get_init_outputs() - self.processor._add_text("", outputs) - self.assertEqual(outputs["input_ids"], []) - self.assertEqual(outputs["cur_position"], 0) - - @patch(f"{MODULE_PATH}.IDS_TYPE_FLAG", {"text": 0}) - def test_add_text_pre_tokenized(self): - """新增：测试 _add_text 传入已 tokenized 的 IDs""" - outputs = self._get_init_outputs() - token_ids = [10, 11, 12] - self.processor._add_text(token_ids, outputs) - - self.mock_tokenizer.tokenize.assert_not_called() - self.assertEqual(outputs["input_ids"], [10, 11, 12]) - self.assertEqual(outputs["token_type_ids"], [0, 0, 0]) - self.assertEqual(outputs["cur_position"], 3) - - @patch(f"{MODULE_PATH}.MultimodalHasher.hash_features", return_value="dummy_hash_456") - @patch(f"{MODULE_PATH}.IDS_TYPE_FLAG", {"text": 0, "image": 1, "video": 2}) - def test_add_video_no_uuid(self, mock_hasher): - """新增：测试 _add_video 在 uuid 为 None 时自动哈希""" - outputs = self._get_init_outputs() - meta = {"fps": 30} - mock_preprocess_return = { - "pixel_values": np.random.rand(256, 3, 14, 14), - "image_grid_thw": np.array([4, 8, 8]), - } - self.mock_image_processor.preprocess.return_value = mock_preprocess_return - - self.processor._add_video(self.dummy_video_frames, meta, outputs, uuid=None) - - mock_hasher.assert_called_once_with(mock_preprocess_return["pixel_values"]) - self.assertEqual(outputs["mm_hashes"][0], "dummy_hash_456") - - @patch(f"{MODULE_PATH}.IDS_TYPE_FLAG", {"text": 0, "image": 1, "video": 2}) - def test_add_processed_image(self): - """新增：测试 _add_processed_image 处理缓存数据""" - outputs = self._get_init_outputs() - outputs["cur_position"] = 3 - - self.processor._add_processed_image(self.dummy_processed_image_cache, outputs, "cached_img_uuid") - - num_tokens = 16 - self.assertEqual(len(outputs["input_ids"]), num_tokens) - self.assertEqual(outputs["input_ids"][0], self.processor.image_patch_id) - - np.testing.assert_array_equal(outputs["images"][0], self.dummy_processed_image_cache[0]) - - self.assertEqual(outputs["mm_hashes"][0], "cached_img_uuid") - self.assertEqual(outputs["cur_position"], 7) - - @patch(f"{MODULE_PATH}.IDS_TYPE_FLAG", {"text": 0, "image": 1, "video": 2}) - def test_add_processed_video(self): - """新增：测试 _add_processed_video 处理缓存数据""" - outputs = self._get_init_outputs() - outputs["cur_position"] = 5 - - self.processor._add_processed_video(self.dummy_processed_video_cache, outputs, "cached_vid_uuid") - - num_tokens = 64 - t, h, w = self.dummy_processed_video_cache[1]["thw"] - - self.assertEqual(len(outputs["input_ids"]), num_tokens) - self.assertEqual(outputs["token_type_ids"], [2] * num_tokens) - - np.testing.assert_array_equal(outputs["images"][0], self.dummy_processed_video_cache[0]) - - self.assertEqual(outputs["mm_hashes"][0], "cached_vid_uuid") - self.assertEqual(outputs["image_type_ids"], [1] * t) - self.assertGreater(outputs["cur_position"], 5) - - def test_text2ids_with_processed_data(self): - """新增：测试 text2ids 调用 _add_processed_image 和 _add_processed_video""" - with ( - patch.object(self.processor, "_add_processed_image") as mock_add_proc_img, - patch.object(self.processor, "_add_processed_video") as mock_add_proc_vid, - ): - - text = "<|IMAGE_PLACEHOLDER|><|video_pad|>" - images = [self.dummy_processed_image_cache] - videos = [self.dummy_processed_video_cache] - image_uuid = ["img1"] - video_uuid = ["vid1"] - - self.processor.text2ids(text, images, videos, image_uuid, video_uuid) - - mock_add_proc_img.assert_called_once_with(self.dummy_processed_image_cache, ANY, "img1") - mock_add_proc_vid.assert_called_once_with(self.dummy_processed_video_cache, ANY, "vid1") - - @patch(f"{MODULE_PATH}.sample_frames") - @patch(f"{MODULE_PATH}.read_video_decord") - def test_load_and_process_video_no_sampling(self, mock_read_video, mock_sample_frames): - """新增：测试 _load_and_process_video 不采样（fps=-1）""" - mock_reader = MagicMock() - mock_reader.__getitem__.return_value.asnumpy.return_value = np.random.randint( - 0, 255, (100, 100, 3), dtype=np.uint8 - ) - mock_meta = {"num_of_frame": 10, "duration": 1.0, "fps": 10.0} - mock_read_video.return_value = (mock_reader, mock_meta, None) - - self.processor.fps = -1 - self.processor.target_frames = -1 - - frames, meta = self.processor._load_and_process_video("dummy_url", {}) - - mock_sample_frames.assert_not_called() - self.assertEqual(frames.shape, (10, 100, 100, 3)) - self.assertEqual(meta["num_of_frame"], 10) - - def test_get_processor_cache(self): - """新增：测试 get_processor_cache (zmq)""" - hashes = ["hash1", "hash2"] - expected_items = ["item1", "item2"] - mock_resp = pickle.dumps(expected_items) - self.mock_zmq_socket.recv_multipart.return_value = (b"", mock_resp) - - items = self.processor.get_processor_cache(self.mock_zmq_socket, hashes) - - self.mock_zmq_socket.send_multipart.assert_called_once_with([b"", pickle.dumps(hashes)]) - self.assertEqual(items, expected_items) - - def test_update_processor_cache(self): - """新增：测试 update_processor_cache (zmq)""" - hashes = ["hash1"] - items = ["item1"] - - self.processor.update_processor_cache(self.mock_zmq_socket, hashes, items) - - expected_req = pickle.dumps((hashes, items)) - self.mock_zmq_socket.send_multipart.assert_called_once_with([b"", expected_req]) - - def test_apply_chat_template(self): - """新增：测试 apply_chat_template 核心逻辑""" - request = {"messages": ["msg1"], "add_generation_prompt": True, "request_id": "req123"} - self.mock_tokenizer.apply_chat_template.return_value = "Prompt <|IMAGE_PLACEHOLDER|> text" - self.mock_tokenizer.tokenize.return_value = ["Prompt", "text"] - - self.mock_tokenizer.convert_tokens_to_ids.side_effect = None - self.mock_tokenizer.convert_tokens_to_ids.return_value = [10, 11] - - token_ids = self.processor.apply_chat_template(request) - - self.assertEqual(token_ids, [10, 11]) - self.assertEqual(request["text_after_process"], "Prompt <|IMAGE_PLACEHOLDER|> text") - - self.mock_tokenizer.tokenize.assert_called_with("Prompt text") - - def test_apply_chat_template_raises_error(self): - """新增：测试 apply_chat_template 在模板不存在时引发 ValueError""" - self.mock_tokenizer.chat_template = None - with self.assertRaises(ValueError) as context: - self.processor.apply_chat_template({"messages": []}) - self.assertIn("does not support chat_template", str(context.exception)) - - @patch(f"{MODULE_PATH}.parse_chat_messages") - def test_request2ids_cache_miss_raises_error(self, mock_parse_chat): - """新增：测试 request2ids 在缓存关闭时缺少数据引发 ValueError""" - messages = [{"role": "user", "content": [{"type": "image", "uuid": "img1"}]}] - request = {"request_id": "test_0", "messages": messages} - request = Request.from_dict(request) - - mock_parse_chat.return_value = messages - - with self.assertRaises(ValueError) as context: - self.processor.request2ids(request) - - self.assertIn("Missing items cannot be retrieved without processor cache.", str(context.exception)) - - @patch(f"{MODULE_PATH}.DataProcessor.get_processor_cache") - @patch(f"{MODULE_PATH}.DataProcessor.update_processor_cache") - @patch(f"{MODULE_PATH}.DataProcessor.text2ids") - @patch(f"{MODULE_PATH}.parse_chat_messages") - def test_request2ids_cache_hit_and_update(self, mock_parse_chat, mock_text2ids, mock_update_cache, mock_get_cache): - """新增：测试 request2ids 缓存命中和缓存更新""" - self.processor = DataProcessor(model_path="dummy_model_path", enable_processor_cache=True) - self._configure_processor_ids() - - messages = [ - { - "role": "user", - "content": [ - {"type": "image", "uuid": "img_cache_hit"}, - {"type": "image", "data": self.dummy_image, "uuid": "img_to_update"}, - ], - } - ] - request = {"request_id": "test_0", "messages": messages} - request = Request.from_dict(request) - - mock_parse_chat.return_value = messages - mock_get_cache.return_value = [self.dummy_processed_image_cache] - - mock_text2ids_output = { - "grid_thw": [(1, 8, 8), (1, 8, 8)], - "fps": [0, 0], - "mm_hashes": ["img_cache_hit", "img_to_update"], - "images": [self.dummy_processed_image_cache[0], self.dummy_processed_image_cache[0]], - } - mock_text2ids.return_value = mock_text2ids_output - self.mock_tokenizer.apply_chat_template.return_value = "<|IMAGE_PLACEHOLDER|><|IMAGE_PLACEHOLDER|>" - - self.processor.request2ids(request) - - self.mock_zmq_context.socket.assert_called_with(zmq.DEALER) - mock_get_cache.assert_called_once_with(self.mock_zmq_socket, ["img_cache_hit"]) - - parsed_images = mock_text2ids.call_args[0][1] - self.assertIs(parsed_images[0], self.dummy_processed_image_cache) - self.assertIs(parsed_images[1], self.dummy_image) - - expected_hash_to_cache = ["img_to_update"] - expected_item_to_cache = (self.dummy_processed_image_cache[0], {"thw": (1, 8, 8), "fps": 0}) - mock_update_cache.assert_called_once() - self.assertEqual(mock_update_cache.call_args[0][1], expected_hash_to_cache) - self.assertEqual(mock_update_cache.call_args[0][2][0][1], expected_item_to_cache[1]) - np.testing.assert_array_equal(mock_update_cache.call_args[0][2][0][0], expected_item_to_cache[0]) - - @patch(f"{MODULE_PATH}.DataProcessor.text2ids") - @patch(f"{MODULE_PATH}.parse_chat_messages") - def test_request2ids_unsupported_type(self, mock_parse_chat, mock_text2ids): - """新增：测试 request2ids 静默忽略不支持的类型""" - messages = [ - { - "role": "user", - "content": [{"type": "text", "text": "Hello"}, {"type": "audio", "data": "...", "uuid": "audio1"}], - } - ] - request = {"request_id": "test_0", "messages": messages} - request = Request.from_dict(request) - - mock_parse_chat.return_value = messages - self.mock_tokenizer.apply_chat_template.return_value = "User: Hello " - - self.processor.request2ids(request) - - mock_text2ids.assert_called_once() - call_args = mock_text2ids.call_args[0] - self.assertEqual(call_args[1], []) # images - self.assertEqual(call_args[2], []) # videos - self.assertEqual(call_args[3], []) # image_uuid - self.assertEqual(call_args[4], []) # video_uuid - - -class TestPaddleOCR_VL_ImageProcessor(unittest.TestCase): - def setUp(self): - # 初始化默认参数 - self.default_params = { - "do_resize": True, - "resample": 3, - "do_rescale": True, - "rescale_factor": 1 / 255, - "do_normalize": True, - "image_mean": [0.48145466, 0.4578275, 0.40821073], - "image_std": [0.26862954, 0.26130258, 0.27577711], - "do_convert_rgb": True, - "min_pixels": 28 * 28 * 130, - "max_pixels": 28 * 28 * 1280, - "patch_size": 14, - "temporal_patch_size": 1, - "merge_size": 2, - } - - # 创建测试图像 - self.test_image = Image.fromarray(np.random.randint(0, 255, (224, 224, 3), dtype=np.uint8)) - - def test_initialization(self): - """测试初始化参数是否正确设置""" - processor = ImageProcessor(**self.default_params) - - for param, value in self.default_params.items(): - self.assertEqual(getattr(processor, param), value) - - def test_smart_resize(self): - """测试智能调整图像大小功能""" - # 测试正常尺寸调整 - h, w = smart_resize(224, 224, factor=28) - self.assertEqual(h % 28, 0) - self.assertEqual(w % 28, 0) - - # 测试小尺寸调整 - h, w = smart_resize(20, 20, factor=28) - self.assertGreaterEqual(h, 28) - self.assertGreaterEqual(w, 28) - - # 测试超大尺寸调整 - h, w = smart_resize(2000, 2000, factor=28) - self.assertLess(h * w, 28 * 28 * 1280) - - def test_preprocess_single_image(self): - """测试单张图像预处理流程""" - processor = ImageProcessor(**self.default_params) - - # 测试正常预处理 - result = processor.preprocess(self.test_image) - self.assertIn("pixel_values", result) - self.assertIn("grid_thw", result) - self.assertEqual(result["pixel_values"].ndim, 4) # [N, C, H, W] - - # 测试关闭某些预处理步骤 - result = processor.preprocess(self.test_image, do_resize=False, do_normalize=False) - self.assertIn("pixel_values", result) - - def test_preprocess_batch_images(self): - """测试批量图像预处理""" - processor = ImageProcessor(**self.default_params) - batch_images = [self.test_image, self.test_image] - - result = processor.preprocess(batch_images) - expected_shape = 1152 - self.assertEqual(result["pixel_values"].shape[0], expected_shape) - - def test_invalid_input(self): - """测试无效输入处理""" - processor = ImageProcessor(**self.default_params) - - # 测试无效图像 - with self.assertRaises(ValueError): - processor.preprocess("invalid_image") - - # 测试视频输入(暂不支持) - with self.assertRaises(NotImplementedError): - processor.preprocess(self.test_image, videos=["video"]) - - def test_from_pretrained(self): - """测试从预训练模型加载配置""" - with patch("builtins.open", unittest.mock.mock_open(read_data='{"do_resize": false}')) as mock_file: - processor = ImageProcessor.from_pretrained("dummy_path") - self.assertFalse(processor.do_resize) - mock_file.assert_called_once() - - -class TestPaddleOCRVLProcessor(unittest.TestCase): - def setUp(self): - # 创建 PaddleOCRVLProcessor 实例的模拟对象 - with patch.object(PaddleOCRVLProcessor, "__init__", return_value=None): - self.processor = PaddleOCRVLProcessor("model_path") - - # 设置必要的属性 - self.processor.tokenizer = MagicMock() - self.processor.tokenizer.eos_token_id = 1 - self.processor.processor = MagicMock() - self.processor.limit_mm_per_prompt = {"image": 1, "video": 1, "audio": 1} - self.processor.eos_token_ids = [1] - self.processor.reasoning_parser = None - self.processor.model_status_dict = {} - - # 模拟 _apply_default_parameters - def mock_apply_default_parameters(request_or_dict): - if isinstance(request_or_dict, dict): - if "top_p" not in request_or_dict: - request_or_dict["top_p"] = 0.9 - return request_or_dict - - if not hasattr(request_or_dict, "top_p"): - request_or_dict.top_p = 0.9 - return request_or_dict - - self.processor._apply_default_parameters = mock_apply_default_parameters - - # 模拟 pack_outputs - def mock_pack_outputs(outputs): - # 简化 position_ids 的处理 - position_ids_list = outputs["position_ids"] - if not position_ids_list: - position_ids = np.array([], dtype=np.int64) - elif isinstance(position_ids_list[0], list): - position_ids = np.array(position_ids_list, dtype=np.int64) - else: - position_ids = np.concatenate(position_ids_list, axis=1, dtype=np.int64) - - if position_ids.ndim == 1: - position_ids = position_ids.reshape(1, -1) - - # 源码的 pack_outputs 会 transpose - position_ids = position_ids.transpose(1, 0) - - return { - "input_ids": np.array(outputs["input_ids"], dtype=np.int64), - "token_type_ids": np.array(outputs["token_type_ids"], dtype=np.int64), - "position_ids": position_ids, - "images": np.vstack(outputs["images"]) if outputs.get("images") else None, - "grid_thw": np.vstack(outputs["grid_thw"]) if outputs.get("grid_thw") else None, - "image_type_ids": np.array(outputs["image_type_ids"]) if outputs.get("image_type_ids") else None, - } - - self.processor.pack_outputs = mock_pack_outputs - self.processor.np = np - - # 模拟 _SAMPLING_EPS 常量 - self.processor._SAMPLING_EPS = 1e-5 - - # 模拟 processor 返回 (position_ids 必须是 2D array 的 list) - self.processor.processor.text2ids.return_value = { - "input_ids": [1, 2, 3], - "token_type_ids": [0, 0, 0], - "position_ids": [np.array([[0, 1, 2]], dtype=np.int64)], # 修正 - "images": ["image_feature"], - "grid_thw": ["grid_feature"], - "image_type_ids": [0], - "cur_position": 3, - } - - self.processor.processor.request2ids.return_value = { - "input_ids": [1, 2, 3], - "token_type_ids": [0, 0, 0], - "position_ids": [np.array([[0, 1, 2]], dtype=np.int64)], # 修正 - "images": ["image_feature"], - "grid_thw": ["grid_feature"], - "image_type_ids": [0], - "cur_position": 3, - } - - # 模拟 _compute_text_positions 方法 (返回 2D array) - self.processor.processor._compute_text_positions = lambda pos, num: np.array( - [list(range(pos, pos + num))], dtype=np.int64 - ) - - # 模拟 update_stop_seq - self.processor.update_stop_seq = MagicMock(return_value=([[99, 98]], [2])) - - # 模拟 pack_outputs 需要的属性 - self.processor.processor.image_token_id = 100 - self.processor.processor.video_token_id = 101 - - def test_process_request_dict_basic(self): - """测试基本请求处理功能""" - request = { - "request_id": "test_request", - "prompt": "test prompt", - "multimodal_data": {"image": ["image1"]}, - "metadata": {"generated_token_ids": []}, - } - request = Request.from_dict(request) - - result = self.processor.process_request_dict(request, max_model_len=512) - self.assertEqual(result.prompt_token_ids, [1, 2, 3]) - self.assertEqual(result.prompt_token_ids_len, 3) - self.assertTrue(hasattr(result, "multimodal_inputs")) - - def test_process_request_dict_with_messages(self): - """测试 messages 格式的请求处理""" - request = { - "request_id": "test_0", - "messages": [ - { - "role": "user", - "content": [{"type": "text", "text": "Hello"}, {"type": "image_url", "url": "image1"}], - } - ], - "metadata": {"generated_token_ids": []}, - } - request = Request.from_dict(request) - - result = self.processor.process_request_dict(request, max_model_len=512) - self.assertEqual(result.prompt_token_ids, [1, 2, 3]) - self.assertTrue(hasattr(result, "multimodal_inputs")) - - def test_process_request_dict_with_max_len(self): - """测试最大长度限制功能""" - request = { - "request_id": "test_0", - "prompt": "test prompt", - "multimodal_data": {"image": ["image1"]}, - "metadata": {"generated_token_ids": []}, - } - request = Request.from_dict(request) - - # 模拟 processor 返回长序列 - self.processor.processor.text2ids.return_value = { - "input_ids": list(range(100)), - "token_type_ids": [0] * 100, - "position_ids": [np.array([list(range(100))], dtype=np.int64)], - "images": ["image_feature"], - "grid_thw": ["grid_feature"], - "image_type_ids": [0], - "cur_position": 100, - } - - max_model_len = 50 - result = self.processor.process_request_dict(request, max_model_len) - # 验证是否截断到 max_model_len - 1 - self.assertEqual(len(result.prompt_token_ids), max_model_len - 1) - self.assertEqual(result.prompt_token_ids, list(range(49))) - # 验证原始输入长度确实超过了限制 - self.assertGreater(len(self.processor.processor.text2ids.return_value["input_ids"]), max_model_len) - - def test_parse_processor_kwargs(self): - """测试处理器参数解析""" - valid_kwargs = {"video_max_frames": 10, "video_min_frames": 1} - result = self.processor._parse_processor_kwargs(valid_kwargs) - self.assertEqual(result, valid_kwargs) - - # 测试无效参数 - invalid_kwargs = {"video_max_frames": "invalid"} - with patch( - "fastdeploy.input.v1.paddleocr_vl_processor.paddleocr_vl_processor.data_processor_logger" - ) as mock_logger: - result = self.processor._parse_processor_kwargs(invalid_kwargs) - self.assertEqual(result, {}) - # 确认警告已被记录 - mock_logger.warning.assert_called() - - def test_parse_limits(self): - """测试输入限制解析""" - custom_limits = {"image": 2, "video": 3} - result = self.processor._parse_limits(custom_limits) - self.assertEqual(result["image"], 2) - self.assertEqual(result["video"], 3) - self.assertEqual(result["audio"], 1) # 默认值 - - def test_check_mm_limits(self): - """测试多模态输入限制检查 (dict path)""" - # 测试不超限 - item = {"image": ["image1"], "video": ["video1"]} - self.processor._check_mm_limits(item) - - # 测试超限 - item_exceeded = {"image": ["image1", "image2"], "video": ["video1"]} - with self.assertRaises(ValueError): - self.processor._check_mm_limits(item_exceeded) - - def test_parse_processor_kwargs_invalid_type(self): - """测试 _parse_processor_kwargs 传入非字典类型""" - invalid_input = ["video_max_frames", 10] - with patch( - "fastdeploy.input.v1.paddleocr_vl_processor.paddleocr_vl_processor.data_processor_logger" - ) as mock_logger: - result = self.processor._parse_processor_kwargs(invalid_input) - self.assertEqual(result, {}) # 触发 - mock_logger.warning.assert_called() - - def test_parse_limits_invalid_type(self): - """测试 _parse_limits 传入非字典类型""" - invalid_input = ["image", 2] - with patch( - "fastdeploy.input.v1.paddleocr_vl_processor.paddleocr_vl_processor.data_processor_logger" - ) as mock_logger: - result = self.processor._parse_limits(invalid_input) - # 应返回默认值 - self.assertEqual(result, {"image": 1, "video": 1, "audio": 1}) - mock_logger.warning.assert_called() - - def test_check_mm_limits_messages_path(self): - """测试 _check_mm_limits (messages path)""" - messages = [ - {"role": "user", "content": [{"type": "text", "text": "Hello"}, {"type": "image_url", "url": "image1"}]} - ] - self.processor._check_mm_limits(messages) # 不应抛出异常 - - def test_check_mm_limits_messages_exceeded(self): - """测试 _check_mm_limits (messages path) 超限""" - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": "Hello"}, - {"type": "image_url", "url": "image1"}, - {"type": "image_url", "url": "image2"}, # 超过限制 1 - ], - } - ] - with self.assertRaises(ValueError): - self.processor._check_mm_limits(messages) - - def test_process_request_dict_no_prompt_or_messages(self): - """测试当请求既没有 prompt 也没有 messages 时抛出异常""" - request = {"request_id": "test_0", "metadata": {"generated_token_ids": []}} - request = Request.from_dict(request) - with self.assertRaises(ValueError): - self.processor.process_request_dict(request, max_model_len=512) - - def test_process_request_dict_with_continuation(self): - """测试续写逻辑 (metadata 包含 generated_token_ids)""" - request = { - "request_id": "test_0", - "prompt": "test prompt", - "multimodal_data": {"image": ["image1"]}, - "metadata": {"generated_token_ids": [10, 11, 12]}, # 已生成的 token - } - request = Request.from_dict(request) - setattr(request, "metadata", {"generated_token_ids": [10, 11, 12]}) - - result = self.processor.process_request_dict(request, max_model_len=512) - self.assertEqual(result.prompt_token_ids, [1, 2, 3, 10, 11, 12]) - self.assertEqual(result.prompt_token_ids_len, 6) - - def test_process_request_dict_with_stop_sequences(self): - """测试 stop_sequences 处理""" - request = { - "request_id": "test_0", - "prompt": "test prompt", - "stop": ["stop1", "stop2"], - "metadata": {"generated_token_ids": []}, - } - request = Request.from_dict(request) - result = self.processor.process_request_dict(request, max_model_len=512) - - # 验证 update_stop_seq 被调用 - self.processor.update_stop_seq.assert_called_with(["stop1", "stop2"]) - # 验证结果被设置到 request 中 - self.assertEqual(result.sampling_params.stop_token_ids, [[99, 98]]) - self.assertEqual(result.sampling_params.stop_seqs_len, [2]) - - def test_process_request_dictefault_max_tokens(self): - """测试默认 max_tokens 计算""" - request = { - "request_id": "test_0", - "prompt": "test prompt", - "metadata": {"generated_token_ids": []}, - } # 长度为 3 - request = Request.from_dict(request) - max_model_len = 10 - result = self.processor.process_request_dict(request, max_model_len) - - self.assertEqual(result.sampling_params.max_tokens, 7) - - def test_process_request_dict_top_p_clamping(self): - """测试 top_p 值被修正 (clamping)""" - request = { - "request_id": "test_0", - "prompt": "test prompt", - "top_p": 0.0, # 低于 _SAMPLING_EPS - "metadata": {"generated_token_ids": []}, - } - request = Request.from_dict(request) - result = self.processor.process_request_dict(request, max_model_len=512) - self.assertEqual(result.sampling_params.top_p, self.processor._SAMPLING_EPS) - - def test_append_generated_tokens(self): - """直接测试 append_generated_tokens 辅助函数""" - # : position_ids 必须是 [2D array] - multimodal_inputs = { - "input_ids": [1, 2, 3], - "token_type_ids": [0, 0, 0], - "position_ids": [np.array([[0, 1, 2]], dtype=np.int64)], - "cur_position": 3, - } - generated_token_ids = [10, 11] - - # 调用 append_generated_tokens (它是 PaddleOCRVLProcessor 的方法) - PaddleOCRVLProcessor.append_generated_tokens(self.processor, multimodal_inputs, generated_token_ids) - - self.assertEqual(multimodal_inputs["input_ids"], [1, 2, 3, 10, 11]) - self.assertEqual(multimodal_inputs["token_type_ids"], [0, 0, 0, 0, 0]) - # : 检查 position_ids 是否为 [np.array(...), np.array(...)] - self.assertEqual(len(multimodal_inputs["position_ids"]), 2) - self.assertTrue(np.array_equal(multimodal_inputs["position_ids"][0], np.array([[0, 1, 2]], dtype=np.int64))) - self.assertTrue(np.array_equal(multimodal_inputs["position_ids"][1], np.array([[3, 4]], dtype=np.int64))) - self.assertEqual(multimodal_inputs["cur_position"], 5) - - def test_pack_outputs_real_no_images(self): - """测试真实的 pack_outputs 方法 (无图像)""" - outputs = { - "input_ids": [1, 2, 3], - "token_type_ids": [0, 0, 0], - # : position_ids 必须是 [2D array] - "position_ids": [np.array([[0, 1, 2]], dtype=np.int64)], - "images": [], # 空列表 - "grid_thw": [], - "image_type_ids": [], - "cur_position": 3, - } - - # 调用真实的类方法，而不是 setUp 中 mock 的实例方法 - result = PaddleOCRVLProcessor.pack_outputs(self.processor, outputs) - - self.assertIsNone(result["images"]) - self.assertIsNone(result["grid_thw"]) - self.assertIsNone(result["image_type_ids"]) - self.assertTrue(np.array_equal(result["input_ids"], np.array([1, 2, 3], dtype=np.int64))) - # 验证 position_ids 被 concatenate 和 transpose - # input: [array([[0, 1, 2]])] -> concat: array([[0, 1, 2]]) (shape 1,3) -> transpose: array([[0], [1], [2]]) (shape 3,1) - self.assertTrue(np.array_equal(result["position_ids"], np.array([[0], [1], [2]], dtype=np.int64))) - self.assertEqual(result["image_patch_id"], 100) - self.assertEqual(result["video_patch_id"], 101) - - def test_pack_outputs_real_with_images(self): - """测试真实的 pack_outputs 方法 (有图像)""" - image_feature = np.array([[0.1, 0.2]]) - grid_feature = np.array([[1, 2, 3]]) - - outputs = { - "input_ids": [1, 2, 3], - "token_type_ids": [0, 0, 0], - # : position_ids 必须是 [2D array] - "position_ids": [np.array([[0, 1, 2]], dtype=np.int64)], - "images": [image_feature], - "grid_thw": [grid_feature], - "image_type_ids": [0], - "cur_position": 3, - } - - result = PaddleOCRVLProcessor.pack_outputs(self.processor, outputs) - - self.assertTrue(np.array_equal(result["images"], image_feature)) - self.assertTrue(np.array_equal(result["grid_thw"], grid_feature)) - self.assertTrue(np.array_equal(result["image_type_ids"], np.array([0]))) - self.assertTrue(np.array_equal(result["position_ids"], np.array([[0], [1], [2]], dtype=np.int64))) - - def test_think_status(self): - """测试 思考机制""" - request = { - "prompt": "hello", - "request_id": "test_1", - "prompt_token_ids": [1, 2, 3], - } - request = Request.from_dict(request) - self.processor.reasoning_parser = MagicMock() - self.processor.reasoning_parser.get_model_status.return_value = "think_start" - self.processor.model_status_dict = {} - self.processor.process_request_dict(request, max_model_len=512) - self.assertEqual(request.enable_thinking, True) - - request = { - "prompt": "hello", - "request_id": "test_2", - "prompt_token_ids": [1, 2, 3], - } - request = Request.from_dict(request) - self.processor.process_request_dict(request, max_model_len=512) - self.assertEqual(request.enable_thinking, True) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/input/v1/test_process_video.py b/tests/input/v1/test_process_video.py deleted file mode 100644 index b8777cb0a76..00000000000 --- a/tests/input/v1/test_process_video.py +++ /dev/null @@ -1,386 +0,0 @@ -""" -# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -import io -import math -import os -import tempfile -import unittest -from unittest.mock import patch - -import numpy as np -from PIL import Image as PILImage - -import fastdeploy.input.v1.ernie4_5_vl_processor.process_video as process_video_module -from fastdeploy.input.v1.ernie4_5_vl_processor.process_video import ( - get_frame_indices, - read_frames_decord, - read_video_decord, -) - - -class _MockFrame: - """Lightweight frame wrapper that mimics the real frame object.""" - - def __init__(self, arr): - self._arr = arr - - def asnumpy(self): - """Return the underlying numpy array.""" - return self._arr - - -class MockVideoReaderWrapper: - """ - Simple mock implementation of a video reader: - - - __len__ returns the total number of frames - - __getitem__ returns a _MockFrame(arr) - - get_avg_fps() returns fps - - Specific indices can be configured to raise errors in __getitem__ - """ - - def __init__( - self, - src, - num_threads=1, - vlen=12, - fps=6, - fail_indices=None, - h=4, - w=5, - c=3, - ): - self.src = src - self._vlen = vlen - self._fps = fps - self._fail = set(fail_indices or []) - self._h, self._w, self._c = h, w, c - - def __len__(self): - return self._vlen - - def get_avg_fps(self): - return self._fps - - def __getitem__(self, idx): - if idx < 0 or idx >= self._vlen: - raise IndexError("index out of range") - if idx in self._fail: - raise ValueError(f"forced fail at {idx}") - # Create a frame whose pixel value encodes the index (for easy debugging) - arr = np.zeros((self._h, self._w, self._c), dtype=np.uint8) - arr[:] = idx % 255 - return _MockFrame(arr) - - -class TestReadVideoDecord(unittest.TestCase): - def test_read_video_decord_with_wrapper(self): - """Test passing an existing VideoReaderWrapper instance directly.""" - # Patch VideoReaderWrapper in the target module so isinstance checks use our mock class - with patch.object(process_video_module, "VideoReaderWrapper", MockVideoReaderWrapper): - mock_reader = MockVideoReaderWrapper("dummy", vlen=10, fps=5) - reader, meta, path = read_video_decord(mock_reader, save_to_disk=False) - - self.assertIs(reader, mock_reader) - self.assertEqual(meta["fps"], 5) - self.assertEqual(meta["num_of_frame"], 10) - self.assertTrue(math.isclose(meta["duration"], 10 / 5, rel_tol=1e-6)) - # The original reader object should be returned unchanged - self.assertIs(path, mock_reader) - - def test_read_video_decord_with_bytes(self): - """Test that bytes input is wrapped into BytesIO and passed to VideoReaderWrapper.""" - with patch.object(process_video_module, "VideoReaderWrapper", MockVideoReaderWrapper): - data = b"\x00\x01\x02\x03" - reader, meta, path = read_video_decord(data, save_to_disk=False) - - self.assertIsInstance(reader, MockVideoReaderWrapper) - self.assertEqual(meta["fps"], 6) - self.assertEqual(meta["num_of_frame"], 12) - self.assertTrue(math.isclose(meta["duration"], 12 / 6, rel_tol=1e-6)) - self.assertIsInstance(path, io.BytesIO) - - -class TestGetFrameIndices(unittest.TestCase): - def test_by_target_frames_middle(self): - """Test target_frames mode with 'middle' sampling strategy.""" - vlen = 12 - out = get_frame_indices( - vlen=vlen, - target_frames=4, - target_fps=-1, - frames_sample="middle", - input_fps=-1, - ) - # 12 frames split into 4 segments -> midpoints [1, 4, 7, 10] - self.assertEqual(out, [1, 4, 7, 10]) - - def test_by_target_frames_leading(self): - """Test target_frames mode with 'leading' sampling strategy.""" - vlen = 10 - out = get_frame_indices( - vlen=vlen, - target_frames=5, - target_fps=-1, - frames_sample="leading", - input_fps=-1, - ) - # 10 frames split into 5 segments -> segment starts [0, 2, 4, 6, 8] - self.assertEqual(out, [0, 2, 4, 6, 8]) - - def test_by_target_frames_rand(self): - """Test target_frames mode with 'rand' sampling strategy.""" - vlen = 10 - out = get_frame_indices( - vlen=vlen, - target_frames=4, - target_fps=-1, - frames_sample="rand", - input_fps=-1, - ) - self.assertEqual(len(out), 4) - self.assertTrue(all(0 <= i < vlen for i in out)) - - def test_by_target_frames_fix_start(self): - """Test target_frames mode with a fixed start offset.""" - vlen = 10 - out = get_frame_indices( - vlen=vlen, - target_frames=5, - target_fps=-1, - frames_sample="middle", # overridden by fix_start - fix_start=1, - input_fps=-1, - ) - # Segment starts [0, 2, 4, 6, 8] -> +1 => [1, 3, 5, 7, 9] - self.assertEqual(out, [1, 3, 5, 7, 9]) - - def test_target_frames_greater_than_vlen(self): - """Test that target_frames > vlen falls back to using vlen samples.""" - vlen = 5 - out = get_frame_indices( - vlen=vlen, - target_frames=10, - target_fps=-1, - frames_sample="middle", - input_fps=-1, - ) - self.assertEqual(len(out), vlen) - self.assertTrue(all(0 <= i < vlen for i in out)) - - def test_by_target_fps_middle(self): - """Test target_fps mode with 'middle' sampling strategy.""" - vlen, in_fps = 12, 6 - out = get_frame_indices( - vlen=vlen, - target_frames=-1, - target_fps=2, - frames_sample="middle", - input_fps=in_fps, - ) - # Roughly 4 frames expected - self.assertTrue(3 <= len(out) <= 5) - self.assertTrue(all(0 <= i < vlen for i in out)) - - def test_by_target_fps_leading(self): - """Test target_fps mode with 'leading' sampling strategy.""" - vlen, in_fps = 12, 6 - out = get_frame_indices( - vlen=vlen, - target_frames=-1, - target_fps=2, - frames_sample="leading", - input_fps=in_fps, - ) - self.assertTrue(3 <= len(out) <= 5) - self.assertTrue(all(0 <= i < vlen for i in out)) - - def test_by_target_fps_rand(self): - """Test target_fps mode with 'rand' sampling strategy.""" - vlen, in_fps = 12, 6 - out = get_frame_indices( - vlen=vlen, - target_frames=-1, - target_fps=2, - frames_sample="rand", - input_fps=in_fps, - ) - self.assertTrue(3 <= len(out) <= 5) - self.assertTrue(all(0 <= i < vlen for i in out)) - - def test_invalid_both_negative(self): - """Test that both target_frames and target_fps being negative raises ValueError.""" - with self.assertRaises(ValueError): - get_frame_indices( - vlen=10, - target_frames=-1, - target_fps=-1, - frames_sample="middle", - ) - - def test_invalid_both_specified(self): - """Test that specifying both target_frames and target_fps raises AssertionError.""" - with self.assertRaises(AssertionError): - get_frame_indices( - vlen=10, - target_frames=4, - target_fps=2, - frames_sample="middle", - input_fps=6, - ) - - def test_invalid_target_fps_missing_input(self): - """Test that target_fps > 0 with invalid input_fps raises AssertionError.""" - with self.assertRaises(AssertionError): - get_frame_indices( - vlen=10, - target_frames=-1, - target_fps=2, - frames_sample="middle", - input_fps=-1, - ) - - -class TestReadFramesDecord(unittest.TestCase): - def test_basic_read_no_save(self): - """Test normal frame reading without saving to disk.""" - reader = MockVideoReaderWrapper("dummy", vlen=8, fps=4) - meta = {"fps": 4, "duration": 8 / 4, "num_of_frame": 8} - - ret, idxs, ts = read_frames_decord( - video_path="dummy", - video_reader=reader, - video_meta=meta, - target_frames=4, - frames_sample="middle", - save_to_disk=False, - ) - - # Should return 4 PIL.Image instances - self.assertEqual(len(ret), 4) - for img in ret: - self.assertIsInstance(img, PILImage.Image) - - self.assertEqual(idxs, [0, 2, 4, 6]) - dur = meta["duration"] - n = meta["num_of_frame"] - for i, t in zip(idxs, ts): - self.assertTrue(math.isclose(t, i * dur / n, rel_tol=1e-6)) - - def test_read_and_save_to_disk(self): - """Test reading frames and saving them as PNG files on disk.""" - reader = MockVideoReaderWrapper("dummy", vlen=4, fps=2) - meta = {"fps": 2, "duration": 4 / 2, "num_of_frame": 4} - - with ( - tempfile.TemporaryDirectory() as tmpdir, - patch.object( - process_video_module, - "get_filename", - return_value="det_id", - ), - ): - ret, idxs, ts = read_frames_decord( - video_path="dummy", - video_reader=reader, - video_meta=meta, - target_frames=2, - frames_sample="leading", - save_to_disk=True, - cache_dir=tmpdir, - ) - - self.assertEqual(len(ret), 2) - for i, pth in enumerate(ret): - self.assertIsInstance(pth, str) - self.assertTrue(os.path.exists(pth)) - self.assertEqual(os.path.basename(pth), f"{i}.png") - - def test_fallback_previous_success(self): - """Test that a failed frame read falls back to a previous valid frame when possible.""" - reader = MockVideoReaderWrapper("dummy", vlen=10, fps=5, fail_indices={3}) - meta = {"fps": 5, "duration": 10 / 5, "num_of_frame": 10} - idxs = [1, 2, 3, 6] - - ret, new_idxs, ts = read_frames_decord( - video_path="dummy", - video_reader=reader, - video_meta=meta, - frame_indices=idxs.copy(), - save_to_disk=False, - tol=5, - ) - - # Index 3 fails and should be replaced by 2 or 4 (previous/next search) - self.assertIn(new_idxs[2], (2, 4)) - self.assertEqual(len(ret), 4) - - def test_fallback_next_when_prev_fails(self): - """Test that when current and previous frames fail, a later frame is used as fallback.""" - reader = MockVideoReaderWrapper("dummy", vlen=10, fps=5, fail_indices={2, 3}) - meta = {"fps": 5, "duration": 10 / 5, "num_of_frame": 10} - idxs = [1, 2, 3, 6] - - ret, new_idxs, ts = read_frames_decord( - video_path="dummy", - video_reader=reader, - video_meta=meta, - frame_indices=idxs.copy(), - save_to_disk=False, - tol=5, - ) - - # Frame 3 should eventually be replaced by 4 - self.assertEqual(new_idxs[2], 4) - self.assertEqual(len(ret), 4) - - def test_len_assert_when_no_fallback(self): - """Test that assertion is triggered when no valid fallback frame can be found.""" - - class FailAllAroundReader(MockVideoReaderWrapper): - """Reader that fails on index 1 and has too small length to find fallback.""" - - def __init__(self, *a, **kw): - super().__init__(*a, **kw) - self._vlen = 2 - self._fps = 2 - self._fail = {1} - - def __getitem__(self, idx): - if idx in self._fail: - raise ValueError("fail hard") - return super().__getitem__(idx) - - reader = FailAllAroundReader("dummy") - meta = {"fps": 2, "duration": 2 / 2, "num_of_frame": 2} - - # Request 2 frames: index 0 succeeds, index 1 always fails, - # and tol=0 disallows searching neighbors -> stack and length assertion should fail - with self.assertRaises(AssertionError): - read_frames_decord( - video_path="dummy", - video_reader=reader, - video_meta=meta, - target_frames=2, - frames_sample="leading", - save_to_disk=False, - tol=0, - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/input/v1/test_qwen3_vl_processor.py b/tests/input/v1/test_qwen3_vl_processor.py deleted file mode 100644 index c858f1e51db..00000000000 --- a/tests/input/v1/test_qwen3_vl_processor.py +++ /dev/null @@ -1,1172 +0,0 @@ -""" -# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -import copy -import unittest -from unittest.mock import MagicMock, patch - -import numpy as np -from PIL import Image - -from fastdeploy.engine.request import Request -from fastdeploy.input.v1.qwen3_vl_processor import Qwen3VLProcessor -from fastdeploy.input.v1.qwen3_vl_processor.process import sample_frames - - -def mock_pil_image(height, width): - """ - Generate mock random RGB image - - Args: - height: Image height in pixels - width: Image width in pixels - - Returns: - PIL.Image object with random RGB data - """ - rgb_image = np.random.randint(0, 256, (height, width, 3), dtype=np.uint8) - return Image.fromarray(rgb_image) - - -def mock_read_frames(height: int, width: int, nums_frame: int, fps: int): - """ - Generate mock video frames with metadata for testing purposes - - Creates synthetic video data by generating random RGB frames and constructing - corresponding metadata to simulate real video processing. - - Args: - height (int): Height of video frames in pixels - width (int): Width of video frames in pixels - nums_frame (int): Number of frames to generate - fps (int): Frames per second for the mock video - - Returns: - tuple: A tuple containing: - frames (numpy.ndarray): Array of shape (nums_frame, height, width, 3) - containing randomly generated RGB frames - meta (dict): Dictionary with video metadata: - - fps (int): Frames per second (same as input) - - duration (float): Calculated duration in seconds (nums_frame/fps) - - num_of_frame (int): Number of frames (same as nums_frame input) - """ - frames = [] - for _ in range(nums_frame): - frame = np.random.randint(0, 256, (height, width, 3), dtype=np.uint8) - frames.append(frame) - frames = np.stack(frames, axis=0) - - meta = { - "fps": fps, - "duration": nums_frame / fps, - "num_of_frame": nums_frame, - } - return frames, meta - - -class TestQwen3VLProcessor(unittest.TestCase): - """ - Unit tests for Qwen Vision-Language Processor functionality - """ - - def setUp(self): - """ - Initialize test case with: - - Mock configuration - - Patched message parsing and video processing methods - - QwenVLProcessor instance with test parameters - """ - config = MagicMock() - config.vision_config.tokens_per_second = 2 - - self.patcher_parse_image = patch( - "fastdeploy.entrypoints.chat_utils.MultimodalPartParser.parse_image", return_value=mock_pil_image(480, 640) - ) - self.patcher_parse_image.start() - - self.patcher_parse_video = patch( - "fastdeploy.entrypoints.chat_utils.MultimodalPartParser.parse_video", return_value=b"123" - ) - self.patcher_parse_video.start() - - self.patcher_read_frames = patch( - "fastdeploy.input.v1.qwen3_vl_processor.process.DataProcessor._load_and_process_video", - return_value=mock_read_frames(480, 640, 5, 2), - ) - self.patcher_read_frames.start() - - mm_processor_kwargs = {"video_max_frames": 10, "video_min_frames": 1} - limit_mm_per_prompt = {"image": 1, "video": 1, "audio": 1} - - self.model_name_or_path = "/ModelData/Qwen3-VL-4B-Instruct" - self.processor = Qwen3VLProcessor( - config=config, - model_name_or_path=self.model_name_or_path, - limit_mm_per_prompt=limit_mm_per_prompt, - mm_processor_kwargs=mm_processor_kwargs, - reasoning_parser_obj=None, - tool_parser_obj=None, - ) - - def tearDown(self) -> None: - """Clean up test case by stopping all mock patches""" - self.patcher_read_frames.stop() - self.patcher_parse_image.stop() - self.patcher_parse_video.stop() - - def test_process_request_dict(self): - """ - Test processing of dictionary-format request with multimodal input - - Validates: - 1. Token ID lengths match position_ids and token_type_ids shapes - 2. Image processing produces expected output dimensions - 3. Video processing produces expected output dimensions - 4. Correct counts for images (1) and videos (1) - """ - num_completion_token_ids = 10 - request = { - "request_id": "12345", - "completion_token_ids": [1] * num_completion_token_ids, - "stop": ["stop", "eof"], - "messages": [ - { - "role": "user", - "content": [ - {"type": "image_url", "image_url": {"url": "file://demo.jpeg"}}, - {"type": "video_url", "video_url": {"url": "file://3_frame_video.mp4"}}, - {"type": "text", "text": "Describe image and video."}, - ], - } - ], - } - request = Request.from_dict(request) - - result = self.processor.process_request_dict(request, 1024 * 100) - - self.assertEqual(result.prompt_token_ids_len, result.multimodal_inputs["position_ids"].shape[0]) - self.assertEqual(result.prompt_token_ids_len, result.multimodal_inputs["token_type_ids"].shape[0]) - self.assertEqual( - result.multimodal_inputs["images"].shape[0], - sum(map(lambda x: x.prod(), result.multimodal_inputs["grid_thw"])), - ) - self.assertEqual( - result.multimodal_inputs["image_type_ids"].shape[0], result.multimodal_inputs["grid_thw"][:, 0].sum() - ) - - def test_prompt(self): - """ - Test processing of prompt with image and video placeholders - - Validates: - 1. Token ID lengths match position_ids and token_type_ids shapes - 2. Image processing produces expected output dimensions - 3. Video processing produces expected output dimensions - 4. Correct counts for images (1) and videos (1) - """ - IMAGE_PLACEHOLDER = "<|image_pad|>" - VIDEO_PLACEHOLDER = "<|video_pad|>" - prompt = { - "request_id": "12345", - "prompt": f"{IMAGE_PLACEHOLDER}{VIDEO_PLACEHOLDER}Describe image and video.", - "multimodal_data": { - "image": [mock_pil_image(10, 2100)], - "video": [{"video": b"123", "fps": 5}], - }, - } - - request = Request.from_dict(prompt) - result = self.processor.process_request_dict(request, 1024 * 100) - - self.assertEqual(result.prompt_token_ids_len, result.multimodal_inputs["position_ids"].shape[0]) - self.assertEqual(result.prompt_token_ids_len, result.multimodal_inputs["token_type_ids"].shape[0]) - self.assertEqual( - result.multimodal_inputs["images"].shape[0], - sum(map(lambda x: x.prod(), result.multimodal_inputs["grid_thw"])), - ) - self.assertEqual( - result.multimodal_inputs["image_type_ids"].shape[0], result.multimodal_inputs["grid_thw"][:, 0].sum() - ) - - def test_message_and_prompt(self): - """ - Test consistency between message-based and prompt-based processing - - Validates that processing a request through: - 1. The message format (with image/video URLs) - 2. The prompt format (with direct image/video data) - produces identical tokenization and multimodal input results. - - Checks: - 1. Prompt token IDs match between both processing methods - 2. Grid dimensions (THW) match between both methods - 3. Position IDs match between both methods - """ - # Create test request in message format - request = { - "request_id": "12345", - "messages": [ - { - "role": "user", - "content": [ - {"type": "image_url", "image_url": {"url": "file://demo.jpeg"}}, - {"type": "video_url", "video_url": {"url": "file://3_frame_video.mp4"}}, - {"type": "text", "text": "Describe image and video."}, - ], - } - ], - } - request = Request.from_dict(request) - result = self.processor.process_request_dict(request, 1024 * 100) - - # Create equivalent request in prompt format - prompt = { - "request_id": "12345", - "prompt": request.prompt_tokens, - "multimodal_data": { - "image": [mock_pil_image(480, 640)], - "video": [{"video": b"123"}], - }, - } - request2 = Request.from_dict(prompt) - result2 = self.processor.process_request_dict(request2, 1024 * 100) - - # Verify both processing methods produce identical results - self.assertEqual(result.prompt_token_ids, result2.prompt_token_ids) - self.assertTrue(np.equal(result.multimodal_inputs["grid_thw"], result2.multimodal_inputs["grid_thw"]).all()) - self.assertTrue( - np.equal(result.multimodal_inputs["position_ids"], result2.multimodal_inputs["position_ids"]).all() - ) - - def test_apply_chat_template(self): - """ - Test the consistency between: - 1. Directly applying chat template using HuggingFace tokenizer - 2. Applying chat template through the processor's request processing - - This test verifies that: - - The processor correctly handles multimodal messages (image, video, text) - - The prompt_tokens field matches the output from direct tokenizer application - - The chat template application preserves the message structure and content - - Test Steps: - 1. Create sample multimodal messages with image, video and text content - 2. Apply chat template directly using the tokenizer - 3. Process the same messages through the processor - 4. Compare the outputs to ensure consistency - """ - from transformers import AutoTokenizer - - tokenizer = AutoTokenizer.from_pretrained(self.model_name_or_path) - - # Sample multimodal messages containing image, video and text - messages = [ - { - "role": "user", - "content": [ - {"type": "image_url", "image_url": {"url": "file://demo.jpeg"}}, - {"type": "video", "video": {"url": "file://3_frame_video.mp4"}}, - {"type": "text", "text": "Describe image and video."}, - ], - } - ] - - # Apply chat template directly using the tokenizer - prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) - - # Create equivalent request dictionary - request = { - "request_id": "12345", - "messages": [ - { - "role": "user", - "content": [ - {"type": "image_url", "image_url": {"url": "file://demo.jpeg"}}, - {"type": "video_url", "video_url": {"url": "file://3_frame_video.mp4"}}, - {"type": "text", "text": "Describe image and video."}, - ], - } - ], - } - request = Request.from_dict(request) - - # Process request through the processor - self.processor.process_request_dict(request, 1024 * 100) - prompt2 = request.prompt_tokens - - # Verify both methods produce identical prompt strings - self.assertEqual(prompt, prompt2) - - def test_add_processed_image(self): - """ - Test DataProcessor._add_processed_image via Qwen3VLProcessor - """ - merge_size = self.processor.processor.image_processor.merge_size - - # shape[0] must be divisible by merge_size^2 - num_tokens = 4 - img = np.zeros( - (num_tokens * merge_size * merge_size, 3, 3), - dtype=np.float32, - ) - meta = { - "thw": (1, 8, 8), - } - uuid = "test-image-uuid" - - img_cache = (img, meta) - - outputs = { - "mm_positions": [], - "input_ids": [], - "token_type_ids": [], - "position_ids": [], - "cur_position": 5, - "images": [], - "mm_hashes": [], - "grid_thw": [], - "image_type_ids": [], - "fps": [], - } - - # ----------------------- - # mock vision position computation - # ----------------------- - dp = self.processor.processor - dp._compute_vision_positions = MagicMock(return_value=np.array([[10, 11, 12]], dtype=np.int64)) - - dp._add_processed_image(img_cache, outputs, uuid) - - # ---- input_ids / token_type_ids ---- - self.assertEqual(len(outputs["input_ids"]), num_tokens) - self.assertEqual( - outputs["input_ids"], - [dp.image_token_id] * num_tokens, - ) - - # ---- mm_positions ---- - self.assertEqual(len(outputs["mm_positions"]), 1) - mm_pos = outputs["mm_positions"][0] - self.assertEqual(mm_pos.length, num_tokens) - - # ---- vision positions ---- - dp._compute_vision_positions.assert_called_once_with(5, 1, 8, 8, 0) - np.testing.assert_array_equal( - outputs["position_ids"][0], - np.array([[10, 11, 12]], dtype=np.int64), - ) - self.assertEqual(outputs["cur_position"], 13) - - # ---- image payload ---- - self.assertEqual(len(outputs["images"]), 1) - np.testing.assert_array_equal(outputs["images"][0], img) - - self.assertEqual(outputs["mm_hashes"], [uuid]) - np.testing.assert_array_equal( - outputs["grid_thw"][0], - np.array([[1, 8, 8]]), - ) - self.assertEqual(outputs["image_type_ids"], [0]) - self.assertEqual(outputs["fps"], [0]) - - def test_multimodal_token_len_validation(self): - """Test token_len validation for raw and processed multimodal paths""" - dp = self.processor.processor - merge_size = dp.image_processor.merge_size - - def build_outputs(image=False, video=False): - outputs = { - "mm_positions": [], - "input_ids": [], - "token_type_ids": [], - "position_ids": [], - "cur_position": 0, - "images": [], - "mm_hashes": [], - "grid_thw": [], - "image_type_ids": [], - "fps": [], - } - if image: - outputs["num_input_image_tokens"] = 0 - if video: - outputs["num_input_video_tokens"] = 0 - return outputs - - processed_image = ( - np.zeros((merge_size * merge_size, 3, 3), dtype=np.float32), - {"thw": (1, 8, 8)}, - ) - processed_video = ( - np.zeros((merge_size * merge_size, 3, 3), dtype=np.float32), - {"thw": (2, 8, 8), "fps": 5}, - ) - - with self.subTest("add_image"): - with patch.object( - dp.image_processor, - "preprocess", - return_value={ - "grid_thw": np.array([1, merge_size * 2, merge_size * 2]), - "pixel_values": np.zeros((1, 3, 3), dtype=np.float32), - }, - ): - with self.assertRaisesRegex(ValueError, "image tokens num not match the size"): - dp._add_image(mock_pil_image(32, 32), build_outputs(image=True), None, token_len=3) - - with self.subTest("add_processed_image"): - with self.assertRaisesRegex(ValueError, "image tokens num not match the size"): - dp._add_processed_image(processed_image, build_outputs(), "uuid", token_len=2) - - with self.subTest("add_video"): - with patch.object( - dp.image_processor, - "preprocess", - return_value={ - "grid_thw": np.array([1, merge_size * 2, merge_size * 2]), - "pixel_values": np.zeros((1, 3, 3), dtype=np.float32), - }, - ): - with self.assertRaisesRegex(ValueError, "video tokens num not match the size"): - dp._add_video( - np.zeros((2, 4, 4, 3), dtype=np.uint8), - {"fps": 4}, - build_outputs(video=True), - None, - token_len=3, - ) - - with self.subTest("add_processed_video"): - with self.assertRaisesRegex(ValueError, "video tokens num not match the size"): - dp._add_processed_video(processed_video, build_outputs(), "uuid", token_len=2) - - def test_prompt_token_ids2outputs_error_branches(self): - """Test prompt_token_ids2outputs error branches with minimal fixtures""" - dp = self.processor.processor - request = Request.from_dict( - { - "request_id": "12345", - "prompt_token_ids": [dp.image_token_id], - "messages": [{"role": "user", "content": [{"type": "image_url", "uuid": "missing-image"}]}], - } - ) - parsed_messages = [{"role": "user", "content": {"type": "image", "data": None, "uuid": "missing-image"}}] - - with self.subTest("missing_without_cache"): - with patch( - "fastdeploy.input.v1.qwen3_vl_processor.process.parse_chat_messages", return_value=parsed_messages - ): - with self.assertRaisesRegex(ValueError, "Missing items cannot be retrieved without processor cache."): - dp.prompt_token_ids2outputs(request) - - with self.subTest("missing_cache_item_not_found"): - old_enable_processor_cache = dp.enable_processor_cache - dp.enable_processor_cache = True - fake_context = MagicMock() - fake_context.socket.return_value = MagicMock() - try: - with patch( - "fastdeploy.input.v1.qwen3_vl_processor.process.parse_chat_messages", return_value=parsed_messages - ): - with patch( - "fastdeploy.input.v1.qwen3_vl_processor.process.zmq.Context", return_value=fake_context - ): - with patch.object(dp, "get_processor_cache", return_value=[None]): - with self.assertRaisesRegex(ValueError, "Missing item 0 not found in processor cache"): - dp.prompt_token_ids2outputs(request) - finally: - dp.enable_processor_cache = old_enable_processor_cache - - with self.subTest("unexpected_multimodal_type"): - - class FlakyTypeItem: - def __init__(self): - self.calls = 0 - - def get(self, key, default=None): - if key == "type": - self.calls += 1 - return "image" if self.calls == 1 else "audio" - if key == "data": - return "bad-data" - if key == "uuid": - return "bad-uuid" - return default - - parsed_messages = [{"role": "user", "content": FlakyTypeItem()}] - with patch( - "fastdeploy.input.v1.qwen3_vl_processor.process.parse_chat_messages", return_value=parsed_messages - ): - with self.assertRaisesRegex(ValueError, "Unsupported multimodal type: audio"): - dp.prompt_token_ids2outputs(request) - - def test_prompt_token_ids2outputs_cache_update_paths(self): - """Test prompt_token_ids2outputs cache update for missing, 1D and 2D grid_thw paths""" - dp = self.processor.processor - merge_size = dp.image_processor.merge_size - old_enable_processor_cache = dp.enable_processor_cache - dp.enable_processor_cache = True - - missing_image = ( - np.zeros((merge_size * merge_size, 3, 3), dtype=np.float32), - {"thw": (1, 8, 8)}, - ) - processed_video = ( - np.zeros((merge_size * merge_size, 3, 3), dtype=np.float32), - {"thw": (2, 8, 8), "fps": 6}, - ) - parsed_messages = [ - { - "role": "user", - "content": [ - {"type": "image", "data": None, "uuid": "missing-image"}, - {"type": "video", "data": {"video": "raw-video", "fps": 4}, "uuid": "raw-video"}, - {"type": "video", "data": processed_video, "uuid": "processed-video"}, - ], - } - ] - request = Request.from_dict( - { - "request_id": "12345", - "prompt_token_ids": [dp.image_token_id, 99, dp.image_token_id, 98, dp.image_token_id], - "messages": [{"role": "user", "content": [{"type": "text", "text": "unused"}]}], - } - ) - fake_socket = MagicMock() - fake_context = MagicMock() - fake_context.socket.return_value = fake_socket - - try: - with patch( - "fastdeploy.input.v1.qwen3_vl_processor.process.parse_chat_messages", return_value=parsed_messages - ): - with patch("fastdeploy.input.v1.qwen3_vl_processor.process.zmq.Context", return_value=fake_context): - with patch.object(dp, "_compute_vision_positions", return_value=np.array([[0]], dtype=np.int64)): - with patch.object( - dp.image_processor, - "preprocess", - return_value={ - "grid_thw": np.array([1, merge_size, merge_size]), - "pixel_values": np.zeros((1, 3, 3), dtype=np.float32), - }, - ): - with patch.object( - dp, "_load_and_process_video", return_value=mock_read_frames(4, 4, 2, 4) - ): - with patch.object( - dp, "get_processor_cache", return_value=[missing_image] - ) as cache_get: - with patch.object(dp, "update_processor_cache") as cache_update: - outputs = dp.prompt_token_ids2outputs(request) - - cache_get.assert_called_once_with(fake_socket, ["missing-image"]) - cache_update.assert_called_once() - _, cached_hashes, cached_items = cache_update.call_args.args - self.assertEqual(cached_hashes, ["raw-video", "processed-video"]) - self.assertEqual(cached_items[0][1]["thw"], (1, merge_size, merge_size)) - self.assertEqual(cached_items[1][1]["thw"], (2, 8, 8)) - self.assertEqual(outputs["mm_hashes"], ["missing-image", "raw-video", "processed-video"]) - self.assertEqual(outputs["input_ids"][-1], dp.image_token_id) - finally: - dp.enable_processor_cache = old_enable_processor_cache - - def test_request2ids_cache_update_paths(self): - """Test request2ids cache update for missing, 1D and 2D grid_thw paths""" - dp = self.processor.processor - merge_size = dp.image_processor.merge_size - old_enable_processor_cache = dp.enable_processor_cache - dp.enable_processor_cache = True - - missing_image = ( - np.zeros((merge_size * merge_size, 3, 3), dtype=np.float32), - {"thw": (1, 8, 8)}, - ) - processed_image = ( - np.zeros((merge_size * merge_size, 3, 3), dtype=np.float32), - {"thw": (1, 8, 8)}, - ) - parsed_messages = [ - { - "role": "user", - "content": [ - {"type": "image", "data": None, "uuid": "missing-image"}, - {"type": "image", "data": processed_image, "uuid": "processed-image"}, - {"type": "video", "data": {"video": "raw-video", "fps": 4}, "uuid": "raw-video"}, - ], - } - ] - request = Request.from_dict( - { - "request_id": "12345", - "messages": [{"role": "user", "content": [{"type": "text", "text": "unused"}]}], - "add_generation_prompt": True, - } - ) - fake_socket = MagicMock() - fake_context = MagicMock() - fake_context.socket.return_value = fake_socket - - try: - with patch( - "fastdeploy.input.v1.qwen3_vl_processor.process.parse_chat_messages", return_value=parsed_messages - ): - with patch("fastdeploy.input.v1.qwen3_vl_processor.process.zmq.Context", return_value=fake_context): - with patch.object(dp, "_compute_vision_positions", return_value=np.array([[0]], dtype=np.int64)): - with patch.object( - dp.image_processor, - "preprocess", - return_value={ - "grid_thw": np.array([1, merge_size, merge_size]), - "pixel_values": np.zeros((1, 3, 3), dtype=np.float32), - }, - ): - with patch.object( - dp, "_load_and_process_video", return_value=mock_read_frames(4, 4, 2, 4) - ): - with patch.object( - dp, "get_processor_cache", return_value=[missing_image] - ) as cache_get: - with patch.object(dp, "update_processor_cache") as cache_update: - with patch.object( - self.processor.tokenizer, - "apply_chat_template", - return_value="<|image_pad|>a<|image_pad|>b<|video_pad|>", - ): - outputs = dp.request2ids(request) - - cache_get.assert_called_once_with(fake_socket, ["missing-image"]) - cache_update.assert_called_once() - _, cached_hashes, cached_items = cache_update.call_args.args - self.assertEqual(cached_hashes, ["processed-image", "raw-video"]) - self.assertEqual(cached_items[0][1]["thw"], (1, 8, 8)) - self.assertEqual(cached_items[1][1]["thw"], (1, merge_size, merge_size)) - self.assertEqual(outputs["mm_hashes"], ["missing-image", "processed-image", "raw-video"]) - finally: - dp.enable_processor_cache = old_enable_processor_cache - - def test_parse_processor_kwargs_valid(self): - """Test _parse_processor_kwargs with valid input""" - valid_kwargs = {"video_max_frames": 10, "video_min_frames": 1} - result = self.processor._parse_processor_kwargs(valid_kwargs) - self.assertEqual(result, valid_kwargs) - - def test_parse_processor_kwargs_empty(self): - """Test _parse_processor_kwargs with empty input""" - result = self.processor._parse_processor_kwargs(None) - self.assertEqual(result, {}) - - def test_parse_processor_kwargs_invalid_type(self): - """Test _parse_processor_kwargs with invalid type""" - result = self.processor._parse_processor_kwargs("invalid") - self.assertEqual(result, {}) - - def test_parse_processor_kwargs_invalid_value_type(self): - """Test _parse_processor_kwargs with invalid value type""" - invalid_kwargs = {"video_max_frames": "10"} # Should be int - result = self.processor._parse_processor_kwargs(invalid_kwargs) - self.assertEqual(result, {}) - - def test_parse_processor_kwargs_mixed_valid_invalid(self): - """Test _parse_processor_kwargs with mixed valid and invalid types""" - mixed_kwargs = {"video_max_frames": 10, "video_min_frames": "invalid"} - result = self.processor._parse_processor_kwargs(mixed_kwargs) - self.assertEqual(result, {}) - - def test_parse_limits_valid(self): - """Test _parse_limits with valid limits""" - limits = {"image": 2, "video": 3} - result = self.processor._parse_limits(limits) - expected = {"image": 2, "video": 3, "audio": 1} - self.assertEqual(result, expected) - - def test_parse_limits_empty(self): - """Test _parse_limits with empty input""" - result = self.processor._parse_limits(None) - expected = {"image": 1, "video": 1, "audio": 1} - self.assertEqual(result, expected) - - def test_parse_limits_invalid_type(self): - """Test _parse_limits with invalid type""" - result = self.processor._parse_limits("invalid") - expected = {"image": 1, "video": 1, "audio": 1} - self.assertEqual(result, expected) - - def test_parse_limits_partial(self): - """Test _parse_limits with partial limits""" - limits = {"image": 5} - result = self.processor._parse_limits(limits) - expected = {"image": 5, "video": 1, "audio": 1} - self.assertEqual(result, expected) - - def test_check_mm_limits_dict_valid(self): - """Test _check_mm_limits with valid dict input""" - mm_data = {"image": [mock_pil_image(10, 10)], "video": [{"video": b"123"}]} - # Should not raise exception - self.processor._check_mm_limits(mm_data) - - def test_check_mm_limits_dict_exceed_limit(self): - """Test _check_mm_limits when dict input exceeds limit""" - mm_data = {"image": [mock_pil_image(10, 10), mock_pil_image(10, 10)]} - with self.assertRaises(ValueError) as context: - self.processor._check_mm_limits(mm_data) - self.assertIn("Too many image items", str(context.exception)) - - def test_check_mm_limits_messages_valid(self): - """Test _check_mm_limits with valid messages input""" - messages = [ - { - "role": "user", - "content": [ - {"type": "image_url", "image_url": {"url": "file://demo.jpeg"}}, - {"type": "text", "text": "Describe this image."}, - ], - } - ] - # Should not raise exception - self.processor._check_mm_limits(messages) - - def test_check_mm_limits_messages_exceed_limit(self): - """Test _check_mm_limits when messages input exceeds limit""" - messages = [ - { - "role": "user", - "content": [ - {"type": "image_url", "image_url": {"url": "file://demo1.jpeg"}}, - {"type": "image_url", "image_url": {"url": "file://demo2.jpeg"}}, - ], - } - ] - with self.assertRaises(ValueError) as context: - self.processor._check_mm_limits(messages) - self.assertIn("Too many image items", str(context.exception)) - - def test_check_mm_limits_video_exceed(self): - """Test _check_mm_limits when video exceeds limit""" - mm_data = {"video": [{"video": b"123"}, {"video": b"456"}]} - with self.assertRaises(ValueError) as context: - self.processor._check_mm_limits(mm_data) - self.assertIn("Too many video items", str(context.exception)) - - def test_process_request_dict_with_prompt(self): - """Test process_request_dict with prompt format""" - request = { - "request_id": "12345", - "prompt": "Test prompt", - "multimodal_data": {"image": [mock_pil_image(10, 10)]}, - } - request = Request.from_dict(request) - result = self.processor.process_request_dict(request, 1024) - self.assertGreater(len(result.prompt_token_ids), 0) - self.assertGreater(len(result.multimodal_inputs), 0) - - def test_process_request_dict_with_messages(self): - """Test process_request_dict with messages format""" - request = { - "request_id": "12345", - "messages": [ - { - "role": "user", - "content": [{"type": "text", "text": "Hello"}], - } - ], - } - request = Request.from_dict(request) - result = self.processor.process_request_dict(request, 1024) - self.assertGreater(len(result.prompt_token_ids), 0) - self.assertGreater(len(result.multimodal_inputs), 0) - - def test_process_request_dict_with_prompt_token_ids_only(self): - """Test process_request_dict with prompt_token_ids only""" - request = Request.from_dict( - { - "request_id": "12345", - "prompt_token_ids": [1, 2, 3], - } - ) - result = self.processor.process_request_dict(request, 1024) - - self.assertEqual(result.prompt_token_ids, [1, 2, 3]) - self.assertEqual(result.prompt_token_ids_len, 3) - self.assertIsNone(result.multimodal_inputs["images"]) - self.assertEqual(result.multimodal_inputs["token_type_ids"].tolist(), [0, 0, 0]) - - def test_process_request_dict_with_prompt_token_ids_and_messages(self): - """Test process_request_dict with prompt_token_ids and multimodal messages""" - source_request = Request.from_dict( - { - "request_id": "12345", - "messages": [ - { - "role": "user", - "content": [ - {"type": "image_url", "image_url": {"url": "file://demo.jpeg"}}, - {"type": "video_url", "video_url": {"url": "file://3_frame_video.mp4"}}, - {"type": "text", "text": "Describe image and video."}, - ], - } - ], - } - ) - source_result = self.processor.process_request_dict(source_request, 1024 * 100) - - token_request = Request.from_dict( - { - "request_id": "12345", - "prompt_token_ids": list(source_result.prompt_token_ids), - "messages": copy.deepcopy(source_request.messages), - } - ) - token_result = self.processor.process_request_dict(token_request, 1024 * 100) - - self.assertEqual(token_result.prompt_token_ids, source_result.prompt_token_ids) - self.assertTrue( - np.equal(token_result.multimodal_inputs["grid_thw"], source_result.multimodal_inputs["grid_thw"]).all() - ) - self.assertTrue( - np.equal( - token_result.multimodal_inputs["position_ids"], - source_result.multimodal_inputs["position_ids"], - ).all() - ) - self.assertTrue( - np.equal( - token_result.multimodal_inputs["image_type_ids"], - source_result.multimodal_inputs["image_type_ids"], - ).all() - ) - - def test_process_request_dict_prompt_token_ids_more_multimodal_segments_than_messages(self): - """Test prompt_token_ids path when token-side multimodal segments exceed messages""" - source_request = Request.from_dict( - { - "request_id": "12345", - "messages": [ - { - "role": "user", - "content": [ - {"type": "image_url", "image_url": {"url": "file://demo.jpeg"}}, - {"type": "video_url", "video_url": {"url": "file://3_frame_video.mp4"}}, - {"type": "text", "text": "Describe image and video."}, - ], - } - ], - } - ) - source_result = self.processor.process_request_dict(source_request, 1024 * 100) - - token_request = Request.from_dict( - { - "request_id": "12345", - "prompt_token_ids": list(source_result.prompt_token_ids), - "messages": [ - { - "role": "user", - "content": [ - {"type": "image_url", "image_url": {"url": "file://demo.jpeg"}}, - {"type": "text", "text": "Describe image and video."}, - ], - } - ], - } - ) - - with self.assertRaisesRegex(ValueError, "more multimodal placeholder"): - self.processor.process_request_dict(token_request, 1024 * 100) - - def test_process_request_dict_prompt_token_ids_unused_multimodal_messages(self): - """Test prompt_token_ids path when messages have unused multimodal items""" - source_request = Request.from_dict( - { - "request_id": "12345", - "messages": [ - { - "role": "user", - "content": [ - {"type": "image_url", "image_url": {"url": "file://demo.jpeg"}}, - {"type": "text", "text": "Describe image."}, - ], - } - ], - } - ) - source_result = self.processor.process_request_dict(source_request, 1024 * 100) - - token_request = Request.from_dict( - { - "request_id": "12345", - "prompt_token_ids": list(source_result.prompt_token_ids), - "messages": [ - { - "role": "user", - "content": [ - {"type": "image_url", "image_url": {"url": "file://demo.jpeg"}}, - {"type": "video_url", "video_url": {"url": "file://3_frame_video.mp4"}}, - {"type": "text", "text": "Describe image."}, - ], - } - ], - } - ) - - with self.assertRaisesRegex(ValueError, "number of multimodal items does not match"): - self.processor.process_request_dict(token_request, 1024 * 100) - - def test_process_request_dict_invalid_format(self): - """Test process_request_dict with invalid format""" - request = {"request_id": "12345"} - request = Request.from_dict(request) - with self.assertRaises(ValueError) as context: - self.processor.process_request_dict(request, 1024) - self.assertIn("must contain 'prompt', or 'messages'", str(context.exception)) - - def test_process_request_dict_with_bad_words(self): - """Test process_request_dict with bad_words""" - request = { - "request_id": "12345", - "prompt": "Test prompt", - "bad_words": ["bad", "word"], - "bad_words_token_ids": [100, 200], - } - request = Request.from_dict(request) - result = self.processor.process_request_dict(request, 1024) - # Verify bad_words_token_ids is set - self.assertIsNotNone(result.sampling_params.bad_words_token_ids) - - def test_process_request_dict_invalid_chat_template_kwargs(self): - """Test process_request_dict with invalid chat_template_kwargs""" - request = { - "request_id": "12345", - "messages": [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}], - } - request = Request.from_dict(request) - request.chat_template_kwargs = "invalid" - with self.assertRaises(ValueError) as context: - self.processor.process_request_dict(request, 1024) - self.assertIn("must be a dict", str(context.exception)) - - def test_process_request_dict_with_completion_token_ids(self): - """Test process_request_dict with completion_token_ids""" - request = {"request_id": "12345", "prompt": "Test"} - request = Request.from_dict(request) - request.completion_token_ids = [1, 2, 3] - result = self.processor.process_request_dict(request, 1024) - # Verify completion tokens are appended - self.assertGreater(len(result.prompt_token_ids), 3) - - def test_process_request_dict_prompt_truncation(self): - """Test process_request_dict with prompt truncation""" - # Create a long prompt that exceeds max_model_len - long_prompt = "Test " * 1000 - request = { - "request_id": "12345", - "prompt": long_prompt, - } - request = Request.from_dict(request) - result = self.processor.process_request_dict(request, 100) - # Verify prompt is truncated - self.assertLessEqual(len(result.prompt_token_ids), 99) - - def test_process_request_dict_default_max_tokens(self): - """Test process_request_dict sets default max_tokens""" - request = { - "request_id": "12345", - "prompt": "Test", - } - request = Request.from_dict(request) - result = self.processor.process_request_dict(request, 1024) - self.assertGreater(result.sampling_params.max_tokens, 0) - - def test_process_request_dict_enable_thinking_false(self): - """Test process_request_dict sets enable_thinking to False""" - request = { - "request_id": "12345", - "prompt": "Test", - "enable_thinking": True, - } - request = Request.from_dict(request) - result = self.processor.process_request_dict(request, 1024) - self.assertFalse(result.enable_thinking) - - def test_append_completion_tokens(self): - """Test append_completion_tokens method""" - multimodal_inputs = { - "input_ids": [1, 2, 3], - "token_type_ids": [0, 0, 0], - "position_ids": [np.array([[0, 1, 2], [0, 1, 2], [0, 1, 2]])], - "cur_position": 3, - } - completion_token_ids = [4, 5] - self.processor.append_completion_tokens(multimodal_inputs, completion_token_ids) - - self.assertEqual(multimodal_inputs["input_ids"], [1, 2, 3, 4, 5]) - self.assertEqual(multimodal_inputs["token_type_ids"], [0, 0, 0, 0, 0]) - self.assertEqual(multimodal_inputs["cur_position"], 5) - - def test_pack_outputs_with_images(self): - """Test pack_outputs with image data""" - outputs = { - "images": [np.array([[1, 2], [3, 4]]), np.array([[5, 6], [7, 8]])], - "grid_thw": [np.array([2, 2, 1]), np.array([2, 2, 1])], - "image_type_ids": [0, 1], - "input_ids": [1, 2, 3], - "token_type_ids": [0, 0, 0], - "position_ids": [np.array([[0, 1, 2], [0, 1, 2], [0, 1, 2]])], - } - result = self.processor.pack_outputs(outputs) - - self.assertIsNotNone(result["images"]) - self.assertIsNotNone(result["grid_thw"]) - self.assertIsNotNone(result["image_type_ids"]) - self.assertEqual(result["images"].shape[0], 4) - self.assertEqual(result["grid_thw"].shape[0], 2) - - def test_pack_outputs_without_images(self): - """Test pack_outputs without image data""" - outputs = { - "images": [], - "grid_thw": [], - "image_type_ids": [], - "input_ids": [1, 2, 3], - "token_type_ids": [0, 0, 0], - "position_ids": [np.array([[0, 1, 2], [0, 1, 2], [0, 1, 2]])], - } - result = self.processor.pack_outputs(outputs) - - # Test that image-related fields are None when no images - self.assertIsNone(result["images"]) - self.assertIsNone(result["grid_thw"]) - self.assertIsNone(result["image_type_ids"]) - - # Test data types - self.assertEqual(result["input_ids"].dtype, np.int64) - self.assertEqual(result["token_type_ids"].dtype, np.int64) - self.assertEqual(result["position_ids"].dtype, np.int64) - - # Test patch IDs are set - self.assertIn("image_patch_id", result) - self.assertIn("video_patch_id", result) - self.assertIn("mm_num_token_func", result) - - -class TestSampleFrames(unittest.TestCase): - """ - Unit tests for sample_frames function - """ - - def setUp(self): - self.metadata = { - "num_of_frame": 100, - "fps": 25, - } - - def test_fps_and_num_frames_mutually_exclusive(self): - with self.assertRaises(ValueError): - sample_frames( - frame_factor=4, - min_frames=8, - max_frames=32, - metadata=self.metadata, - fps=2, - num_frames=16, - ) - - def test_num_frames_round_to_factor(self): - indices = sample_frames( - frame_factor=4, - min_frames=8, - max_frames=64, - metadata=self.metadata, - num_frames=18, # round(18 / 4) * 4 = 16 - ) - - self.assertEqual(len(indices), 16) - self.assertEqual(indices[0], 0) - self.assertLess(indices[-1], self.metadata["num_of_frame"]) - - def test_fps_sampling_basic(self): - # total = 100 frames, fps=25, target fps=5 → 20 frames - indices = sample_frames( - frame_factor=4, - min_frames=8, - max_frames=64, - metadata=self.metadata, - fps=5, - ) - - self.assertEqual(len(indices), 20) - self.assertEqual(indices.dtype, np.int32) - self.assertEqual(indices[0], 0) - - def test_fps_respects_min_frames(self): - indices = sample_frames( - frame_factor=4, - min_frames=24, - max_frames=64, - metadata=self.metadata, - fps=1, # very small fps - ) - - self.assertEqual(len(indices), 24) - - def test_num_frames_exceeds_total_raises(self): - with self.assertRaises(ValueError): - sample_frames( - frame_factor=4, - min_frames=8, - max_frames=200, - metadata=self.metadata, - num_frames=200, - ) - - def test_force_multiple_of_4_hack(self): - indices = sample_frames( - frame_factor=2, - min_frames=2, - max_frames=100, - metadata=self.metadata, - num_frames=10, # 10 % 4 != 0 → hack → 8 - ) - - self.assertEqual(len(indices), 8) - self.assertEqual(len(indices) % 4, 0) - - def test_keep_all_frames_when_num_frames_zero(self): - indices = sample_frames( - frame_factor=4, - min_frames=0, - max_frames=100, - metadata=self.metadata, - num_frames=0, - ) - - self.assertEqual(len(indices), self.metadata["num_of_frame"]) - np.testing.assert_array_equal(indices, np.arange(0, 100, dtype=np.int32)) - - def test_indices_evenly_spaced(self): - indices = sample_frames( - frame_factor=4, - min_frames=8, - max_frames=32, - metadata=self.metadata, - num_frames=16, - ) - - diffs = np.diff(indices) - self.assertTrue(np.all(diffs > 0)) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/input/v1/test_qwen_vl_processor.py b/tests/input/v1/test_qwen_vl_processor.py deleted file mode 100644 index 5de7df1da39..00000000000 --- a/tests/input/v1/test_qwen_vl_processor.py +++ /dev/null @@ -1,776 +0,0 @@ -""" -# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -import unittest -from unittest.mock import MagicMock, patch - -import numpy as np -from PIL import Image - -from fastdeploy.engine.request import Request -from fastdeploy.input.v1.qwen_vl_processor import QwenVLProcessor -from fastdeploy.input.v1.qwen_vl_processor.process_video import sample_frames - - -def mock_pil_image(height, width): - """ - Generate mock random RGB image - - Args: - height: Image height in pixels - width: Image width in pixels - - Returns: - PIL.Image object with random RGB data - """ - rgb_image = np.random.randint(0, 256, (height, width, 3), dtype=np.uint8) - return Image.fromarray(rgb_image) - - -def mock_read_frames(height: int, width: int, nums_frame: int, fps: int): - """ - Generate mock video frames with metadata for testing purposes - - Creates synthetic video data by generating random RGB frames and constructing - corresponding metadata to simulate real video processing. - - Args: - height (int): Height of video frames in pixels - width (int): Width of video frames in pixels - nums_frame (int): Number of frames to generate - fps (int): Frames per second for the mock video - - Returns: - tuple: A tuple containing: - frames (numpy.ndarray): Array of shape (nums_frame, height, width, 3) - containing randomly generated RGB frames - meta (dict): Dictionary with video metadata: - - fps (int): Frames per second (same as input) - - duration (float): Calculated duration in seconds (nums_frame/fps) - - num_of_frame (int): Number of frames (same as nums_frame input) - """ - frames = [] - for _ in range(nums_frame): - frame = np.random.randint(0, 256, (height, width, 3), dtype=np.uint8) - frames.append(frame) - frames = np.stack(frames, axis=0) - - meta = { - "fps": fps, - "duration": nums_frame / fps, - "num_of_frame": nums_frame, - } - return frames, meta - - -class TestQwenVLProcessor(unittest.TestCase): - """ - Unit tests for Qwen Vision-Language Processor functionality - """ - - def setUp(self): - """ - Initialize test case with: - - Mock configuration - - Patched message parsing and video processing methods - - QwenVLProcessor instance with test parameters - """ - config = MagicMock() - config.vision_config.tokens_per_second = 2 - - self.patcher_parse_image = patch( - "fastdeploy.entrypoints.chat_utils.MultimodalPartParser.parse_image", return_value=mock_pil_image(480, 640) - ) - self.patcher_parse_image.start() - - self.patcher_parse_video = patch( - "fastdeploy.entrypoints.chat_utils.MultimodalPartParser.parse_video", return_value=b"123" - ) - self.patcher_parse_video.start() - - self.patcher_read_frames = patch( - "fastdeploy.input.v1.qwen_vl_processor.process.DataProcessor._load_and_process_video", - return_value=mock_read_frames(480, 640, 5, 2), - ) - self.patcher_read_frames.start() - - mm_processor_kwargs = { - "video_max_frames": 10, - "video_min_frames": 1, - } - limit_mm_per_prompt = {"image": 1, "video": 1, "audio": 1} - - self.model_name_or_path = "/ModelData/Qwen2.5-VL-7B-Instruct" - self.processor = QwenVLProcessor( - config=config, - model_name_or_path=self.model_name_or_path, - limit_mm_per_prompt=limit_mm_per_prompt, - mm_processor_kwargs=mm_processor_kwargs, - reasoning_parser_obj=None, - tool_parser_obj=None, - ) - - def tearDown(self) -> None: - """Clean up test case by stopping all mock patches""" - self.patcher_read_frames.stop() - self.patcher_parse_image.stop() - self.patcher_parse_video.stop() - - def test_process_request(self): - """ - Test processing of Request object with multimodal input - - Validates: - 1. Token ID lengths match position_ids and token_type_ids shapes - 2. Image processing produces expected output dimensions - 3. Video processing produces expected output dimensions - 4. Correct counts for images (1) and videos (1) - """ - message = { - "request_id": "12345", - "messages": [ - { - "role": "user", - "content": [ - {"type": "image_url", "image_url": {"url": "file://demo.jpeg"}}, - {"type": "video_url", "video_url": {"url": "file://3_frame_video.mp4"}}, - {"type": "text", "text": "Describe image and video."}, - ], - } - ], - } - - request = Request.from_dict(message) - result = self.processor.process_request_dict(request, 1024 * 100) - - self.assertEqual(result.prompt_token_ids_len, result.multimodal_inputs["position_ids"].shape[0]) - self.assertEqual(result.prompt_token_ids_len, result.multimodal_inputs["token_type_ids"].shape[0]) - self.assertEqual( - result.multimodal_inputs["images"].shape[0], - sum(map(lambda x: x.prod(), result.multimodal_inputs["grid_thw"])), - ) - self.assertEqual( - result.multimodal_inputs["image_type_ids"].shape[0], result.multimodal_inputs["grid_thw"][:, 0].sum() - ) - - def test_process_request_dict(self): - """ - Test processing of dictionary-format request with multimodal input - - Validates: - 1. Token ID lengths match position_ids and token_type_ids shapes - 2. Image processing produces expected output dimensions - 3. Video processing produces expected output dimensions - 4. Correct counts for images (1) and videos (1) - """ - num_completion_token_ids = 10 - request = { - "request_id": "12345", - "completion_token_ids": [1] * num_completion_token_ids, - "stop": ["stop", "eof"], - "messages": [ - { - "role": "user", - "content": [ - {"type": "image_url", "image_url": {"url": "file://demo.jpeg"}}, - {"type": "video_url", "video_url": {"url": "file://3_frame_video.mp4"}}, - {"type": "text", "text": "Describe image and video."}, - ], - } - ], - } - request = Request.from_dict(request) - - result = self.processor.process_request_dict(request, 1024 * 100) - - self.assertEqual(result.prompt_token_ids_len, result.multimodal_inputs["position_ids"].shape[0]) - self.assertEqual(result.prompt_token_ids_len, result.multimodal_inputs["token_type_ids"].shape[0]) - self.assertEqual( - result.multimodal_inputs["images"].shape[0], - sum(map(lambda x: x.prod(), result.multimodal_inputs["grid_thw"])), - ) - self.assertEqual( - result.multimodal_inputs["image_type_ids"].shape[0], result.multimodal_inputs["grid_thw"][:, 0].sum() - ) - - def test_process_request_dict_enable_thinking(self): - num_completion_token_ids = 10 - request = { - "request_id": "12345", - "completion_token_ids": [1] * num_completion_token_ids, - "stop": ["stop", "eof"], - "messages": [ - { - "role": "user", - "content": [ - {"type": "text", "text": "Hello"}, - ], - } - ], - "chat_template_kwargs": {"enable_thinking": True}, - } - request = Request.from_dict(request) - - result = self.processor.process_request_dict(request, 100) - self.assertEqual(result.enable_thinking, False) - - def test_prompt(self): - """ - Test processing of prompt with image and video placeholders - - Validates: - 1. Token ID lengths match position_ids and token_type_ids shapes - 2. Image processing produces expected output dimensions - 3. Video processing produces expected output dimensions - 4. Correct counts for images (1) and videos (1) - """ - IMAGE_PLACEHOLDER = "<|image_pad|>" - VIDEO_PLACEHOLDER = "<|video_pad|>" - prompt = { - "request_id": "12345", - "prompt": f"{IMAGE_PLACEHOLDER}{VIDEO_PLACEHOLDER}Describe image and video.", - "multimodal_data": { - "image": [mock_pil_image(10, 2100)], - "video": [{"video": b"123", "fps": 5}], - }, - } - - request = Request.from_dict(prompt) - result = self.processor.process_request_dict(request, 1024 * 100) - - self.assertEqual(result.prompt_token_ids_len, result.multimodal_inputs["position_ids"].shape[0]) - self.assertEqual(result.prompt_token_ids_len, result.multimodal_inputs["token_type_ids"].shape[0]) - self.assertEqual( - result.multimodal_inputs["images"].shape[0], - sum(map(lambda x: x.prod(), result.multimodal_inputs["grid_thw"])), - ) - self.assertEqual( - result.multimodal_inputs["image_type_ids"].shape[0], result.multimodal_inputs["grid_thw"][:, 0].sum() - ) - - def test_message_and_prompt(self): - """ - Test consistency between message-based and prompt-based processing - - Validates that processing a request through: - 1. The message format (with image/video URLs) - 2. The prompt format (with direct image/video data) - produces identical tokenization and multimodal input results. - - Checks: - 1. Prompt token IDs match between both processing methods - 2. Grid dimensions (THW) match between both methods - 3. Position IDs match between both methods - """ - # Create test request in message format - request = { - "request_id": "12345", - "messages": [ - { - "role": "user", - "content": [ - {"type": "image_url", "image_url": {"url": "file://demo.jpeg"}}, - {"type": "video_url", "video_url": {"url": "file://3_frame_video.mp4"}}, - {"type": "text", "text": "Describe image and video."}, - ], - } - ], - } - request = Request.from_dict(request) - result = self.processor.process_request_dict(request, 1024 * 100) - - # Create equivalent request in prompt format - prompt = { - "request_id": "12345", - "prompt": request.prompt_tokens, - "multimodal_data": { - "image": [mock_pil_image(480, 640)], - "video": [{"video": b"123"}], - }, - } - request2 = Request.from_dict(prompt) - result2 = self.processor.process_request_dict(request2, 1024 * 100) - - # Verify both processing methods produce identical results - self.assertEqual(result.prompt_token_ids, result2.prompt_token_ids) - self.assertTrue(np.equal(result.multimodal_inputs["grid_thw"], result2.multimodal_inputs["grid_thw"]).all()) - self.assertTrue( - np.equal(result.multimodal_inputs["position_ids"], result2.multimodal_inputs["position_ids"]).all() - ) - - def test_apply_chat_template(self): - """ - Test the consistency between: - 1. Directly applying chat template using HuggingFace tokenizer - 2. Applying chat template through the processor's request processing - - This test verifies that: - - The processor correctly handles multimodal messages (image, video, text) - - The prompt_tokens field matches the output from direct tokenizer application - - The chat template application preserves the message structure and content - - Test Steps: - 1. Create sample multimodal messages with image, video and text content - 2. Apply chat template directly using the tokenizer - 3. Process the same messages through the processor - 4. Compare the outputs to ensure consistency - """ - from transformers import AutoTokenizer - - tokenizer = AutoTokenizer.from_pretrained(self.model_name_or_path) - - # Sample multimodal messages containing image, video and text - messages = [ - { - "role": "user", - "content": [ - {"type": "image_url", "image_url": {"url": "file://demo.jpeg"}}, - {"type": "video", "video": {"url": "file://3_frame_video.mp4"}}, - {"type": "text", "text": "Describe image and video."}, - ], - } - ] - - # Apply chat template directly using the tokenizer - prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) - - # Create equivalent request dictionary - request = { - "request_id": "12345", - "messages": [ - { - "role": "user", - "content": [ - {"type": "image_url", "image_url": {"url": "file://demo.jpeg"}}, - {"type": "video_url", "video_url": {"url": "file://3_frame_video.mp4"}}, - {"type": "text", "text": "Describe image and video."}, - ], - } - ], - } - request = Request.from_dict(request) - - # Process request through the processor - self.processor.process_request_dict(request, 1024 * 100) - prompt2 = request.prompt_tokens - - # Verify both methods produce identical prompt strings - self.assertEqual(prompt, prompt2) - - def test_think_status(self): - """测试 思考机制""" - request = { - "prompt": "hello", - "request_id": "test_1", - "prompt_token_ids": [1, 2, 3], - "temperature": 0.7, - "top_p": 0.9, - } - request = Request.from_dict(request) - self.processor.reasoning_parser = MagicMock() - self.processor.reasoning_parser.get_model_status.return_value = "think_start" - self.processor.model_status_dict = {} - self.processor.process_request_dict(request, max_model_len=512) - self.assertEqual(request.enable_thinking, True) - - request = { - "prompt": "hello", - "request_id": "test", - "prompt_token_ids": [1, 2, 3], - "temperature": 0.7, - "top_p": 0.9, - } - request = Request.from_dict(request) - self.processor.process_request_dict(request, max_model_len=512) - self.assertEqual(request.enable_thinking, True) - - def test_parse_processor_kwargs_valid(self): - """Test _parse_processor_kwargs with valid input""" - valid_kwargs = {"video_max_frames": 10, "video_min_frames": 1} - result = self.processor._parse_processor_kwargs(valid_kwargs) - self.assertEqual(result, valid_kwargs) - - def test_parse_processor_kwargs_empty(self): - """Test _parse_processor_kwargs with empty input""" - result = self.processor._parse_processor_kwargs(None) - self.assertEqual(result, {}) - - def test_parse_processor_kwargs_invalid_type(self): - """Test _parse_processor_kwargs with invalid type""" - result = self.processor._parse_processor_kwargs("invalid") - self.assertEqual(result, {}) - - def test_parse_processor_kwargs_invalid_value_type(self): - """Test _parse_processor_kwargs with invalid value type""" - invalid_kwargs = {"video_max_frames": "10"} # Should be int - result = self.processor._parse_processor_kwargs(invalid_kwargs) - self.assertEqual(result, {}) - - def test_parse_processor_kwargs_mixed_valid_invalid(self): - """Test _parse_processor_kwargs with mixed valid and invalid types""" - mixed_kwargs = {"video_max_frames": 10, "video_min_frames": "invalid"} - result = self.processor._parse_processor_kwargs(mixed_kwargs) - self.assertEqual(result, {}) - - def test_parse_limits_valid(self): - """Test _parse_limits with valid limits""" - limits = {"image": 2, "video": 3} - result = self.processor._parse_limits(limits) - expected = {"image": 2, "video": 3, "audio": 1} - self.assertEqual(result, expected) - - def test_parse_limits_empty(self): - """Test _parse_limits with empty input""" - result = self.processor._parse_limits(None) - expected = {"image": 1, "video": 1, "audio": 1} - self.assertEqual(result, expected) - - def test_parse_limits_invalid_type(self): - """Test _parse_limits with invalid type""" - result = self.processor._parse_limits("invalid") - expected = {"image": 1, "video": 1, "audio": 1} - self.assertEqual(result, expected) - - def test_parse_limits_partial(self): - """Test _parse_limits with partial limits""" - limits = {"image": 5} - result = self.processor._parse_limits(limits) - expected = {"image": 5, "video": 1, "audio": 1} - self.assertEqual(result, expected) - - def test_check_mm_limits_dict_valid(self): - """Test _check_mm_limits with valid dict input""" - mm_data = {"image": [mock_pil_image(10, 10)], "video": [{"video": b"123"}]} - # Should not raise exception - self.processor._check_mm_limits(mm_data) - - def test_check_mm_limits_dict_exceed_limit(self): - """Test _check_mm_limits when dict input exceeds limit""" - mm_data = {"image": [mock_pil_image(10, 10), mock_pil_image(10, 10)]} - with self.assertRaises(ValueError) as context: - self.processor._check_mm_limits(mm_data) - self.assertIn("Too many image items", str(context.exception)) - - def test_check_mm_limits_messages_valid(self): - """Test _check_mm_limits with valid messages input""" - messages = [ - { - "role": "user", - "content": [ - {"type": "image_url", "image_url": {"url": "file://demo.jpeg"}}, - {"type": "text", "text": "Describe this image."}, - ], - } - ] - # Should not raise exception - self.processor._check_mm_limits(messages) - - def test_check_mm_limits_messages_exceed_limit(self): - """Test _check_mm_limits when messages input exceeds limit""" - messages = [ - { - "role": "user", - "content": [ - {"type": "image_url", "image_url": {"url": "file://demo1.jpeg"}}, - {"type": "image_url", "image_url": {"url": "file://demo2.jpeg"}}, - ], - } - ] - with self.assertRaises(ValueError) as context: - self.processor._check_mm_limits(messages) - self.assertIn("Too many image items", str(context.exception)) - - def test_check_mm_limits_video_exceed(self): - """Test _check_mm_limits when video exceeds limit""" - mm_data = {"video": [{"video": b"123"}, {"video": b"456"}]} - with self.assertRaises(ValueError) as context: - self.processor._check_mm_limits(mm_data) - self.assertIn("Too many video items", str(context.exception)) - - def test_process_request_dict_with_prompt(self): - """Test process_request_dict with prompt format""" - request = { - "request_id": "12345", - "prompt": "Test prompt", - "multimodal_data": {"image": [mock_pil_image(10, 10)]}, - } - request = Request.from_dict(request) - result = self.processor.process_request_dict(request, 1024) - self.assertGreater(len(result.prompt_token_ids), 0) - self.assertGreater(len(result.multimodal_inputs), 0) - - def test_process_request_dict_with_messages(self): - """Test process_request_dict with messages format""" - request = { - "request_id": "12345", - "messages": [ - { - "role": "user", - "content": [{"type": "text", "text": "Hello"}], - } - ], - } - request = Request.from_dict(request) - result = self.processor.process_request_dict(request, 1024) - self.assertGreater(len(result.prompt_token_ids), 0) - self.assertGreater(len(result.multimodal_inputs), 0) - - def test_process_request_dict_invalid_format(self): - """Test process_request_dict with invalid format""" - request = {"request_id": "12345"} - request = Request.from_dict(request) - with self.assertRaises(ValueError) as context: - self.processor.process_request_dict(request, 1024) - self.assertIn("must contain 'prompt', or 'messages'", str(context.exception)) - - def test_process_request_dict_with_bad_words(self): - """Test process_request_dict with bad_words""" - request = { - "request_id": "12345", - "prompt": "Test prompt", - "bad_words": ["bad", "word"], - "bad_words_token_ids": [100, 200], - } - request = Request.from_dict(request) - result = self.processor.process_request_dict(request, 1024) - # Verify bad_words_token_ids is set - self.assertIsNotNone(result.sampling_params.bad_words_token_ids) - - def test_process_request_dict_invalid_chat_template_kwargs(self): - """Test process_request_dict with invalid chat_template_kwargs""" - request = { - "request_id": "12345", - "messages": [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}], - } - request = Request.from_dict(request) - request.chat_template_kwargs = "invalid" - with self.assertRaises(ValueError) as context: - self.processor.process_request_dict(request, 1024) - self.assertIn("must be a dict", str(context.exception)) - - def test_process_request_dict_with_completion_token_ids(self): - """Test process_request_dict with completion_token_ids""" - request = {"request_id": "12345", "prompt": "Test"} - request = Request.from_dict(request) - request.completion_token_ids = [1, 2, 3] - result = self.processor.process_request_dict(request, 1024) - # Verify completion tokens are appended - self.assertGreater(len(result.prompt_token_ids), 3) - - def test_process_request_dict_prompt_truncation(self): - """Test process_request_dict with prompt truncation""" - # Create a long prompt that exceeds max_model_len - long_prompt = "Test " * 1000 - request = { - "request_id": "12345", - "prompt": long_prompt, - } - request = Request.from_dict(request) - result = self.processor.process_request_dict(request, 100) - # Verify prompt is truncated - self.assertLessEqual(len(result.prompt_token_ids), 99) - - def test_process_request_dict_default_max_tokens(self): - """Test process_request_dict sets default max_tokens""" - request = { - "request_id": "12345", - "prompt": "Test", - } - request = Request.from_dict(request) - result = self.processor.process_request_dict(request, 1024) - self.assertGreater(result.sampling_params.max_tokens, 0) - - def test_process_request_dict_enable_thinking_false(self): - """Test process_request_dict sets enable_thinking to False""" - request = { - "request_id": "12345", - "prompt": "Test", - "enable_thinking": True, - } - request = Request.from_dict(request) - result = self.processor.process_request_dict(request, 1024) - self.assertFalse(result.enable_thinking) - - def test_append_completion_tokens(self): - """Test append_completion_tokens method""" - multimodal_inputs = { - "input_ids": [1, 2, 3], - "token_type_ids": [0, 0, 0], - "position_ids": [np.array([[0, 1, 2], [0, 1, 2], [0, 1, 2]])], - "cur_position": 3, - } - completion_token_ids = [4, 5] - self.processor.append_completion_tokens(multimodal_inputs, completion_token_ids) - - self.assertEqual(multimodal_inputs["input_ids"], [1, 2, 3, 4, 5]) - self.assertEqual(multimodal_inputs["token_type_ids"], [0, 0, 0, 0, 0]) - self.assertEqual(multimodal_inputs["cur_position"], 5) - - def test_pack_outputs_with_images(self): - """Test pack_outputs with image data""" - outputs = { - "images": [np.array([[1, 2], [3, 4]]), np.array([[5, 6], [7, 8]])], - "grid_thw": [np.array([2, 2, 1]), np.array([2, 2, 1])], - "image_type_ids": [0, 1], - "input_ids": [1, 2, 3], - "token_type_ids": [0, 0, 0], - "position_ids": [np.array([[0, 1, 2], [0, 1, 2], [0, 1, 2]])], - } - result = self.processor.pack_outputs(outputs) - - self.assertIsNotNone(result["images"]) - self.assertIsNotNone(result["grid_thw"]) - self.assertIsNotNone(result["image_type_ids"]) - self.assertEqual(result["images"].shape[0], 4) - self.assertEqual(result["grid_thw"].shape[0], 2) - - def test_pack_outputs_without_images(self): - """Test pack_outputs without image data""" - outputs = { - "images": [], - "grid_thw": [], - "image_type_ids": [], - "input_ids": [1, 2, 3], - "token_type_ids": [0, 0, 0], - "position_ids": [np.array([[0, 1, 2], [0, 1, 2], [0, 1, 2]])], - } - result = self.processor.pack_outputs(outputs) - - # Test that image-related fields are None when no images - self.assertIsNone(result["images"]) - self.assertIsNone(result["grid_thw"]) - self.assertIsNone(result["image_type_ids"]) - - # Test data types - self.assertEqual(result["input_ids"].dtype, np.int64) - self.assertEqual(result["token_type_ids"].dtype, np.int64) - self.assertEqual(result["position_ids"].dtype, np.int64) - - # Test patch IDs are set - self.assertIn("image_patch_id", result) - self.assertIn("video_patch_id", result) - self.assertIn("mm_num_token_func", result) - - -class TestSampleFrames(unittest.TestCase): - """ - Unit tests for sample_frames function - """ - - def setUp(self): - self.metadata = { - "num_of_frame": 100, - "fps": 25, - } - - def test_fps_and_num_frames_mutually_exclusive(self): - with self.assertRaises(ValueError): - sample_frames( - frame_factor=4, - min_frames=8, - max_frames=32, - metadata=self.metadata, - fps=2, - num_frames=16, - ) - - def test_num_frames_round_to_factor(self): - indices = sample_frames( - frame_factor=4, - min_frames=8, - max_frames=64, - metadata=self.metadata, - num_frames=18, # round(18 / 4) * 4 = 16 - ) - - self.assertEqual(len(indices), 16) - self.assertEqual(indices[0], 0) - self.assertLess(indices[-1], self.metadata["num_of_frame"]) - - def test_fps_sampling_basic(self): - # total = 100 frames, fps=25, target fps=5 → 20 frames - indices = sample_frames( - frame_factor=4, - min_frames=8, - max_frames=64, - metadata=self.metadata, - fps=5, - ) - - self.assertEqual(len(indices), 20) - self.assertEqual(indices.dtype, np.int32) - self.assertEqual(indices[0], 0) - - def test_fps_respects_min_frames(self): - indices = sample_frames( - frame_factor=4, - min_frames=24, - max_frames=64, - metadata=self.metadata, - fps=1, # very small fps - ) - - self.assertEqual(len(indices), 24) - - def test_num_frames_exceeds_total_raises(self): - with self.assertRaises(ValueError): - sample_frames( - frame_factor=4, - min_frames=8, - max_frames=200, - metadata=self.metadata, - num_frames=200, - ) - - def test_force_multiple_of_4_hack(self): - indices = sample_frames( - frame_factor=2, - min_frames=2, - max_frames=100, - metadata=self.metadata, - num_frames=10, # 10 % 4 != 0 → hack → 8 - ) - - self.assertEqual(len(indices), 8) - self.assertEqual(len(indices) % 4, 0) - - def test_keep_all_frames_when_num_frames_zero(self): - indices = sample_frames( - frame_factor=4, - min_frames=0, - max_frames=100, - metadata=self.metadata, - num_frames=0, - ) - - self.assertEqual(len(indices), self.metadata["num_of_frame"]) - np.testing.assert_array_equal(indices, np.arange(0, 100, dtype=np.int32)) - - def test_indices_evenly_spaced(self): - indices = sample_frames( - frame_factor=4, - min_frames=8, - max_frames=32, - metadata=self.metadata, - num_frames=16, - ) - - diffs = np.diff(indices) - self.assertTrue(np.all(diffs > 0)) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/input/v1/test_text_processor.py b/tests/input/v1/test_text_processor.py deleted file mode 100644 index 147d843e85d..00000000000 --- a/tests/input/v1/test_text_processor.py +++ /dev/null @@ -1,586 +0,0 @@ -""" -# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -import importlib -import importlib.util -import sys -import types -import unittest -from pathlib import Path -from types import SimpleNamespace -from unittest import mock - -import numpy as np - -from fastdeploy.engine.request import Request, RequestOutput -from fastdeploy.engine.sampling_params import SamplingParams - - -class DummyTokenizer: - bos_token = "" - cls_token = "" - sep_token = "" - eos_token = "" - mask_token = "" - chat_template = "dummy" - - def __init__(self): - self.pad_token_id = 1 - self.eos_token_id = 2 - self.eos_token = 2 - self.vocab_size = 256 - self.bos_token_id = self._convert_token_to_id(self.bos_token) - self.cls_token_id = self._convert_token_to_id(self.cls_token) - self.sep_token_id = self._convert_token_to_id(self.sep_token) - self.mask_token_id = self._convert_token_to_id(self.mask_token) - - def _convert_token_to_id(self, token): - return len(str(token)) - - def __call__(self, text, **kwargs): - if isinstance(text, list): - values = [self._value(item) for item in text] - else: - values = [self._value(text)] - max_length = kwargs.get("max_length") - if max_length is not None: - values = values[:max_length] - return {"input_ids": np.array([values], dtype=np.int64)} - - def _value(self, item): - if isinstance(item, str): - return len(item) - return int(item) - - def tokenize(self, text): - if isinstance(text, str): - return [text] - return [str(text)] - - def convert_tokens_to_ids(self, tokens): - return [self._value(token) for token in tokens] - - def decode(self, token_ids, **kwargs): - return " ".join(str(t) for t in token_ids) - - def decode_token(self, token_ids, prefix_offset, read_offset): - start = read_offset - delta_tokens = token_ids[start:] - delta = "".join(str(t) for t in delta_tokens) - prefix_offset += len(token_ids) - read_offset += len(delta_tokens) - return delta, prefix_offset, read_offset - - def batch_decode(self, batch, **kwargs): - return [self.decode(seq) for seq in batch] - - def apply_chat_template(self, request, **kwargs): - if isinstance(request, dict): - system = request.get("system") - messages = request.get("messages", []) - else: - system = getattr(request, "system", None) - messages = getattr(request, "messages", []) - parts = [system] if system else [] - parts.extend(msg.get("content", "") for msg in messages) - return " ".join(part for part in parts if part) - - -class DummyLlamaTokenizer(DummyTokenizer): - pass - - -class DummyAutoTokenizer: - @classmethod - def from_pretrained(cls, *args, **kwargs): - return DummyTokenizer() - - -class DummyHFTokenizer: - @classmethod - def from_pretrained(cls, *args, **kwargs): - return DummyTokenizer() - - -def _create_dummy_modules(): - """Create all dummy modules needed for testing fastdeploy.input.text_processor.""" - repo_root = Path(__file__).resolve().parents[2] - - dummy_logger = SimpleNamespace( - info=lambda *args, **kwargs: None, - warning=lambda *args, **kwargs: None, - debug=lambda *args, **kwargs: None, - ) - - utils_module = types.ModuleType("fastdeploy.utils") - utils_module.data_processor_logger = dummy_logger - - envs_module = types.ModuleType("fastdeploy.envs") - envs_module.FD_USE_HF_TOKENIZER = False - - generation_module = types.ModuleType("paddleformers.generation") - - class DummyGenerationConfig: - def __init__(self): - self.top_p = 0.8 - self.temperature = 0.9 - self.repetition_penalty = 1.1 - self.frequency_penalty = 0.2 - self.presence_penalty = 0.1 - - @classmethod - def from_pretrained(cls, *args, **kwargs): - return cls() - - generation_module.GenerationConfig = DummyGenerationConfig - - transformers_module = types.ModuleType("paddleformers.transformers") - transformers_module.AutoTokenizer = DummyAutoTokenizer - transformers_module.LlamaTokenizer = DummyLlamaTokenizer - transformers_module.Llama3Tokenizer = DummyLlamaTokenizer - - hf_transformers_module = types.ModuleType("transformers") - hf_transformers_module.AutoTokenizer = DummyHFTokenizer - - llm_utils_module = types.ModuleType("paddleformers.cli.utils.llm_utils") - llm_utils_module.get_eos_token_id = lambda tokenizer, config: [tokenizer.eos_token_id] - - fastdeploy_module = types.ModuleType("fastdeploy") - fastdeploy_module.__path__ = [str(repo_root / "fastdeploy")] - fastdeploy_module.utils = utils_module - fastdeploy_module.envs = envs_module - - return { - "fastdeploy": fastdeploy_module, - "fastdeploy.utils": utils_module, - "fastdeploy.envs": envs_module, - "paddleformers.generation": generation_module, - "paddleformers.transformers": transformers_module, - "transformers": hf_transformers_module, - "paddleformers.cli.utils.llm_utils": llm_utils_module, - } - - -def _import_text_processor(use_hf_tokenizer=False): - modules = _create_dummy_modules() - - modules["fastdeploy.envs"].FD_USE_HF_TOKENIZER = use_hf_tokenizer - - previous_modules = {} - for name, module in modules.items(): - previous_modules[name] = sys.modules.get(name) - sys.modules[name] = module - - try: - text_processor_module = importlib.import_module("fastdeploy.input.v1.text_processor") - importlib.reload(text_processor_module) - except Exception: - for name, original in previous_modules.items(): - if original is None: - sys.modules.pop(name, None) - else: - sys.modules[name] = original - raise - - def cleanup(): - sys.modules.pop("fastdeploy.input.text_processor", None) - for name, original in previous_modules.items(): - if original is None: - sys.modules.pop(name, None) - else: - sys.modules[name] = original - - return text_processor_module, cleanup - - -class DummyRequest: - def __init__(self, **kwargs): - self.request_id = kwargs.get("request_id", "req") - self.prompt = kwargs.get("prompt") - self.prompt_token_ids = kwargs.get("prompt_token_ids") - self.messages = kwargs.get("messages") - self.eos_token_ids = kwargs.get("eos_token_ids") - self.chat_template = kwargs.get("chat_template") - self.enable_thinking = kwargs.get("enable_thinking") - self.history = kwargs.get("history") - self.tools = kwargs.get("tools") - self.system = kwargs.get("system") - self.sampling_params = SimpleNamespace( - top_p=kwargs.get("top_p"), - temperature=kwargs.get("temperature"), - repetition_penalty=kwargs.get("repetition_penalty"), - frequency_penalty=kwargs.get("frequency_penalty"), - presence_penalty=kwargs.get("presence_penalty"), - stop=kwargs.get("stop"), - stop_token_ids=kwargs.get("stop_token_ids"), - stop_seqs_len=kwargs.get("stop_seqs_len"), - bad_words=kwargs.get("bad_words"), - bad_words_token_ids=kwargs.get("bad_words_token_ids"), - max_tokens=kwargs.get("max_tokens"), - ) - - def get(self, key, default=None): - if hasattr(self, key) and getattr(self, key) is not None: - return getattr(self, key) - return getattr(self.sampling_params, key, default) - - def set(self, key, value): - if hasattr(self.sampling_params, key): - setattr(self.sampling_params, key, value) - else: - setattr(self, key, value) - - def to_dict(self): - return { - "request_id": self.request_id, - "messages": self.messages, - "prompt": self.prompt, - "system": self.system, - "history": self.history, - "tools": self.tools, - "chat_template": self.chat_template, - "enable_thinking": self.enable_thinking, - } - - def __getitem__(self, key): - return self.get(key) - - def __setitem__(self, key, value): - self.set(key, value) - - -class DataProcessorTestCase(unittest.TestCase): - @staticmethod - def create_dummy_reasoning(tokenizer, reasoning_content="think"): - class DummyReasoning: - def __init__(self, tokenizer): - self.tokenizer = tokenizer - - def extract_reasoning_content(self, full_text, response_dict, model_status): - return reasoning_content, f"{full_text}!" - - return DummyReasoning(tokenizer) - - @staticmethod - def create_dummy_tool_parser(tokenizer, content="tool-text"): - class DummyToolParser: - def __init__(self, tokenizer): - self.tokenizer = tokenizer - - def extract_tool_calls(self, full_text, response_dict): - return SimpleNamespace(tools_called=True, tool_calls=["tool"], content=content) - - return DummyToolParser - - def setUp(self): - module, cleanup = _import_text_processor() - self.text_processor_module = module - self.addCleanup(cleanup) - self.processor = self.text_processor_module.DataProcessor("stub-model") - - def test_base_data_processor_contract(self): - text_processor_module = self.text_processor_module - - class MinimalProcessor(text_processor_module.BaseDataProcessor): - def __init__(self): - self.generation_config = SimpleNamespace( - top_p=0.5, - temperature=0.6, - repetition_penalty=1.1, - frequency_penalty=0.2, - presence_penalty=0.3, - ) - super().__init__() - - def _load_tokenizer(self): - return DummyTokenizer() - - def process_request_dict(self, request, **kwargs): - return super().process_request_dict(request, **kwargs) - - def process_response_dict(self, response_obj): - return super().process_response_dict(response_obj) - - processor = MinimalProcessor() - request = Request(request_id="test_0", sampling_params=SamplingParams()) - defaults = processor._apply_default_parameters(request) - self.assertAlmostEqual(defaults.sampling_params.top_p, 0.5) - with self.assertRaises(NotImplementedError): - processor.process_request_dict({}, max_model_len=None) - with self.assertRaises(NotImplementedError): - processor.process_response_dict({}) - with self.assertRaises(NotImplementedError): - processor.text2ids("text") - with self.assertRaises(NotImplementedError): - processor.messages2ids([]) - with self.assertRaises(NotImplementedError): - processor.ids2tokens([1], "task") - - def test_process_request_dict_prompt_defaults(self): - request = {"request_id": "test_0", "prompt": "hi", "temperature": 0, "top_p": 0, "stop": ["stop"]} - request = Request.from_dict(request) - processed = self.processor.process_request_dict(request, max_model_len=5) - - self.assertEqual(processed.prompt_token_ids, [2]) - self.assertEqual(processed.sampling_params.stop_token_ids, [[4]]) - self.assertEqual(processed.sampling_params.stop_seqs_len, [1]) - self.assertEqual(processed.sampling_params.temperature, 1) - self.assertAlmostEqual(processed.sampling_params.top_p, 1e-5) - self.assertEqual(processed.sampling_params.max_tokens, 4) - - def test_process_request_dict_messages_template(self): - request = { - "request_id": "chat", - "messages": [{"role": "user", "content": "hello"}], - "chat_template_kwargs": {"system": "system prompt"}, - } - request = Request.from_dict(request) - request.chat_template_kwargs = {"system": "system prompt"} - processed = self.processor.process_request_dict(request, max_model_len=6) - - self.assertEqual(processed.prompt_token_ids, [len("system prompt hello")]) - self.assertEqual(processed.system, "system prompt") - self.assertTrue(processed.enable_thinking) - self.assertEqual(processed.prompt_tokens, "system prompt hello") - - def test_process_request_dictect_handles_sequences(self): - request = DummyRequest( - prompt=[1, 2, 3, 4, 5, 6], - stop=["stop"], - bad_words=["zz"], - temperature=0, - top_p=0, - ) - processed = self.processor.process_request_dict(request, max_model_len=5) - - self.assertEqual(processed.prompt_token_ids, [1, 2, 3, 4]) - self.assertEqual(processed.sampling_params.max_tokens, 1) - self.assertEqual(processed.sampling_params.stop_token_ids, [[4]]) - self.assertEqual(set(processed.sampling_params.bad_words_token_ids), {2, 3}) - self.assertEqual(processed.sampling_params.temperature, 1) - self.assertAlmostEqual(processed.sampling_params.top_p, 1e-5) - - def test_process_request_requires_prompt_or_messages(self): - request = DummyRequest(prompt=None, messages=None, prompt_token_ids=None) - with self.assertRaisesRegex(ValueError, "Request must contain 'prompt_token_ids', 'prompt', or 'messages'"): - self.processor.process_request_dict(request, max_model_len=5) - - def test_process_request_dict_rejects_bad_kwargs(self): - request = { - "request_id": "test_0", - "messages": [{"role": "user", "content": "hi"}], - "chat_template_kwargs": "invalid", - } - request = Request.from_dict(request) - request.chat_template_kwargs = "invalid" - request.sampling_params = SamplingParams() - with self.assertRaisesRegex(ValueError, "chat_template_kwargs must be a dict"): - self.processor.process_request_dict(request) - - def test_ids2tokens_and_clear_request_status(self): - delta, _, _ = self.processor.ids2tokens([3], "task-1") - self.assertEqual(delta, "3") - delta, _, _ = self.processor.ids2tokens([4], "task-1") - self.assertEqual(delta, "4") - - combined = self.processor.clear_request_status("task-1") - self.assertEqual(combined, "34") - self.assertNotIn("task-1", self.processor.decode_status) - - def test_clear_request_status_hf_branch(self): - module, cleanup = _import_text_processor(use_hf_tokenizer=True) - self.addCleanup(cleanup) - processor = module.DataProcessor("stub-model") - processor.decode_status = {"task": [[], [], "transcript"]} - - self.assertEqual(processor.clear_request_status("task"), "transcript") - self.assertNotIn("task", processor.decode_status) - - def test_data_processor_init_handles_missing_generation_config(self): - with mock.patch.object( - self.text_processor_module.GenerationConfig, - "from_pretrained", - side_effect=OSError("missing"), - ): - processor = self.text_processor_module.DataProcessor("stub-model") - self.assertIsNone(processor.generation_config) - - def test_process_response_with_reasoning_and_tools(self): - processor = self.processor - processor.model_status_dict = {"resp": "normal"} - - processor.reasoning_parser = self.create_dummy_reasoning(processor.tokenizer) - processor.tool_parser_obj = self.create_dummy_tool_parser(processor.tokenizer, content="tool-only") - - response = SimpleNamespace( - request_id="resp", outputs=SimpleNamespace(token_ids=[1, processor.tokenizer.eos_token_id]), finished=True - ) - - processed = processor.process_response_obj_normal(response) - self.assertEqual(processed.outputs.text, "tool-only") - self.assertEqual(processed.outputs.reasoning_content, "think") - self.assertEqual(processed.outputs.tool_calls, ["tool"]) - - def test_process_response_streaming_clears_state(self): - processor = self.processor - req_id = "stream" - processor.decode_status[req_id] = [0, 0, [], ""] - response = {"finished": True, "request_id": req_id, "outputs": {"token_ids": [7]}} - response = RequestOutput.from_dict(response) - - result = processor.process_response_obj_streaming(response, enable_thinking=False) - self.assertEqual(result.outputs.text, "7") - self.assertNotIn(req_id, processor.decode_status) - - def test_process_response_obj_normal_with_reasoning(self): - processor = self.processor - processor.model_status_dict = {"normal": "normal"} - processor.reasoning_parser = self.create_dummy_reasoning(processor.tokenizer, reasoning_content="because") - processor.tool_parser_obj = self.create_dummy_tool_parser(processor.tokenizer, content="tool-text") - - response = { - "finished": True, - "request_id": "normal", - "outputs": {"token_ids": [7, processor.tokenizer.eos_token_id]}, - } - response = RequestOutput.from_dict(response) - - result = processor.process_response_obj_normal(response, enable_thinking=True) - self.assertEqual(result.outputs.completion_tokens, "7") - self.assertEqual(result.outputs.text, "tool-text") - self.assertEqual(result.outputs.reasoning_content, "because") - self.assertEqual(result.outputs.reasoning_token_num, 1) - - def test_process_response_dict_dispatch(self): - processor = self.processor - calls = {} - - def fake_stream(response_obj, **kwargs): - calls["stream"] = kwargs - return "stream" - - def fake_normal(response_obj, **kwargs): - calls["normal"] = kwargs - return "normal" - - original_stream = processor.process_response_obj_streaming - original_normal = processor.process_response_obj_normal - processor.process_response_obj_streaming = fake_stream - processor.process_response_obj_normal = fake_normal - self.addCleanup(lambda: setattr(processor, "process_response_obj_streaming", original_stream)) - self.addCleanup(lambda: setattr(processor, "process_response_obj_normal", original_normal)) - - response = {"outputs": {}, "finished": False, "request_id": "req"} - self.assertEqual(processor.process_response_dict(response, stream=True, enable_thinking=True), "stream") - self.assertTrue(calls["stream"]["enable_thinking"]) - self.assertEqual( - processor.process_response_dict(response, stream=False, enable_thinking=True), - "normal", - ) - self.assertTrue(calls["normal"]["enable_thinking"]) - - def test_update_stop_seq_excludes_eos(self): - stop_seqs, stop_len = self.processor.update_stop_seq(["stop", self.processor.tokenizer.eos_token_id]) - self.assertEqual(stop_seqs, [[4]]) - self.assertEqual(stop_len, [1]) - - def test_pad_batch_data_left_padding(self): - padded, lengths = self.processor.pad_batch_data( - [[1], [2, 3]], - pad_id=-1, - return_seq_len=True, - return_array=False, - pad_style="left", - ) - self.assertEqual(padded, [[-1, 1], [2, 3]]) - self.assertEqual(lengths, [1, 2]) - - def test_pad_batch_data_empty_returns_array(self): - padded, lengths = self.processor.pad_batch_data([], return_seq_len=True) - self.assertEqual(padded.shape, (1, 0)) - self.assertEqual(lengths.shape, (0,)) - - def test_get_pad_id_prefers_eos_when_missing(self): - processor = self.text_processor_module.DataProcessor("stub-model") - llama_tokenizer = DummyLlamaTokenizer() - llama_tokenizer.pad_token_id = None - llama_tokenizer.eos_token = 99 - processor.tokenizer = llama_tokenizer - - self.assertEqual(processor.get_pad_id(), 99) - - def test_load_tokenizer_hf_branch(self): - module, cleanup = _import_text_processor(use_hf_tokenizer=True) - self.addCleanup(cleanup) - processor = module.DataProcessor("stub-model") - self.assertIsInstance(processor.tokenizer, DummyTokenizer) - - def test_text2ids_hf_branch(self): - module, cleanup = _import_text_processor(use_hf_tokenizer=True) - self.addCleanup(cleanup) - processor = module.DataProcessor("stub-model") - ids = processor.text2ids("hi", max_model_len=5) - self.assertEqual(ids.tolist(), [2, 0, 0, 0, 0][: len(ids)]) - - def test_process_logprob_response(self): - self.assertEqual(self.processor.process_logprob_response([1, 2]), "1 2") - - def test_process_request_dict_uses_existing_ids(self): - request = {"request_id": "test_0", "prompt_token_ids": [1, 2, 3], "max_tokens": 5} - request = Request.from_dict(request) - processed = self.processor.process_request_dict(request, max_model_len=6) - self.assertEqual(processed.prompt_token_ids, [1, 2, 3]) - self.assertEqual(processed.sampling_params.max_tokens, 3) - - def test_process_request_dict_requires_chat_template(self): - original_template = self.processor.tokenizer.chat_template - self.processor.tokenizer.chat_template = None - self.addCleanup(lambda: setattr(self.processor.tokenizer, "chat_template", original_template)) - with self.assertRaisesRegex(ValueError, "chat_template"): - request = {"request_id": "test_0", "messages": [{"role": "user", "content": "hi"}]} - request = Request.from_dict(request) - self.processor.process_request_dict(request) - - def test_update_bad_words_with_warnings(self): - processor = self.processor - - def custom_tokenize(text): - base = text.strip() - if base == "combo": - return ["co", "mbo"] - if base == "oversize": - return [base] - return [base] - - def custom_convert(tokens): - if tokens == ["co", "mbo"]: - return [1, 2] - if tokens == ["oversize"]: - return [processor.tokenizer.vocab_size + 1] - return [len(tokens[0])] - - original_tokenize = processor.tokenizer.tokenize - original_convert = processor.tokenizer.convert_tokens_to_ids - processor.tokenizer.tokenize = custom_tokenize - processor.tokenizer.convert_tokens_to_ids = custom_convert - self.addCleanup(lambda: setattr(processor.tokenizer, "tokenize", original_tokenize)) - self.addCleanup(lambda: setattr(processor.tokenizer, "convert_tokens_to_ids", original_convert)) - - self.assertEqual(processor.update_bad_words(["combo", "oversize"], []), []) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/input/v1/test_tokenizer_client.py b/tests/input/v1/test_tokenizer_client.py deleted file mode 100644 index 06804ebade3..00000000000 --- a/tests/input/v1/test_tokenizer_client.py +++ /dev/null @@ -1,101 +0,0 @@ -""" -# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -import httpx -import pytest -import respx - -from fastdeploy.input.tokenzier_client import ( - AsyncTokenizerClient, - ImageEncodeRequest, - VideoEncodeRequest, -) - - -@pytest.mark.asyncio -@respx.mock -async def test_encode_image_success(): - base_url = "http://testserver" - client = AsyncTokenizerClient(base_url=base_url) - - # Mock 创建任务接口 - respx.post(f"{base_url}/image/encode").mock( - return_value=httpx.Response(200, json={"code": 0, "task_tag": "task123"}) - ) - # Mock 轮询接口，返回完成状态 - mock_get_ret = { - "state": "Finished", - "result": {"feature_url": "bos://host:port/key", "feature_shape": [80, 45, 1563]}, - } - respx.get(f"{base_url}/encode/get").mock(return_value=httpx.Response(200, json=mock_get_ret)) - - request = ImageEncodeRequest( - version="v1", req_id="req_img_001", is_gen=False, resolution=512, image_url="http://example.com/image.jpg" - ) - - result = await client.encode_image(request) - assert result["feature_url"] == "bos://host:port/key" - assert result["feature_shape"] == [80, 45, 1563] - - -@pytest.mark.asyncio -@respx.mock -async def test_encode_video_failure(): - base_url = "http://testserver" - client = AsyncTokenizerClient(base_url=base_url, max_wait=1) - - respx.post(f"{base_url}/video/encode").mock( - return_value=httpx.Response(200, json={"code": 0, "task_tag": "task_vid_001"}) - ) - # 模拟轮询接口失败状态 - respx.get(f"{base_url}/encode/get").mock( - return_value=httpx.Response(200, json={"state": "Error", "message": "Encode failed"}) - ) - - request = VideoEncodeRequest( - version="v1", - req_id="req_vid_001", - is_gen=True, - resolution=720, - video_url="http://example.com/video.mp4", - start_ts=0.0, - end_ts=10.0, - frames=30, - vit_merge=True, - ) - - with pytest.raises(RuntimeError, match="Encode failed"): - await client.encode_video(request) - - -@pytest.mark.asyncio -@respx.mock -async def test_encode_timeout(): - base_url = "http://testserver" - client = AsyncTokenizerClient(base_url=base_url, max_wait=1, poll_interval=0.1) - - respx.post(f"{base_url}/image/encode").mock( - return_value=httpx.Response(200, json={"code": 0, "task_tag": "task_timeout"}) - ) - # 模拟轮询接口一直返回等待状态，导致超时 - respx.get(f"{base_url}/encode/get").mock(return_value=httpx.Response(200, json={"status": "processing"})) - - request = ImageEncodeRequest( - version="v1", req_id="req_img_timeout", is_gen=False, resolution=256, image_url="http://example.com/image.jpg" - ) - - with pytest.raises(TimeoutError): - await client.encode_image(request) diff --git a/tests/inter_communicator/test_zmq_server.py b/tests/inter_communicator/test_zmq_server.py index 629551b1707..57c9a0c479a 100644 --- a/tests/inter_communicator/test_zmq_server.py +++ b/tests/inter_communicator/test_zmq_server.py @@ -9,7 +9,6 @@ import types import unittest from collections import defaultdict -from multiprocessing.reduction import ForkingPickler from unittest import mock import msgpack @@ -264,19 +263,6 @@ def send(self, msg, flags=0, **kwargs): with self.assertRaises(RuntimeError): server.send_pyobj({"boom": True}) - def test_pack_aggregated_data_respects_env_flag(self): - server = _DummyServer() - responses = [_DummyResponse(1), _DummyResponse(2, finished=True)] - with mock.patch.object(envs, "ENABLE_V1_DATA_PROCESSOR", False): - packed = server.pack_aggregated_data(responses) - unpacked = ForkingPickler.loads(packed) - self.assertEqual(unpacked[0]["tensor_sum"], 3) - - with mock.patch.object(envs, "ENABLE_V1_DATA_PROCESSOR", True): - packed = server.pack_aggregated_data(responses) - unpacked = ForkingPickler.loads(packed) - self.assertIsInstance(unpacked[0], _DummyResponse) - def test_receive_json_once_paths(self): fake_socket = _FakeSocket() fake_socket.closed = True @@ -360,8 +346,7 @@ def test_send_response_per_query_cache_and_flush(self): self.assertIn(req_id, server.cached_results) server.req_dict[req_id] = b"client" - with mock.patch.object(envs, "ENABLE_V1_DATA_PROCESSOR", False): - server._send_response_per_query(req_id, [_DummyResponse(4, finished=True)]) + server._send_response_per_query(req_id, [_DummyResponse(4, finished=True)]) self.assertNotIn(req_id, server.req_dict) self.assertEqual(fake_socket.sent[-1][0], "send_multipart") @@ -370,17 +355,7 @@ def test_send_response_per_query_aggregate(self): server = _DummyServer(socket=fake_socket) server.req_dict["req-agg"] = b"client" server.aggregate_send = True - with mock.patch.object(envs, "ENABLE_V1_DATA_PROCESSOR", False): - server._send_response_per_query("req-agg", [_DummyResponse(5, finished=True)]) - self.assertEqual(fake_socket.sent[-1][0], "send_multipart") - - def test_send_response_per_query_v1_processor(self): - fake_socket = _FakeSocket() - server = _DummyServer(socket=fake_socket) - server.req_dict["req-v1"] = b"client" - server.aggregate_send = False - with mock.patch.object(envs, "ENABLE_V1_DATA_PROCESSOR", True): - server._send_response_per_query("req-v1", [_DummyResponse(6, finished=True)]) + server._send_response_per_query("req-agg", [_DummyResponse(5, finished=True)]) self.assertEqual(fake_socket.sent[-1][0], "send_multipart") def test_send_response_per_query_send_failure(self): @@ -391,8 +366,7 @@ def send_multipart(self, parts, copy=True): server = _DummyServer(socket=_ErrorSocket()) server.req_dict["req-error"] = b"client" server.aggregate_send = False - with mock.patch.object(envs, "ENABLE_V1_DATA_PROCESSOR", False): - server._send_response_per_query("req-error", [_DummyResponse(7, finished=True)]) + server._send_response_per_query("req-error", [_DummyResponse(7, finished=True)]) self.assertEqual(server.req_dict, {}) def test_send_response_per_query_raises_without_socket(self): @@ -436,22 +410,11 @@ def test_send_batch_response_success(self): fake_socket = _FakeSocket() server = _DummyServer(socket=fake_socket) server.address = "test-address" - with mock.patch.object(envs, "ENABLE_V1_DATA_PROCESSOR", False): - batch_data = [[_DummyResponse(1, finished=True)]] - server._send_batch_response(batch_data) + batch_data = [[_DummyResponse(1, finished=True)]] + server._send_batch_response(batch_data) self.assertEqual(len(fake_socket.sent), 1) self.assertEqual(fake_socket.sent[0][0], "send") - def test_send_batch_response_v1_processor(self): - """Test _send_batch_response with ENABLE_V1_DATA_PROCESSOR=True""" - fake_socket = _FakeSocket() - server = _DummyServer(socket=fake_socket) - server.address = "test-address" - with mock.patch.object(envs, "ENABLE_V1_DATA_PROCESSOR", True): - batch_data = [[_DummyResponse(1, finished=True)]] - server._send_batch_response(batch_data) - self.assertEqual(len(fake_socket.sent), 1) - def test_send_batch_response_raises_without_socket(self): """Test _send_batch_response logs error and returns when socket is None""" server = _DummyServer(socket=None) @@ -470,9 +433,8 @@ def send(self, msg, flags=0, **kwargs): server = _DummyServer(socket=_ErrorSocket()) server.address = "test-address" batch_data = [[_DummyResponse(1)]] - with mock.patch.object(envs, "ENABLE_V1_DATA_PROCESSOR", False): - # Should not raise, error is caught and logged - server._send_batch_response(batch_data) + # Should not raise, error is caught and logged + server._send_batch_response(batch_data) def test_recv_result_handle_paths(self): fake_socket = _FakeSocket() @@ -640,10 +602,9 @@ def test_send_batch_response_with_worker_pid_none_uses_default_socket(self): server = _DummyServer(socket=fake_socket) server.address = "test-address" - with mock.patch.object(envs, "ENABLE_V1_DATA_PROCESSOR", False): - batch_data = [[_DummyResponse(1, finished=True)]] - # worker_pid=None -> goes to the else branch that calls _ensure_socket / uses self.socket - server._send_batch_response(batch_data, worker_pid=None) + batch_data = [[_DummyResponse(1, finished=True)]] + # worker_pid=None -> goes to the else branch that calls _ensure_socket / uses self.socket + server._send_batch_response(batch_data, worker_pid=None) # The default socket should have been used to send the data self.assertEqual(len(fake_socket.sent), 1) From 81a322439697c95aba972430d585c4a1b520f3e4 Mon Sep 17 00:00:00 2001 From: luukunn <981429396@qq.com> Date: Fri, 27 Mar 2026 19:36:20 +0800 Subject: [PATCH 2/2] fix unit test --- tests/model_executor/test_thinking_budget.py | 299 ------------------- 1 file changed, 299 deletions(-) diff --git a/tests/model_executor/test_thinking_budget.py b/tests/model_executor/test_thinking_budget.py index 8ba9319ff7d..d9dd8f4b9b9 100644 --- a/tests/model_executor/test_thinking_budget.py +++ b/tests/model_executor/test_thinking_budget.py @@ -27,13 +27,6 @@ Ernie4_5_VLProcessor as ErnieVLDataProcessor, ) from fastdeploy.input.text_processor import DataProcessor as TextDataProcessor -from fastdeploy.input.v1.ernie4_5_processor import ( - Ernie4_5Processor as V1ErnieTextDataProcessor, -) -from fastdeploy.input.v1.ernie4_5_vl_processor.ernie4_5_vl_processor import ( - Ernie4_5_VLProcessor as V1ErnieVLDataProcessor, -) -from fastdeploy.input.v1.text_processor import DataProcessor as V1TextDataProcessor from fastdeploy.model_executor.logits_processor import ThinkingBudgetLogitsProcessor from fastdeploy.scheduler import SchedulerConfig @@ -711,31 +704,6 @@ def parallel_config(self): ips = None -class DummyRequestV1(SimpleNamespace): - def get(self, key, default=None): - if hasattr(self, key): - value = getattr(self, key) - if value is not None: - return value - if hasattr(self, "sampling_params") and hasattr(self.sampling_params, key): - value = getattr(self.sampling_params, key) - if value is not None: - return value - return default - - def __getitem__(self, key): - return getattr(self, key) - - def __setitem__(self, key, value): - setattr(self, key, value) - - def set(self, key, value): - if hasattr(self, "sampling_params") and hasattr(self.sampling_params, key): - setattr(self.sampling_params, key, value) - else: - setattr(self, key, value) - - class TestThinkingBudgetSupplemental(unittest.TestCase): def test_update_thinking_prompt_state_from_text_processor(self): processor = TextDataProcessor.__new__(TextDataProcessor) @@ -750,43 +718,6 @@ def test_update_thinking_prompt_state_from_text_processor(self): self.assertEqual(updated["think_prompt_tokens_after_start"], 0) self.assertEqual(updated["think_prompt_last_token_id"], 3) - def test_v1_process_request_missing_logits_processors_args(self): - processor = V1TextDataProcessor.__new__(V1TextDataProcessor) - processor.generation_config = SimpleNamespace( - top_p=0.7, - temperature=1.0, - repetition_penalty=1.0, - frequency_penalty=0.0, - presence_penalty=0.0, - ) - processor.eos_token_ids = [1] - processor.update_stop_seq = lambda *args, **kwargs: None - processor.update_bad_words = lambda bad_words, bad_words_token_ids: bad_words_token_ids - processor.encode_with_cache = lambda *args, **kwargs: [1] - processor._update_thinking_prompt_state = lambda prompt_token_ids, args: args - processor.reasoning_parser = None - request = DummyRequestV1( - request_id="req", - eos_token_ids=None, - prompt_token_ids=[1], - prompt=None, - messages=None, - max_tokens=1, - chat_template_kwargs=None, - sampling_params=SimpleNamespace( - bad_words=None, - bad_words_token_ids=None, - max_tokens=1, - temperature=1.0, - top_p=0.9, - repetition_penalty=1.0, - frequency_penalty=0.0, - presence_penalty=0.0, - ), - ) - with patch("fastdeploy.input.v1.text_processor.process_stop_token_ids", lambda *args, **kwargs: None): - processor.process_request(request, max_model_len=8) - def test_engine_line_break_id_from_dict(self): tokenizer = DummyTokenizerForTextProcessor() data_processor = SimpleNamespace(tokenizer=tokenizer, eos_token_id_len=1, pad_token_id=0) @@ -835,27 +766,6 @@ def _text2ids(text, max_model_len=None, add_special_tokens=False): self.assertEqual(processor.encode_with_cache("iter"), [21, 22]) self.assertNotIn(("np", False), processor._tokenize_cache) - def test_v1_encode_with_cache_branches(self): - processor = V1TextDataProcessor.__new__(V1TextDataProcessor) - processor._tokenize_cache = OrderedDict() - processor._tokenize_cache_capacity = 1 - call_counter = {"np": 0, "iter": 0} - - def _text2ids(text, max_model_len=None, add_special_tokens=False): - if text == "np": - call_counter["np"] += 1 - return np.array([31, 32], dtype=np.int64) - call_counter["iter"] += 1 - return (v for v in [41, 42]) - - processor.text2ids = _text2ids - - self.assertEqual(processor.encode_with_cache("np"), [31, 32]) - self.assertEqual(processor.encode_with_cache("np"), [31, 32]) - self.assertEqual(call_counter["np"], 1) - self.assertEqual(processor.encode_with_cache("iter"), [41, 42]) - self.assertNotIn(("np", False), processor._tokenize_cache) - def test_text_encode_with_cache_lazy_init(self): processor = TextDataProcessor.__new__(TextDataProcessor) call_counter = {"count": 0} @@ -872,22 +782,6 @@ def _text2ids(text, max_model_len=None, add_special_tokens=False): self.assertEqual(processor.encode_with_cache("lazy"), [51, 52]) self.assertEqual(call_counter["count"], 1) - def test_v1_encode_with_cache_lazy_init(self): - processor = V1TextDataProcessor.__new__(V1TextDataProcessor) - call_counter = {"count": 0} - - def _text2ids(text, max_model_len=None, add_special_tokens=False): - call_counter["count"] += 1 - return np.array([61, 62], dtype=np.int64) - - processor.text2ids = _text2ids - - self.assertFalse(hasattr(processor, "_tokenize_cache")) - self.assertEqual(processor.encode_with_cache("lazy"), [61, 62]) - self.assertTrue(hasattr(processor, "_tokenize_cache")) - self.assertEqual(processor.encode_with_cache("lazy"), [61, 62]) - self.assertEqual(call_counter["count"], 1) - def test_ernie_encode_literal_text_with_cache(self): processor = ErnieTextDataProcessor.__new__(ErnieTextDataProcessor) processor.tokenizer = SimpleNamespace( @@ -898,16 +792,6 @@ def test_ernie_encode_literal_text_with_cache(self): self.assertEqual(processor._encode_literal_text_with_cache("fallback"), [71, 72]) self.assertEqual(processor._encode_literal_text_with_cache("fallback"), [71, 72]) - def test_v1_ernie_encode_literal_text_with_cache(self): - processor = V1ErnieTextDataProcessor.__new__(V1ErnieTextDataProcessor) - processor.tokenizer = SimpleNamespace( - tokenize=lambda text: ["token_c", "token_d"], - convert_tokens_to_ids=lambda tokens: [81, 82], - ) - - self.assertEqual(processor._encode_literal_text_with_cache("fallback"), [81, 82]) - self.assertEqual(processor._encode_literal_text_with_cache("fallback"), [81, 82]) - def test_text_update_thinking_prompt_state_branches(self): processor = TextDataProcessor.__new__(TextDataProcessor) processor._think_token_ids = None @@ -949,29 +833,6 @@ def test_text_update_thinking_prompt_state_branches(self): # 命中 _get_think_token_ids 的缓存分支 self.assertEqual(processor._get_think_token_ids(), (THINKING_START_TOKEN_ID, THINKING_END_TOKEN_ID)) - def test_v1_update_thinking_prompt_state_branches(self): - processor = V1TextDataProcessor.__new__(V1TextDataProcessor) - processor._think_token_ids = None - processor.tokenizer = DummyTokenizerForTextProcessor() - - self.assertEqual(processor._update_thinking_prompt_state([1], "not-dict"), "not-dict") - self.assertEqual( - processor._update_thinking_prompt_state([1], {"thinking_budget": -1}), {"thinking_budget": -1} - ) - self.assertEqual(processor._update_thinking_prompt_state(None, {"thinking_budget": 1}), {"thinking_budget": 1}) - - with_start_no_end = processor._update_thinking_prompt_state( - np.array([1, THINKING_START_TOKEN_ID, 2, 3], dtype=np.int64), - {"thinking_budget": 4}, - ) - self.assertTrue(with_start_no_end["think_prompt_started"]) - self.assertFalse(with_start_no_end["think_prompt_ended"]) - self.assertEqual(with_start_no_end["think_prompt_tokens_after_start"], 0) - self.assertEqual(with_start_no_end["think_prompt_last_token_id"], 3) - - # 命中 _get_think_token_ids 的缓存分支 - self.assertEqual(processor._get_think_token_ids(), (THINKING_START_TOKEN_ID, THINKING_END_TOKEN_ID)) - def test_text_process_request_dict_think_stop_sentence(self): processor = TextDataProcessor.__new__(TextDataProcessor) processor._apply_default_parameters = lambda request: request @@ -1003,74 +864,6 @@ def test_text_process_request_dict_think_stop_sentence(self): ) self.assertNotIn("think_stop_sentence", processed["logits_processors_args"]) - def test_v1_process_request_think_stop_sentence(self): - processor = V1TextDataProcessor.__new__(V1TextDataProcessor) - processor._apply_default_parameters = lambda request: request - processor.eos_token_ids = [1] - processor.update_stop_seq = lambda *args, **kwargs: None - processor.update_bad_words = lambda bad_words, bad_words_token_ids: bad_words_token_ids - processor._encode_literal_text_with_cache = lambda text: [301, 302] - processor._update_thinking_prompt_state = lambda prompt_token_ids, args: args - processor.reasoning_parser = None - - request = DummyRequestV1( - request_id="req_v1", - eos_token_ids=[1], - prompt_token_ids=[10], - prompt=None, - messages=None, - logits_processors_args={"thinking_budget": 20, "think_stop_sentence": "done"}, - bad_words=None, - bad_words_token_ids=None, - max_tokens=1, - temperature=1.0, - top_p=0.9, - ) - with patch("fastdeploy.input.v1.text_processor.process_stop_token_ids", lambda *args, **kwargs: None): - processed = processor.process_request(request, max_model_len=16) - self.assertEqual( - processed.logits_processors_args.get("think_stop_sentence_token_ids"), - [301, 302], - ) - self.assertNotIn("think_stop_sentence", processed.logits_processors_args) - - def test_v1_process_request_dict_think_stop_sentence(self): - processor = V1TextDataProcessor.__new__(V1TextDataProcessor) - processor._apply_default_parameters = lambda request: request - processor.eos_token_ids = [1] - processor.update_stop_seq = lambda *args, **kwargs: None - processor.update_bad_words = lambda bad_words, bad_words_token_ids: bad_words_token_ids - processor._encode_literal_text_with_cache = lambda text: [401, 402] - processor._update_thinking_prompt_state = lambda prompt_token_ids, args: args - processor.reasoning_parser = None - - request = DummyRequestV1( - request_id="req_v1_dict", - eos_token_ids=[1], - prompt_token_ids=[11], - prompt=None, - messages=None, - chat_template_kwargs=None, - sampling_params=SimpleNamespace( - bad_words=None, - bad_words_token_ids=None, - max_tokens=1, - temperature=1.0, - top_p=0.9, - repetition_penalty=1.0, - frequency_penalty=0.0, - presence_penalty=0.0, - logits_processors_args={"thinking_budget": 20, "think_stop_sentence": "done"}, - ), - ) - with patch("fastdeploy.input.v1.text_processor.process_stop_token_ids", lambda *args, **kwargs: None): - processed = processor.process_request_dict(request, max_model_len=16) - self.assertEqual( - processed.sampling_params.logits_processors_args.get("think_stop_sentence_token_ids"), - [401, 402], - ) - self.assertNotIn("think_stop_sentence", processed.sampling_params.logits_processors_args) - def test_ernie_process_request_dict_prepares_thinking_budget_args(self): processor = ErnieTextDataProcessor.__new__(ErnieTextDataProcessor) processor._apply_default_parameters = lambda request: request @@ -1104,46 +897,6 @@ def test_ernie_process_request_dict_prepares_thinking_budget_args(self): self.assertFalse(processed["logits_processors_args"]["think_prompt_ended"]) self.assertEqual(processed["logits_processors_args"]["think_prompt_tokens_after_start"], 0) - def test_v1_ernie_process_request_dict_prepares_thinking_budget_args(self): - processor = V1ErnieTextDataProcessor.__new__(V1ErnieTextDataProcessor) - processor._apply_default_parameters = lambda request: request - processor.eos_token_ids = [1] - processor.update_stop_seq = lambda *args, **kwargs: None - processor.update_bad_words = lambda bad_words, bad_words_token_ids: bad_words_token_ids - processor._encode_literal_text_with_cache = lambda text: [601, 602] - processor.tokenizer = DummyTokenizerForTextProcessor() - processor.reasoning_parser = None - - request = DummyRequestV1( - request_id="req_v1_ernie_text", - eos_token_ids=[1], - prompt_token_ids=[1, THINKING_START_TOKEN_ID, 2], - prompt=None, - messages=None, - chat_template_kwargs=None, - enable_thinking=True, - sampling_params=SimpleNamespace( - bad_words=None, - bad_words_token_ids=None, - max_tokens=1, - temperature=1.0, - top_p=0.9, - repetition_penalty=1.0, - frequency_penalty=0.0, - presence_penalty=0.0, - response_max_tokens=None, - n=1, - logits_processors_args={"thinking_budget": 20, "think_stop_sentence": "done"}, - ), - ) - with patch("fastdeploy.input.v1.ernie4_5_processor.process_stop_token_ids", lambda *args, **kwargs: None): - processed = processor.process_request_dict(request, max_model_len=16) - - self.assertEqual(processed.sampling_params.logits_processors_args["think_stop_sentence_token_ids"], [601, 602]) - self.assertTrue(processed.sampling_params.logits_processors_args["think_prompt_started"]) - self.assertFalse(processed.sampling_params.logits_processors_args["think_prompt_ended"]) - self.assertEqual(processed.sampling_params.logits_processors_args["think_prompt_tokens_after_start"], 0) - def test_ernie_vl_process_request_dict_prepares_thinking_budget_args(self): processor = ErnieVLDataProcessor.__new__(ErnieVLDataProcessor) processor._apply_default_parameters = lambda request: request @@ -1182,58 +935,6 @@ def test_ernie_vl_process_request_dict_prepares_thinking_budget_args(self): self.assertFalse(processed["logits_processors_args"]["think_prompt_ended"]) self.assertEqual(processed["logits_processors_args"]["think_prompt_tokens_after_start"], 0) - def test_v1_ernie_vl_process_request_dict_prepares_thinking_budget_args(self): - processor = V1ErnieVLDataProcessor.__new__(V1ErnieVLDataProcessor) - processor._apply_default_parameters = lambda request: request - processor.eos_token_ids = [1] - processor.update_stop_seq = lambda *args, **kwargs: None - processor.update_bad_words = lambda bad_words, bad_words_token_ids: bad_words_token_ids - processor._encode_literal_text_with_cache = lambda text: [801, 802] - processor.tokenizer = DummyTokenizerForTextProcessor() - processor.reasoning_parser = None - processor._check_mm_limits = lambda *args, **kwargs: None - processor.append_completion_tokens = lambda *args, **kwargs: None - processor.pack_outputs = lambda outs: outs - processor.ernie4_5_processor = SimpleNamespace( - request2ids=lambda request: {"input_ids": np.array([1, THINKING_START_TOKEN_ID, 2], dtype=np.int64)} - ) - - request = DummyRequestV1( - request_id="req_v1_ernie_vl", - eos_token_ids=[1], - prompt_token_ids=None, - prompt=None, - messages=[{"role": "user", "content": "hi"}], - chat_template_kwargs=None, - enable_thinking=True, - completion_token_ids=None, - multimodal_data=None, - sampling_params=SimpleNamespace( - bad_words=None, - bad_words_token_ids=None, - max_tokens=1, - temperature=1.0, - top_p=0.9, - repetition_penalty=1.0, - frequency_penalty=0.0, - presence_penalty=0.0, - response_max_tokens=None, - reasoning_max_tokens=None, - n=1, - logits_processors_args={"thinking_budget": 20, "think_stop_sentence": "done"}, - ), - ) - with patch( - "fastdeploy.input.v1.ernie4_5_vl_processor.ernie4_5_vl_processor.process_stop_token_ids", - lambda *args, **kwargs: None, - ): - processed = processor.process_request_dict(request, max_model_len=16) - - self.assertEqual(processed.sampling_params.logits_processors_args["think_stop_sentence_token_ids"], [801, 802]) - self.assertTrue(processed.sampling_params.logits_processors_args["think_prompt_started"]) - self.assertFalse(processed.sampling_params.logits_processors_args["think_prompt_ended"]) - self.assertEqual(processed.sampling_params.logits_processors_args["think_prompt_tokens_after_start"], 0) - if __name__ == "__main__": unittest.main()