diff --git a/astrbot/core/config/default.py b/astrbot/core/config/default.py index 5bf7de7a45..29159d17f8 100644 --- a/astrbot/core/config/default.py +++ b/astrbot/core/config/default.py @@ -4149,7 +4149,7 @@ "provider_settings.image_compress_options.max_size": { "description": "最大边长", "type": "int", - "hint": "压缩后图片的最长边，单位为像素。超过该尺寸时会按比例缩放。", + "hint": "压缩后图片的最长边，单位为像素，超出则按比例缩放。CUA 沙箱下输入图片不缩放，大图可能超出服务商上传限制。", "condition": { "provider_settings.image_compress_enabled": True, }, diff --git a/astrbot/core/pipeline/process_stage/method/agent_sub_stages/image_input.py b/astrbot/core/pipeline/process_stage/method/agent_sub_stages/image_input.py index 002225c3c3..61e237b8bd 100644 --- a/astrbot/core/pipeline/process_stage/method/agent_sub_stages/image_input.py +++ b/astrbot/core/pipeline/process_stage/method/agent_sub_stages/image_input.py @@ -24,6 +24,7 @@ async def prepare_request_images( output_dir: Path, prepared: dict[str, str | None], quote_image_ref: str | None = None, + montage_max_size: int | None = None, ) -> None: """Replace current images on a working request and track their owned files. @@ -36,6 +37,7 @@ async def prepare_request_images( output_dir: Event working file directory, separate from the shared cache. prepared: Per-request mapping reused after the request hook. quote_image_ref: Optional input for the dedicated quote caption branch. + montage_max_size: Optional montage-specific limit; defaults to ``max_size``. """ req.image_urls = normalize_and_dedupe_strings(req.image_urls) refs = list(req.image_urls) @@ -52,7 +54,11 @@ async def prepare_request_images( path = None if enabled: path = await prepare_model_image( - ref, max_size=max_size, output_dir=output_dir, quality=quality + ref, + max_size=max_size, + output_dir=output_dir, + quality=quality, + montage_max_size=montage_max_size, ) if path: event.track_temporary_local_file(path) diff --git a/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py b/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py index c42d992cad..322506e3a8 100644 --- a/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py +++ b/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py @@ -62,6 +62,9 @@ ) from .image_input import prepare_request_images +# Anthropic rejects images above 5 MB; OpenAI and Gemini allow roughly 20 MB. +_CUA_IMAGE_WARN_BYTES = 5 * 1024 * 1024 + class InternalAgentSubStage(Stage): async def initialize(self, ctx: PipelineContext) -> None: @@ -243,9 +246,24 @@ async def process( settings = self.ctx.astrbot_config["provider_settings"] enabled = settings.get("image_compress_enabled", True) is not False options = settings.get("image_compress_options", {}) - max_size = normalize_model_image_max_size( + montage_max_size = normalize_model_image_max_size( options.get("max_size") if isinstance(options, dict) else None ) + max_size = montage_max_size + sandbox_cfg = settings.get("sandbox") + cua_pixel_mode = ( + settings.get("computer_use_runtime") == "sandbox" + and isinstance(sandbox_cfg, dict) + and sandbox_cfg.get("booter") == "cua" + ) + if cua_pixel_mode: + # CUA pixel tools read coordinates 1:1 on stills, so the + # still-image resize is lifted; compliant images pass through + # byte-exact since lossy re-encoding would shift colors. + # Montages are never used for coordinates and keep the + # configured cap, which bounds the 3x3 canvas. Oversized + # passthrough images warn below. + max_size = 1_000_000 quality = ( options.get("quality") if isinstance(options, dict) else None ) @@ -275,6 +293,7 @@ async def process( output_dir=output_dir, prepared=prepared, quote_image_ref=quote_image_ref, + montage_max_size=montage_max_size, ) await _process_quote_message( event, @@ -341,7 +360,26 @@ async def process( quality=quality, output_dir=output_dir, prepared=prepared, + montage_max_size=montage_max_size, ) + if cua_pixel_mode: + oversized = [] + for path in {p for p in prepared.values() if p}: + try: + size = Path(path).stat().st_size + except OSError: + continue + if size > _CUA_IMAGE_WARN_BYTES: + oversized.append(size) + if oversized: + logger.warning( + "CUA session sends %d image(s) larger than %d MB " + "(largest %.1f MB) without resize; this may exceed " + "provider image upload limits.", + len(oversized), + _CUA_IMAGE_WARN_BYTES // 1048576, + max(oversized) / 1048576, + ) # apply reset if reset_coro: await reset_coro diff --git a/astrbot/core/utils/media_utils.py b/astrbot/core/utils/media_utils.py index a722a4a7ec..ff552293d5 100644 --- a/astrbot/core/utils/media_utils.py +++ b/astrbot/core/utils/media_utils.py @@ -1366,14 +1366,20 @@ async def prepare_model_image( max_size: int, output_dir: Path, quality: int = IMAGE_COMPRESS_DEFAULT_QUALITY, + montage_max_size: int | None = None, ) -> str | None: """Prepare a single local model-ready image for the caller to own until consumption. Args: image_ref: Source reference accepted by MediaResolver. - max_size: Validated longest-edge limit for stills and animation montages. + max_size: Validated longest-edge limit for stills. output_dir: Directory for independent request-owned working files. quality: JPEG output quality in the range 1-100. + montage_max_size: Optional longest-edge limit for animation montages. + CUA sessions lift the still-image cap to keep pixel coordinates 1:1, + but montages are never used for coordinates, so callers pass the + configured limit here to keep the 3x3 canvas bounded. Defaults to + ``max_size``. Returns: An existing JPEG or PNG path, or None for a recoverable input or write @@ -1386,7 +1392,10 @@ async def prepare_model_image( frame_count = await asyncio.to_thread(_inspect_image, image_bytes) if frame_count > 1: converted_bytes, _ = await asyncio.to_thread( - _extract_animation_montage_sync, image_bytes, max_size, quality + _extract_animation_montage_sync, + image_bytes, + montage_max_size if montage_max_size is not None else max_size, + quality, ) else: converted_bytes = await asyncio.to_thread( diff --git a/docs/en/providers/image-formats.md b/docs/en/providers/image-formats.md index bf2f807ded..0c9fb96eca 100644 --- a/docs/en/providers/image-formats.md +++ b/docs/en/providers/image-formats.md @@ -10,6 +10,9 @@ - Stills and montages share `image_compress_options.max_size` (default 1280). Small images are not enlarged. - Disabling compression keeps generic localization and reading, without resizing, transcoding, sampling or consulting the derived-image cache. + +> [!TIP] +> When the computer-use runtime is `sandbox` and the sandbox booter is `cua`, input images are not resized, so pixel coordinates read by coordinate-based tools stay 1:1. Compliant images pass through byte-exact (no lossy re-encoding, avoiding JPEG color shifts); format conversion for other formats and animation montages still apply. Images above roughly 5 MB may exceed provider image upload limits and trigger a warning in the logs. The Agent receives readable local paths. Original image files and event components keep their original content, and attachment text continues to reference the source image. Providers only read/encode references and assemble their protocols. ## Errors and lifetime diff --git a/docs/zh/providers/image-formats.md b/docs/zh/providers/image-formats.md index 1138c1c6c8..4fe3997318 100644 --- a/docs/zh/providers/image-formats.md +++ b/docs/zh/providers/image-formats.md @@ -8,6 +8,9 @@ **最大边长**默认为 1280 像素，静图和动图拼图共用此设置，小图不会放大。**压缩质量**（1-100，默认 95）控制 JPEG 输出的画质与体积。 + +> [!TIP] +> 当“电脑使用”运行时为沙箱且沙箱 Booter 为 CUA 时，为避免像素坐标漂移，输入图片不按最大边长缩放，合规图片原样发送（不重新编码，避免 JPEG 变色）；格式转换与动图拼图等其余处理仍然生效。超过约 5 MB 的大图可能超出服务商的图片体积限制，此时日志会给出警告。 ## 处理方式 - **合规静图原样发送**：已是 JPEG/PNG、方向正常且未超过最大边长的图片不做任何改动。 diff --git a/tests/test_process_stage_images.py b/tests/test_process_stage_images.py index 8583b1f952..44c2aba77b 100644 --- a/tests/test_process_stage_images.py +++ b/tests/test_process_stage_images.py @@ -4,6 +4,7 @@ import base64 import copy import inspect +import os from contextlib import nullcontext from pathlib import Path from types import SimpleNamespace @@ -235,6 +236,97 @@ async def test_profile_toggle_and_preprocess_to_first_model( assert "image_settings" not in harness.provider.text_chat.await_args.kwargs +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("runtime", "booter", "fmt"), + [ + ("sandbox", "cua", "PNG"), + ("sandbox", "cua", "WEBP"), + ("sandbox", "shipyard_neo", "PNG"), + ("local", "cua", "PNG"), + ], +) +async def test_cua_runtime_keeps_input_image_geometry( + harness, tmp_path, runtime, booter, fmt +): + """CUA pixel tools read coordinates 1:1, so only the resize is lifted.""" + path = tmp_path / f"big.{fmt.lower()}" + PILImage.new("RGB", (200, 100), "red").save(path, fmt) + original = path.read_bytes() + harness.config["provider_settings"]["computer_use_runtime"] = runtime + harness.config["provider_settings"]["sandbox"] = {"booter": booter} + event = make_event([Image(file=str(path))], text="") + await process_event(harness, event, preprocess_first=True) + assert len(harness.captured) == 1 + req = harness.captured[0].req + gated = runtime == "sandbox" and booter == "cua" + with PILImage.open(req.image_urls[0]) as image: + assert image.size == ((200, 100) if gated else (90, 45)) + if gated: + # A compliant source is reused byte-exact; other formats are still + # re-encoded (to JPEG) without any resize. + expected = fmt if fmt in {"JPEG", "PNG"} else "JPEG" + assert image.format == expected + if gated and fmt == "PNG": + assert Path(req.image_urls[0]).read_bytes() == original + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("runtime", "booter", "big"), + [ + ("sandbox", "cua", True), + ("sandbox", "cua", False), + ("sandbox", "shipyard_neo", True), + ], +) +async def test_cua_oversize_image_warns( + harness, tmp_path, monkeypatch, runtime, booter, big +): + """Byte-exact CUA passthrough warns when an image may exceed upload limits.""" + dims = (1500, 1500) if big else (60, 30) + path = tmp_path / "shot.png" + if big: + # Noise stays incompressible, keeping the PNG above the 5 MB threshold. + PILImage.frombytes("RGB", dims, os.urandom(dims[0] * dims[1] * 3)).save( + path, "PNG" + ) + else: + PILImage.new("RGB", dims, "red").save(path, "PNG") + original = path.read_bytes() + harness.config["provider_settings"]["computer_use_runtime"] = runtime + harness.config["provider_settings"]["sandbox"] = {"booter": booter} + warning = MagicMock() + monkeypatch.setattr(internal.logger, "warning", warning) + event = make_event([Image(file=str(path))], text="") + await process_event(harness, event, preprocess_first=True) + assert len(harness.captured) == 1 + req = harness.captured[0].req + gated = runtime == "sandbox" and booter == "cua" + if gated: + assert Path(req.image_urls[0]).read_bytes() == original + warned = any("upload limits" in str(call) for call in warning.call_args_list) + assert warned == (big and gated) + + +@pytest.mark.asyncio +async def test_cua_montage_keeps_configured_cap(harness, tmp_path): + """Animated inputs keep the configured montage cap under CUA pixel mode.""" + path = tmp_path / "anim.gif" + frames = [PILImage.new("RGB", (600, 400), c) for c in ("red", "green", "blue")] + frames[0].save(path, "GIF", save_all=True, append_images=frames[1:]) + harness.config["provider_settings"]["computer_use_runtime"] = "sandbox" + harness.config["provider_settings"]["sandbox"] = {"booter": "cua"} + event = make_event([Image(file=str(path))], text="") + await process_event(harness, event, preprocess_first=True) + assert len(harness.captured) == 1 + req = harness.captured[0].req + with PILImage.open(req.image_urls[0]) as image: + # With the configured cap 90, 600x400 frames produce a 90x60 montage; + # the CUA still-image passthrough must not unbound the montage canvas. + assert max(image.size) <= 90 + + @pytest.mark.asyncio async def test_profile_reload_and_concurrent_requests(harness, tmp_path): source = source_image(tmp_path)