From a41f7edb18c823a51a0fbfa4dcfc97d1779aaaf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B0=95=E6=B0=99?= Date: Mon, 14 Sep 2026 02:44:44 +0800 Subject: [PATCH 1/3] fix: skip input image resize under CUA sandbox to keep pixel coordinates 1:1 --- astrbot/core/config/default.py | 2 +- .../method/agent_sub_stages/internal.py | 11 ++++++ docs/en/providers/image-formats.md | 3 ++ docs/zh/providers/image-formats.md | 3 ++ tests/test_process_stage_images.py | 35 +++++++++++++++++++ 5 files changed, 53 insertions(+), 1 deletion(-) diff --git a/astrbot/core/config/default.py b/astrbot/core/config/default.py index 5bf7de7a45..52b7143579 100644 --- a/astrbot/core/config/default.py +++ b/astrbot/core/config/default.py @@ -4149,7 +4149,7 @@ "provider_settings.image_compress_options.max_size": { "description": "最大边长", "type": "int", - "hint": "压缩后图片的最长边，单位为像素。超过该尺寸时会按比例缩放。", + "hint": "压缩后图片的最长边，单位为像素。超过该尺寸时会按比例缩放。电脑使用（CUA 沙箱）场景下输入图片不缩放，以保持像素坐标一致。", "condition": { "provider_settings.image_compress_enabled": True, }, diff --git a/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py b/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py index c42d992cad..4a6dd2f82c 100644 --- a/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py +++ b/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py @@ -246,6 +246,17 @@ async def process( max_size = normalize_model_image_max_size( options.get("max_size") if isinstance(options, dict) else None ) + sandbox_cfg = settings.get("sandbox") + if ( + settings.get("computer_use_runtime") == "sandbox" + and isinstance(sandbox_cfg, dict) + and sandbox_cfg.get("booter") == "cua" + ): + # CUA pixel tools read coordinates 1:1, so input images keep + # their geometry: only the long-edge resize is lifted, while + # format normalization, quality and oversized-PNG flattening + # still apply. + max_size = 1_000_000 quality = ( options.get("quality") if isinstance(options, dict) else None ) diff --git a/docs/en/providers/image-formats.md b/docs/en/providers/image-formats.md index bf2f807ded..c2727a6f7e 100644 --- a/docs/en/providers/image-formats.md +++ b/docs/en/providers/image-formats.md @@ -10,6 +10,9 @@ - Stills and montages share `image_compress_options.max_size` (default 1280). Small images are not enlarged. - Disabling compression keeps generic localization and reading, without resizing, transcoding, sampling or consulting the derived-image cache. + +> [!TIP] +> When the computer-use runtime is `sandbox` and the sandbox booter is `cua`, input images are not resized, so pixel coordinates read by coordinate-based tools stay 1:1. Format conversion, quality and animation montages still apply. Tool-result images such as CUA screenshots never go through this preparation. The Agent receives readable local paths. Original image files and event components keep their original content, and attachment text continues to reference the source image. Providers only read/encode references and assemble their protocols. ## Errors and lifetime diff --git a/docs/zh/providers/image-formats.md b/docs/zh/providers/image-formats.md index 1138c1c6c8..74b60933c8 100644 --- a/docs/zh/providers/image-formats.md +++ b/docs/zh/providers/image-formats.md @@ -8,6 +8,9 @@ **最大边长**默认为 1280 像素，静图和动图拼图共用此设置，小图不会放大。**压缩质量**（1-100，默认 95）控制 JPEG 输出的画质与体积。 + +> [!TIP] +> 当“电脑使用”运行时为沙箱且沙箱 Booter 为 CUA 时，为避免像素坐标漂移，输入图片不按最大边长缩放；格式转换、压缩质量与动图拼图等其余处理仍然生效。CUA 截图等工具回图本就不经过此功能。 ## 处理方式 - **合规静图原样发送**：已是 JPEG/PNG、方向正常且未超过最大边长的图片不做任何改动。 diff --git a/tests/test_process_stage_images.py b/tests/test_process_stage_images.py index 8583b1f952..c0ab044414 100644 --- a/tests/test_process_stage_images.py +++ b/tests/test_process_stage_images.py @@ -235,6 +235,41 @@ async def test_profile_toggle_and_preprocess_to_first_model( assert "image_settings" not in harness.provider.text_chat.await_args.kwargs +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("runtime", "booter", "fmt"), + [ + ("sandbox", "cua", "PNG"), + ("sandbox", "cua", "WEBP"), + ("sandbox", "shipyard_neo", "PNG"), + ("local", "cua", "PNG"), + ], +) +async def test_cua_runtime_keeps_input_image_geometry( + harness, tmp_path, runtime, booter, fmt +): + """CUA pixel tools read coordinates 1:1, so only the resize is lifted.""" + path = tmp_path / f"big.{fmt.lower()}" + PILImage.new("RGB", (200, 100), "red").save(path, fmt) + original = path.read_bytes() + harness.config["provider_settings"]["computer_use_runtime"] = runtime + harness.config["provider_settings"]["sandbox"] = {"booter": booter} + event = make_event([Image(file=str(path))], text="") + await process_event(harness, event, preprocess_first=True) + assert len(harness.captured) == 1 + req = harness.captured[0].req + gated = runtime == "sandbox" and booter == "cua" + with PILImage.open(req.image_urls[0]) as image: + assert image.size == ((200, 100) if gated else (90, 45)) + if gated: + # A compliant source is reused byte-exact; other formats are still + # re-encoded (to JPEG) without any resize. + expected = fmt if fmt in {"JPEG", "PNG"} else "JPEG" + assert image.format == expected + if gated and fmt == "PNG": + assert Path(req.image_urls[0]).read_bytes() == original + + @pytest.mark.asyncio async def test_profile_reload_and_concurrent_requests(harness, tmp_path): source = source_image(tmp_path) From 2d0ee979de082631c10402add96eb45d56376206 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B0=95=E6=B0=99?= Date: Mon, 14 Sep 2026 03:14:00 +0800 Subject: [PATCH 2/3] feat: warn when CUA passthrough images may exceed provider upload limits --- astrbot/core/config/default.py | 2 +- .../method/agent_sub_stages/internal.py | 35 ++++++++++++++--- docs/en/providers/image-formats.md | 2 +- docs/zh/providers/image-formats.md | 2 +- tests/test_process_stage_images.py | 39 +++++++++++++++++++ 5 files changed, 71 insertions(+), 9 deletions(-) diff --git a/astrbot/core/config/default.py b/astrbot/core/config/default.py index 52b7143579..29159d17f8 100644 --- a/astrbot/core/config/default.py +++ b/astrbot/core/config/default.py @@ -4149,7 +4149,7 @@ "provider_settings.image_compress_options.max_size": { "description": "最大边长", "type": "int", - "hint": "压缩后图片的最长边，单位为像素。超过该尺寸时会按比例缩放。电脑使用（CUA 沙箱）场景下输入图片不缩放，以保持像素坐标一致。", + "hint": "压缩后图片的最长边，单位为像素，超出则按比例缩放。CUA 沙箱下输入图片不缩放，大图可能超出服务商上传限制。", "condition": { "provider_settings.image_compress_enabled": True, }, diff --git a/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py b/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py index 4a6dd2f82c..0831a71fc3 100644 --- a/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py +++ b/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py @@ -62,6 +62,9 @@ ) from .image_input import prepare_request_images +# Anthropic rejects images above 5 MB; OpenAI and Gemini allow roughly 20 MB. +_CUA_IMAGE_WARN_BYTES = 5 * 1024 * 1024 + class InternalAgentSubStage(Stage): async def initialize(self, ctx: PipelineContext) -> None: @@ -247,15 +250,17 @@ async def process( options.get("max_size") if isinstance(options, dict) else None ) sandbox_cfg = settings.get("sandbox") - if ( + cua_pixel_mode = ( settings.get("computer_use_runtime") == "sandbox" and isinstance(sandbox_cfg, dict) and sandbox_cfg.get("booter") == "cua" - ): - # CUA pixel tools read coordinates 1:1, so input images keep - # their geometry: only the long-edge resize is lifted, while - # format normalization, quality and oversized-PNG flattening - # still apply. + ) + if cua_pixel_mode: + # CUA pixel tools read coordinates 1:1, so the long-edge + # resize is lifted; compliant images pass through byte-exact + # since lossy re-encoding would shift colors. Format + # normalization still applies to other formats, and oversized + # passthrough images warn below. max_size = 1_000_000 quality = ( options.get("quality") if isinstance(options, dict) else None @@ -353,6 +358,24 @@ async def process( output_dir=output_dir, prepared=prepared, ) + if cua_pixel_mode: + oversized = [] + for path in {p for p in prepared.values() if p}: + try: + size = Path(path).stat().st_size + except OSError: + continue + if size > _CUA_IMAGE_WARN_BYTES: + oversized.append(size) + if oversized: + logger.warning( + "CUA session sends %d image(s) larger than %d MB " + "(largest %.1f MB) without resize; this may exceed " + "provider image upload limits.", + len(oversized), + _CUA_IMAGE_WARN_BYTES // 1048576, + max(oversized) / 1048576, + ) # apply reset if reset_coro: await reset_coro diff --git a/docs/en/providers/image-formats.md b/docs/en/providers/image-formats.md index c2727a6f7e..0c9fb96eca 100644 --- a/docs/en/providers/image-formats.md +++ b/docs/en/providers/image-formats.md @@ -12,7 +12,7 @@ > [!TIP] -> When the computer-use runtime is `sandbox` and the sandbox booter is `cua`, input images are not resized, so pixel coordinates read by coordinate-based tools stay 1:1. Format conversion, quality and animation montages still apply. Tool-result images such as CUA screenshots never go through this preparation. +> When the computer-use runtime is `sandbox` and the sandbox booter is `cua`, input images are not resized, so pixel coordinates read by coordinate-based tools stay 1:1. Compliant images pass through byte-exact (no lossy re-encoding, avoiding JPEG color shifts); format conversion for other formats and animation montages still apply. Images above roughly 5 MB may exceed provider image upload limits and trigger a warning in the logs. The Agent receives readable local paths. Original image files and event components keep their original content, and attachment text continues to reference the source image. Providers only read/encode references and assemble their protocols. ## Errors and lifetime diff --git a/docs/zh/providers/image-formats.md b/docs/zh/providers/image-formats.md index 74b60933c8..4fe3997318 100644 --- a/docs/zh/providers/image-formats.md +++ b/docs/zh/providers/image-formats.md @@ -10,7 +10,7 @@ > [!TIP] -> 当“电脑使用”运行时为沙箱且沙箱 Booter 为 CUA 时，为避免像素坐标漂移，输入图片不按最大边长缩放；格式转换、压缩质量与动图拼图等其余处理仍然生效。CUA 截图等工具回图本就不经过此功能。 +> 当“电脑使用”运行时为沙箱且沙箱 Booter 为 CUA 时，为避免像素坐标漂移，输入图片不按最大边长缩放，合规图片原样发送（不重新编码，避免 JPEG 变色）；格式转换与动图拼图等其余处理仍然生效。超过约 5 MB 的大图可能超出服务商的图片体积限制，此时日志会给出警告。 ## 处理方式 - **合规静图原样发送**：已是 JPEG/PNG、方向正常且未超过最大边长的图片不做任何改动。 diff --git a/tests/test_process_stage_images.py b/tests/test_process_stage_images.py index c0ab044414..38850fb2af 100644 --- a/tests/test_process_stage_images.py +++ b/tests/test_process_stage_images.py @@ -4,6 +4,7 @@ import base64 import copy import inspect +import os from contextlib import nullcontext from pathlib import Path from types import SimpleNamespace @@ -270,6 +271,44 @@ async def test_cua_runtime_keeps_input_image_geometry( assert Path(req.image_urls[0]).read_bytes() == original +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("runtime", "booter", "big"), + [ + ("sandbox", "cua", True), + ("sandbox", "cua", False), + ("sandbox", "shipyard_neo", True), + ], +) +async def test_cua_oversize_image_warns( + harness, tmp_path, monkeypatch, runtime, booter, big +): + """Byte-exact CUA passthrough warns when an image may exceed upload limits.""" + dims = (1500, 1500) if big else (60, 30) + path = tmp_path / "shot.png" + if big: + # Noise stays incompressible, keeping the PNG above the 5 MB threshold. + PILImage.frombytes("RGB", dims, os.urandom(dims[0] * dims[1] * 3)).save( + path, "PNG" + ) + else: + PILImage.new("RGB", dims, "red").save(path, "PNG") + original = path.read_bytes() + harness.config["provider_settings"]["computer_use_runtime"] = runtime + harness.config["provider_settings"]["sandbox"] = {"booter": booter} + warning = MagicMock() + monkeypatch.setattr(internal.logger, "warning", warning) + event = make_event([Image(file=str(path))], text="") + await process_event(harness, event, preprocess_first=True) + assert len(harness.captured) == 1 + req = harness.captured[0].req + gated = runtime == "sandbox" and booter == "cua" + if gated: + assert Path(req.image_urls[0]).read_bytes() == original + warned = any("upload limits" in str(call) for call in warning.call_args_list) + assert warned == (big and gated) + + @pytest.mark.asyncio async def test_profile_reload_and_concurrent_requests(harness, tmp_path): source = source_image(tmp_path) From 7970b4b6cfffcf78068337d7c114ef5fa439a909 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B0=95=E6=B0=99?= Date: Mon, 14 Sep 2026 03:41:04 +0800 Subject: [PATCH 3/3] fix: keep configured montage cap for animated inputs under CUA sandbox --- .../method/agent_sub_stages/image_input.py | 8 +++++++- .../method/agent_sub_stages/internal.py | 14 +++++++++----- astrbot/core/utils/media_utils.py | 13 +++++++++++-- tests/test_process_stage_images.py | 18 ++++++++++++++++++ 4 files changed, 45 insertions(+), 8 deletions(-) diff --git a/astrbot/core/pipeline/process_stage/method/agent_sub_stages/image_input.py b/astrbot/core/pipeline/process_stage/method/agent_sub_stages/image_input.py index 002225c3c3..61e237b8bd 100644 --- a/astrbot/core/pipeline/process_stage/method/agent_sub_stages/image_input.py +++ b/astrbot/core/pipeline/process_stage/method/agent_sub_stages/image_input.py @@ -24,6 +24,7 @@ async def prepare_request_images( output_dir: Path, prepared: dict[str, str | None], quote_image_ref: str | None = None, + montage_max_size: int | None = None, ) -> None: """Replace current images on a working request and track their owned files. @@ -36,6 +37,7 @@ async def prepare_request_images( output_dir: Event working file directory, separate from the shared cache. prepared: Per-request mapping reused after the request hook. quote_image_ref: Optional input for the dedicated quote caption branch. + montage_max_size: Optional montage-specific limit; defaults to ``max_size``. """ req.image_urls = normalize_and_dedupe_strings(req.image_urls) refs = list(req.image_urls) @@ -52,7 +54,11 @@ async def prepare_request_images( path = None if enabled: path = await prepare_model_image( - ref, max_size=max_size, output_dir=output_dir, quality=quality + ref, + max_size=max_size, + output_dir=output_dir, + quality=quality, + montage_max_size=montage_max_size, ) if path: event.track_temporary_local_file(path) diff --git a/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py b/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py index 0831a71fc3..322506e3a8 100644 --- a/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py +++ b/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py @@ -246,9 +246,10 @@ async def process( settings = self.ctx.astrbot_config["provider_settings"] enabled = settings.get("image_compress_enabled", True) is not False options = settings.get("image_compress_options", {}) - max_size = normalize_model_image_max_size( + montage_max_size = normalize_model_image_max_size( options.get("max_size") if isinstance(options, dict) else None ) + max_size = montage_max_size sandbox_cfg = settings.get("sandbox") cua_pixel_mode = ( settings.get("computer_use_runtime") == "sandbox" @@ -256,10 +257,11 @@ async def process( and sandbox_cfg.get("booter") == "cua" ) if cua_pixel_mode: - # CUA pixel tools read coordinates 1:1, so the long-edge - # resize is lifted; compliant images pass through byte-exact - # since lossy re-encoding would shift colors. Format - # normalization still applies to other formats, and oversized + # CUA pixel tools read coordinates 1:1 on stills, so the + # still-image resize is lifted; compliant images pass through + # byte-exact since lossy re-encoding would shift colors. + # Montages are never used for coordinates and keep the + # configured cap, which bounds the 3x3 canvas. Oversized # passthrough images warn below. max_size = 1_000_000 quality = ( @@ -291,6 +293,7 @@ async def process( output_dir=output_dir, prepared=prepared, quote_image_ref=quote_image_ref, + montage_max_size=montage_max_size, ) await _process_quote_message( event, @@ -357,6 +360,7 @@ async def process( quality=quality, output_dir=output_dir, prepared=prepared, + montage_max_size=montage_max_size, ) if cua_pixel_mode: oversized = [] diff --git a/astrbot/core/utils/media_utils.py b/astrbot/core/utils/media_utils.py index a722a4a7ec..ff552293d5 100644 --- a/astrbot/core/utils/media_utils.py +++ b/astrbot/core/utils/media_utils.py @@ -1366,14 +1366,20 @@ async def prepare_model_image( max_size: int, output_dir: Path, quality: int = IMAGE_COMPRESS_DEFAULT_QUALITY, + montage_max_size: int | None = None, ) -> str | None: """Prepare a single local model-ready image for the caller to own until consumption. Args: image_ref: Source reference accepted by MediaResolver. - max_size: Validated longest-edge limit for stills and animation montages. + max_size: Validated longest-edge limit for stills. output_dir: Directory for independent request-owned working files. quality: JPEG output quality in the range 1-100. + montage_max_size: Optional longest-edge limit for animation montages. + CUA sessions lift the still-image cap to keep pixel coordinates 1:1, + but montages are never used for coordinates, so callers pass the + configured limit here to keep the 3x3 canvas bounded. Defaults to + ``max_size``. Returns: An existing JPEG or PNG path, or None for a recoverable input or write @@ -1386,7 +1392,10 @@ async def prepare_model_image( frame_count = await asyncio.to_thread(_inspect_image, image_bytes) if frame_count > 1: converted_bytes, _ = await asyncio.to_thread( - _extract_animation_montage_sync, image_bytes, max_size, quality + _extract_animation_montage_sync, + image_bytes, + montage_max_size if montage_max_size is not None else max_size, + quality, ) else: converted_bytes = await asyncio.to_thread( diff --git a/tests/test_process_stage_images.py b/tests/test_process_stage_images.py index 38850fb2af..44c2aba77b 100644 --- a/tests/test_process_stage_images.py +++ b/tests/test_process_stage_images.py @@ -309,6 +309,24 @@ async def test_cua_oversize_image_warns( assert warned == (big and gated) +@pytest.mark.asyncio +async def test_cua_montage_keeps_configured_cap(harness, tmp_path): + """Animated inputs keep the configured montage cap under CUA pixel mode.""" + path = tmp_path / "anim.gif" + frames = [PILImage.new("RGB", (600, 400), c) for c in ("red", "green", "blue")] + frames[0].save(path, "GIF", save_all=True, append_images=frames[1:]) + harness.config["provider_settings"]["computer_use_runtime"] = "sandbox" + harness.config["provider_settings"]["sandbox"] = {"booter": "cua"} + event = make_event([Image(file=str(path))], text="") + await process_event(harness, event, preprocess_first=True) + assert len(harness.captured) == 1 + req = harness.captured[0].req + with PILImage.open(req.image_urls[0]) as image: + # With the configured cap 90, 600x400 frames produce a 90x60 montage; + # the CUA still-image passthrough must not unbound the montage canvas. + assert max(image.size) <= 90 + + @pytest.mark.asyncio async def test_profile_reload_and_concurrent_requests(harness, tmp_path): source = source_image(tmp_path)