From 7cf41150cff845a4fc3cd5e6bdce40fbe1354e74 Mon Sep 17 00:00:00 2001 From: L4XB Date: Wed, 9 Sep 2026 22:34:35 +0200 Subject: [PATCH 1/2] fix(kb): keep the requested chunk parameters when URL cleaning falls back _clean_and_rechunk_content() fell back to self.chunker.chunk(content) when cleaning was enabled but no cleaning_provider_id was given, or when the provider lookup failed. Both fallbacks dropped the chunk_size and chunk_overlap passed for this upload and used the shared chunker's defaults (500/100), so a request for 128/16 was stored as 500-character chunks and later upload_document(pre_chunked_text=...) could not correct it. The non-cleaning path already forwarded the parameters. Forward chunk_size and chunk_overlap on both fallback paths. Fixes #9999 --- astrbot/core/knowledge_base/kb_helper.py | 13 +++-- tests/test_kb_clean_rechunk_fallback.py | 69 ++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 4 deletions(-) create mode 100644 tests/test_kb_clean_rechunk_fallback.py diff --git a/astrbot/core/knowledge_base/kb_helper.py b/astrbot/core/knowledge_base/kb_helper.py index 75234840cb..22a557a694 100644 --- a/astrbot/core/knowledge_base/kb_helper.py +++ b/astrbot/core/knowledge_base/kb_helper.py @@ -834,9 +834,12 @@ async def _clean_and_rechunk_content( if not cleaning_provider_id: logger.warning( - "启用了内容清洗，但未提供 cleaning_provider_id，跳过清洗并使用默认分块。" + "启用了内容清洗，但未提供 cleaning_provider_id，跳过清洗并使用指定参数分块: " + f"chunk_size={chunk_size}, chunk_overlap={chunk_overlap}" + ) + return await self.chunker.chunk( + content, chunk_size=chunk_size, chunk_overlap=chunk_overlap ) - return await self.chunker.chunk(content) if progress_callback: await progress_callback("cleaning", 0, 100) @@ -891,5 +894,7 @@ async def _clean_and_rechunk_content( except Exception as e: logger.error(f"使用 Provider '{cleaning_provider_id}' 清洗内容失败: {e}") - # 清洗失败，返回默认分块结果，保证流程不中断 - return await self.chunker.chunk(content) + # 清洗失败，回退到普通分块，但保留本次传入的分块参数，保证流程不中断 + return await self.chunker.chunk( + content, chunk_size=chunk_size, chunk_overlap=chunk_overlap + ) diff --git a/tests/test_kb_clean_rechunk_fallback.py b/tests/test_kb_clean_rechunk_fallback.py new file mode 100644 index 0000000000..98859c50f0 --- /dev/null +++ b/tests/test_kb_clean_rechunk_fallback.py @@ -0,0 +1,69 @@ +from unittest.mock import AsyncMock + +import pytest + +from astrbot.core.knowledge_base.kb_helper import KBHelper + + +class _RecordingChunker: + def __init__(self) -> None: + self.calls: list[dict] = [] + + async def chunk(self, text: str, **kwargs) -> list[str]: + self.calls.append(kwargs) + size = kwargs.get("chunk_size", 500) + return [text[i : i + size] for i in range(0, len(text), size)] + + +def _helper(chunker: _RecordingChunker, provider_error: Exception | None = None): + helper = KBHelper.__new__(KBHelper) + helper.chunker = chunker + helper.prov_mgr = AsyncMock() + helper.prov_mgr.get_provider_by_id = AsyncMock(side_effect=provider_error) + return helper + + +CONTENT = "x" * 1000 +PARAMS = {"chunk_size": 128, "chunk_overlap": 16} + + +@pytest.mark.asyncio +async def test_missing_cleaning_provider_keeps_requested_chunk_params(): + chunker = _RecordingChunker() + helper = _helper(chunker) + + chunks = await helper._clean_and_rechunk_content( + CONTENT, "https://example.com", enable_cleaning=True, **PARAMS + ) + + assert chunker.calls == [PARAMS] + assert max(len(c) for c in chunks) <= 128 + + +@pytest.mark.asyncio +async def test_failed_cleaning_provider_lookup_keeps_requested_chunk_params(): + chunker = _RecordingChunker() + helper = _helper(chunker, provider_error=RuntimeError("provider unavailable")) + + chunks = await helper._clean_and_rechunk_content( + CONTENT, + "https://example.com", + enable_cleaning=True, + cleaning_provider_id="llm-1", + **PARAMS, + ) + + assert chunker.calls == [PARAMS] + assert max(len(c) for c in chunks) <= 128 + + +@pytest.mark.asyncio +async def test_cleaning_disabled_still_passes_chunk_params(): + chunker = _RecordingChunker() + helper = _helper(chunker) + + await helper._clean_and_rechunk_content( + CONTENT, "https://example.com", enable_cleaning=False, **PARAMS + ) + + assert chunker.calls == [PARAMS] From 066796fc7cf2cda28e75d8703bd172c64d6a0240 Mon Sep 17 00:00:00 2001 From: L4XB Date: Wed, 9 Sep 2026 22:36:56 +0200 Subject: [PATCH 2/2] test(kb): import the core lifecycle first to avoid the knowledge_base import cycle --- tests/test_kb_clean_rechunk_fallback.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_kb_clean_rechunk_fallback.py b/tests/test_kb_clean_rechunk_fallback.py index 98859c50f0..d43d020d83 100644 --- a/tests/test_kb_clean_rechunk_fallback.py +++ b/tests/test_kb_clean_rechunk_fallback.py @@ -2,6 +2,9 @@ import pytest +# Importing the core lifecycle first resolves the import cycle between +# astrbot.core.provider.manager and astrbot.core.knowledge_base. +import astrbot.core.core_lifecycle # noqa: F401 from astrbot.core.knowledge_base.kb_helper import KBHelper