Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 9 additions & 4 deletions astrbot/core/knowledge_base/kb_helper.py
Original file line number Diff line number Diff line change
Expand Up @@ -834,9 +834,12 @@ async def _clean_and_rechunk_content(

if not cleaning_provider_id:
logger.warning(
"启用了内容清洗,但未提供 cleaning_provider_id,跳过清洗并使用默认分块。"
"启用了内容清洗,但未提供 cleaning_provider_id,跳过清洗并使用指定参数分块: "
f"chunk_size={chunk_size}, chunk_overlap={chunk_overlap}"
)
return await self.chunker.chunk(
content, chunk_size=chunk_size, chunk_overlap=chunk_overlap
)
return await self.chunker.chunk(content)

if progress_callback:
await progress_callback("cleaning", 0, 100)
Expand Down Expand Up @@ -891,5 +894,7 @@ async def _clean_and_rechunk_content(

except Exception as e:
logger.error(f"使用 Provider '{cleaning_provider_id}' 清洗内容失败: {e}")
# 清洗失败,返回默认分块结果,保证流程不中断
return await self.chunker.chunk(content)
# 清洗失败,回退到普通分块,但保留本次传入的分块参数,保证流程不中断
return await self.chunker.chunk(
content, chunk_size=chunk_size, chunk_overlap=chunk_overlap
)
72 changes: 72 additions & 0 deletions tests/test_kb_clean_rechunk_fallback.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
from unittest.mock import AsyncMock

import pytest

# Importing the core lifecycle first resolves the import cycle between
# astrbot.core.provider.manager and astrbot.core.knowledge_base.
import astrbot.core.core_lifecycle # noqa: F401
from astrbot.core.knowledge_base.kb_helper import KBHelper


class _RecordingChunker:
def __init__(self) -> None:
self.calls: list[dict] = []

async def chunk(self, text: str, **kwargs) -> list[str]:
self.calls.append(kwargs)
size = kwargs.get("chunk_size", 500)
return [text[i : i + size] for i in range(0, len(text), size)]


def _helper(chunker: _RecordingChunker, provider_error: Exception | None = None):
helper = KBHelper.__new__(KBHelper)
helper.chunker = chunker
helper.prov_mgr = AsyncMock()
helper.prov_mgr.get_provider_by_id = AsyncMock(side_effect=provider_error)
return helper


CONTENT = "x" * 1000
PARAMS = {"chunk_size": 128, "chunk_overlap": 16}


@pytest.mark.asyncio
async def test_missing_cleaning_provider_keeps_requested_chunk_params():
chunker = _RecordingChunker()
helper = _helper(chunker)

chunks = await helper._clean_and_rechunk_content(
CONTENT, "https://example.com", enable_cleaning=True, **PARAMS
)

assert chunker.calls == [PARAMS]
assert max(len(c) for c in chunks) <= 128


@pytest.mark.asyncio
async def test_failed_cleaning_provider_lookup_keeps_requested_chunk_params():
chunker = _RecordingChunker()
helper = _helper(chunker, provider_error=RuntimeError("provider unavailable"))

chunks = await helper._clean_and_rechunk_content(
CONTENT,
"https://example.com",
enable_cleaning=True,
cleaning_provider_id="llm-1",
**PARAMS,
)

assert chunker.calls == [PARAMS]
assert max(len(c) for c in chunks) <= 128


@pytest.mark.asyncio
async def test_cleaning_disabled_still_passes_chunk_params():
chunker = _RecordingChunker()
helper = _helper(chunker)

await helper._clean_and_rechunk_content(
CONTENT, "https://example.com", enable_cleaning=False, **PARAMS
)

assert chunker.calls == [PARAMS]