From 732c4a5fdfd2ce571efdf2b6a99a2534093942e5 Mon Sep 17 00:00:00 2001 From: "Dana K. Williams" Date: Thu, 2 Jul 2026 23:45:01 +0000 Subject: [PATCH] fix(tree_renderer): collapse duplicate sibling/cousin summaries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PageIndex's whole-page text slicing (not a finer offset) can hand byte-identical source text to multiple nodes that happen to land on the same physical page (see VectifyAI/PageIndex#340) — not just parent/child pairs, but siblings and cousins anywhere earlier in the document. The LLM then writes the same summary for each, which the renderer previously repeated verbatim at every one of them. Track every summary already rendered, mapped to the title that first produced it. A later node with an identical summary now renders as a short pointer back to that title instead of repeating the block. --- openkb/tree_renderer.py | 26 ++++++++-- tests/test_tree_renderer.py | 95 +++++++++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+), 5 deletions(-) diff --git a/openkb/tree_renderer.py b/openkb/tree_renderer.py index 2424ba10..b9a03cda 100644 --- a/openkb/tree_renderer.py +++ b/openkb/tree_renderer.py @@ -15,8 +15,18 @@ def _yaml_frontmatter(source_name: str, doc_id: str, description: str = "") -> s return "---\n" + "\n".join(lines) + "\n---\n" -def _render_nodes_summary(nodes: list[dict], depth: int) -> str: - """Recursively render nodes for the *summary* view (summaries only).""" +def _render_nodes_summary(nodes: list[dict], depth: int, seen: dict[str, str]) -> str: + """Recursively render nodes for the *summary* view (summaries only). + + Nodes on the same source page can be handed byte-identical text by the + underlying PageIndex tree (whole-page text slicing, not a finer offset — + see PageIndex#340), which then produces duplicate LLM summaries. This + isn't confined to parent/child pairs — siblings and cousins anywhere + earlier in the document can collide too — so ``seen`` maps every summary + already rendered to the title that first produced it, and any later node + with the same summary is rendered as a short pointer back to that title + instead of repeating the block. + """ lines: list[str] = [] heading_prefix = "#" * min(depth, 6) for node in nodes: @@ -27,10 +37,16 @@ def _render_nodes_summary(nodes: list[dict], depth: int) -> str: children = node.get("nodes", []) lines.append(f"{heading_prefix} {title} (pages {start}–{end})\n") - if summary: + + first_seen_title = seen.get(summary) if summary else None + if first_seen_title is not None: + lines.append(f'_(same content as "{first_seen_title}" above)_\n') + elif summary: lines.append(f"Summary: {summary}\n") + seen[summary] = title + if children: - lines.append(_render_nodes_summary(children, depth + 1)) + lines.append(_render_nodes_summary(children, depth + 1, seen)) return "\n".join(lines) @@ -44,5 +60,5 @@ def render_summary_md(tree: dict, source_name: str, doc_id: str, description: st """ frontmatter = _yaml_frontmatter(source_name, doc_id, description) structure = tree.get("structure", []) - body = _render_nodes_summary(structure, depth=1) + body = _render_nodes_summary(structure, depth=1, seen={}) return frontmatter + "\n" + body diff --git a/tests/test_tree_renderer.py b/tests/test_tree_renderer.py index 3786cfe4..86df5719 100644 --- a/tests/test_tree_renderer.py +++ b/tests/test_tree_renderer.py @@ -52,6 +52,101 @@ def test_summary_md_has_type_and_description(): assert 'full_text: "sources/my-doc.json"' in md +def test_duplicate_sibling_summaries_collapse_to_a_pointer(): + # Two sibling nodes on the same physical page can be handed the exact + # same LLM-written summary (PageIndex#340). The second occurrence should + # collapse to a pointer instead of repeating the block verbatim. + tree = { + "structure": [ + { + "title": "1.1 First item", + "start_index": 5, + "end_index": 5, + "summary": "Shared duplicate summary text.", + "nodes": [], + }, + { + "title": "1.2 Second item", + "start_index": 5, + "end_index": 5, + "summary": "Shared duplicate summary text.", + "nodes": [], + }, + ] + } + md = render_summary_md(tree, "my-doc", "doc-123") + assert md.count("Summary: Shared duplicate summary text.") == 1 + assert '_(same content as "1.1 First item" above)_' in md + + +def test_duplicate_summaries_collapse_across_cousins_not_just_siblings(): + # The collision isn't confined to direct siblings — a node nested under + # a different parent, seen later in document order, can repeat the same + # summary too. + tree = { + "structure": [ + { + "title": "Parent A", + "start_index": 1, + "end_index": 1, + "summary": "", + "nodes": [ + { + "title": "Child A.1", + "start_index": 1, + "end_index": 1, + "summary": "Cousin duplicate.", + "nodes": [], + } + ], + }, + { + "title": "Parent B", + "start_index": 2, + "end_index": 2, + "summary": "", + "nodes": [ + { + "title": "Child B.1", + "start_index": 2, + "end_index": 2, + "summary": "Cousin duplicate.", + "nodes": [], + } + ], + }, + ] + } + md = render_summary_md(tree, "my-doc", "doc-123") + assert md.count("Summary: Cousin duplicate.") == 1 + assert '_(same content as "Child A.1" above)_' in md + + +def test_distinct_summaries_are_not_collapsed(): + tree = { + "structure": [ + { + "title": "A", + "start_index": 1, + "end_index": 1, + "summary": "Summary one.", + "nodes": [], + }, + { + "title": "B", + "start_index": 2, + "end_index": 2, + "summary": "Summary two.", + "nodes": [], + }, + ] + } + md = render_summary_md(tree, "my-doc", "doc-123") + assert "Summary: Summary one." in md + assert "Summary: Summary two." in md + assert "same content as" not in md + + def test_summary_full_text_quoted_yaml_safe(): import yaml