diff --git a/pageindex/flash/embedded_toc.py b/pageindex/flash/embedded_toc.py index 6bd38704d..fab88865d 100644 --- a/pageindex/flash/embedded_toc.py +++ b/pageindex/flash/embedded_toc.py @@ -51,6 +51,11 @@ # Same-page titles at or above this similarity refer to the same section. _REPAIR_SIMILARITY = 0.7 +_DISSOLVE_SIMILARITY = 0.9 + +# A climb past a same-page frame heading must leave it at least this much +# of the page as content, or it is claiming the section is empty. +_MIN_SECTION_GAP = 0.15 # Backfill noise filters: a normalized title recurring this often across the # detected tree is a running label, not a section; a title longer than this @@ -254,6 +259,25 @@ def bookmarks_to_structure(entries: list[dict], n_pages: int) -> list[dict]: return _finalize(_build_bookmark_nodes(entries), n_pages) +_NUMERAL_RE = re.compile(r"^([A-Z]\.\d*(?:\.\d+)*|[A-Z]\.$|\d+(?:\.\d+)*)") + + +def _numeral_prefix(title: str) -> Optional[str]: + match = _NUMERAL_RE.match(title.strip()) + return match.group(1) if match else None + + +def _numeral_child(parent_title: str, child_title: str) -> bool: + """'1'->'1.1', '1.1'->'1.1.1', 'D.'->'D.1' are parent->child pairs.""" + parent = _numeral_prefix(parent_title) + child = _numeral_prefix(child_title) + if not parent or not child: + return False + if parent.endswith(".") and len(parent) == 2: + return child.startswith(parent) and len(child) > 2 + return child.startswith(parent + ".") and len(child) > len(parent) + + def _same_heading(node: dict, chapter: dict) -> bool: if node["start_index"] != chapter["start_index"]: return False @@ -481,22 +505,49 @@ def merge_bookmark_tree( anchored to the deepest bookmark section active at its start page. A detected subtree that fits inside its anchor's range moves under it whole; one that spans ranges is dissolved, leaving a leaf, with its - children placed individually. A node that duplicates its anchor - heading dissolves into it. Two noise filters prune the whole detected - tree before placement: titles recurring across it (running labels, - not sections) and overlong titles (paragraph leads picked up as - headings) are spliced out, their children promoted into their place. - Nodes wholly before the first bookmark stay at the root. + children placed individually. A node whose normalized title matches + a frame heading starting on its page -- equal, apart by a numbering + prefix, or apart by character-level garble -- dissolves without a + trace, its children re-placed individually. Dissolving holds fuzzy + matches to a higher bar than title repair: at the repair bar, + wordy-overlapping titles of distinct sections false-match, and a + false dissolve deletes a real heading, while an escaped garbled + duplicate only leaves a stray leaf. + + A surviving node attaches along its anchor's ancestor chain -- the + open sections at its page, the only parents page containment allows. + It climbs past a candidate only on positive evidence that it is the + candidate's peer, not its child: a numbering prefix outside the + candidate's numbering, or the same style fingerprint -- face, weight + and size as one hash; a larger raw size in a different face is not + seniority (margin-box titles run larger-regular than bold subsection + headings). Frame nodes learn style and position from the duplicates + that dissolve into them; detection exports the ``_style``/``_y`` + keys and ``extract_toc`` strips them after the merge. A + same-page climb must also leave the candidate real content below its + heading -- climbing past it on a near-zero gap would claim the + section is empty, the signature of margin-box noise rather than of a + tail section. Numbering-driven climbs are exempt: the numbering + itself proves the level relation. With no evidence the node stays at + the anchor, and a climb past the last ancestor makes it top-level. + + Two noise filters prune the whole detected tree before placement: + titles recurring across it (running labels, not sections) and + overlong titles (paragraph leads picked up as headings) are spliced + out, their children promoted into their place. Nodes wholly before + the first bookmark stay at the root. """ roots = _build_bookmark_nodes(entries) flat: list[dict] = [] + parent_pos: list[Optional[int]] = [] - def collect(nodes: list[dict]) -> None: + def collect(nodes: list[dict], parent: Optional[int]) -> None: for node in nodes: flat.append(node) - collect(node["nodes"]) + parent_pos.append(parent) + collect(node["nodes"], len(flat) - 1) - collect(roots) + collect(roots, None) if not flat: return structure @@ -514,6 +565,50 @@ def anchor_index(page: int): break return idx + def dissolve_target(node: dict, idx: int) -> Optional[int]: + node_norm = _normalize_title(node["title"]) + if not node_norm: + return None + pos = idx + while pos >= 0 and flat[pos]["start_index"] == node["start_index"]: + frame_norm = _normalize_title(flat[pos]["title"]) + if frame_norm and (node_norm == frame_norm + or node_norm.endswith(frame_norm) + or frame_norm.endswith(node_norm) + or _similarity(node_norm, frame_norm) + >= _DISSOLVE_SIMILARITY): + return pos + pos -= 1 + return None + + def _style_climbs(node: dict, cand: dict) -> bool: + return (node.get("_style") is not None + and node["_style"] == cand.get("_style")) + + def _claims_empty(node: dict, cand: dict) -> bool: + if cand["start_index"] != node["start_index"]: + return False + node_y = node.get("_y") + cand_y = cand.get("_y") + if node_y is None or cand_y is None: + return True + return node_y - cand_y < _MIN_SECTION_GAP + + def choose_parent(node: dict, idx: int) -> Optional[int]: + pos: Optional[int] = idx + while pos is not None: + cand = flat[pos] + if (_numeral_prefix(cand["title"]) is not None + and _numeral_prefix(node["title"]) is not None): + if _numeral_child(cand["title"], node["title"]): + return pos + pos = parent_pos[pos] + continue + if not _style_climbs(node, cand) or _claims_empty(node, cand): + return pos + pos = parent_pos[pos] + return None + title_counts: Counter = Counter() def count_titles(nodes: list[dict]) -> None: @@ -561,16 +656,21 @@ def place(node: dict) -> None: for child in children: place(child) return - target = flat[idx] + matched = dissolve_target(node, idx) + if matched is not None: + frame = flat[matched] + for key in ("_style", "_y"): + if frame.get(key) is None and node.get(key) is not None: + frame[key] = node[key] + for child in children: + place(child) + return + chosen = choose_parent(node, idx) + siblings = roots if chosen is None else flat[chosen]["nodes"] if _subtree_max_start(node) < range_end[idx]: - if _same_heading(node, target): - for child in children: - insert_by_page(target["nodes"], child) - else: - insert_by_page(target["nodes"], node) + insert_by_page(siblings, node) return - if not _same_heading(node, target): - insert_by_page(target["nodes"], dict(node, nodes=[])) + insert_by_page(siblings, dict(node, nodes=[])) for child in children: place(child) diff --git a/pageindex/flash/main.py b/pageindex/flash/main.py index 2b8c74885..19ad01734 100644 --- a/pageindex/flash/main.py +++ b/pageindex/flash/main.py @@ -326,6 +326,14 @@ def extract_toc( result["structure"], result["toc_source"] = apply_embedded_toc( structure, doc_handle, len(pages), page_texts=page_texts ) + + def _drop_signal_keys(nodes: list[dict]) -> None: + for node in nodes: + node.pop("_style", None) + node.pop("_y", None) + _drop_signal_keys(node.get("nodes") or []) + + _drop_signal_keys(result["structure"]) return result diff --git a/pageindex/flash/outline_assembly/assembly.py b/pageindex/flash/outline_assembly/assembly.py index ea0ff4027..faa41eb71 100644 --- a/pageindex/flash/outline_assembly/assembly.py +++ b/pageindex/flash/outline_assembly/assembly.py @@ -282,6 +282,7 @@ def _walk_nodes(items: list[OutlineNode]) -> list[dict]: if item.child_nodes: result.extend(_walk_nodes(item.child_nodes)) continue + heading_block = item.heading.group_slot node = { "title": title, "node_id": "", @@ -289,6 +290,12 @@ def _walk_nodes(items: list[OutlineNode]) -> list[dict]: "end_index": item.heading.page.page_index, "nodes": _walk_nodes(item.child_nodes) if item.child_nodes else [], "_appear_start": _heading_appears_at_page_top(item.heading), + # Heading-level signals for the embedded-TOC merge; + # extract_toc strips them from the final output. + "_style": ((dominant_style_of(heading_block) + + ("|C" if is_caps_heavy(heading_block) else "")) + if heading_block is not None else None), + "_y": item.heading.auxiliary_slot, } flat_nodes.append(node) result.append(node) diff --git a/tests/test_flash_embedded_toc.py b/tests/test_flash_embedded_toc.py new file mode 100644 index 000000000..854816f9f --- /dev/null +++ b/tests/test_flash_embedded_toc.py @@ -0,0 +1,123 @@ +from pageindex.flash.embedded_toc import merge_bookmark_tree + + +def _node(title, start, children=(), style=None, y=None): + node = {"title": title, "node_id": "", "start_index": start, + "end_index": start, "nodes": [dict(c) for c in children]} + if style is not None: + node["_style"] = style + if y is not None: + node["_y"] = float(y) + return node + + +def _titles(nodes, out=None): + if out is None: + out = [] + for n in nodes: + out.append(n["title"]) + _titles(n.get("nodes", []), out) + return out + + +def test_numbering_prefixed_duplicates_dissolve_into_frame(): + entries = [ + {"title": "Experiments", "level": 1, "page": 8}, + {"title": "Scaling Laws", "level": 2, "page": 8}, + {"title": "Main Results", "level": 2, "page": 10}, + ] + detected = [_node("5 Experiments", 8, + [_node("5.1 Scaling Laws", 8)])] + merged = merge_bookmark_tree(detected, entries, n_pages=12) + assert _titles(merged) == ["Experiments", "Scaling Laws", "Main Results"] + + +def test_wordy_overlap_of_distinct_section_survives(): + entries = [{"title": "Federal Reserve Banks and Branches", "level": 1, "page": 3}] + detected = [_node("Federal Reserve Banks and Branches", 3, + [_node("Reserve Bank and Branch Directors", 3, + [_node("District 1", 4)])])] + merged = merge_bookmark_tree(detected, entries, n_pages=6) + assert _titles(merged) == ["Federal Reserve Banks and Branches", + "Reserve Bank and Branch Directors", + "District 1"] + chapter = merged[0] + assert _titles(chapter["nodes"]) == ["Reserve Bank and Branch Directors", + "District 1"] + + +def test_garbled_duplicate_dissolves(): + entries = [ + {"title": "Applications of PCA", "level": 1, "page": 2}, + {"title": "Kernel PCA", "level": 1, "page": 4}, + ] + detected = [_node("Applications of peA", 2)] + merged = merge_bookmark_tree(detected, entries, n_pages=6) + assert _titles(merged) == ["Applications of PCA", "Kernel PCA"] + + +def test_same_style_tail_section_climbs(): + entries = [ + {"title": "Related Work", "level": 1, "page": 16}, + {"title": "Contributions", "level": 1, "page": 20}, + ] + detected = [_node("7 Related Work", 16, style="F1", y=0.08, + children=[_node("Conclusion", 16, style="F1", y=0.55), + _node("References", 17, style="F1", y=0.10)])] + merged = merge_bookmark_tree(detected, entries, n_pages=22) + assert _titles(merged) == ["Related Work", "Conclusion", "References", + "Contributions"] + assert all(not n.get("nodes") for n in merged) + + +def test_empty_section_claim_blocks_climb(): + entries = [ + {"title": "Bayesian probabilities", "level": 1, "page": 41}, + {"title": "The Gaussian distribution", "level": 1, "page": 44}, + ] + detected = [_node("Bayesian probabilities", 41, style="F1", y=0.10, + children=[_node("Thomas Bayes", 41, style="F1", + y=0.14)])] + merged = merge_bookmark_tree(detected, entries, n_pages=50) + assert _titles(merged) == ["Bayesian probabilities", "Thomas Bayes", + "The Gaussian distribution"] + assert _titles(merged[0]["nodes"]) == ["Thomas Bayes"] + + +def test_different_style_stays_child(): + entries = [ + {"title": "Introduction", "level": 1, "page": 2}, + {"title": "Motivation", "level": 1, "page": 3}, + ] + detected = [_node("1 Introduction", 2, style="F1", y=0.05, + children=[_node("Contributions", 2, style="F2", + y=0.60)])] + merged = merge_bookmark_tree(detected, entries, n_pages=6) + assert _titles(merged) == ["Introduction", "Contributions", "Motivation"] + assert _titles(merged[0]["nodes"]) == ["Contributions"] + + +def test_numbering_outside_candidate_climbs(): + entries = [ + {"title": "2. Background", "level": 1, "page": 3}, + {"title": "3. Methods", "level": 1, "page": 5}, + ] + detected = [_node("2.1 Setup", 3), _node("4 Results", 5)] + merged = merge_bookmark_tree(detected, entries, n_pages=8) + assert _titles(merged) == ["2. Background", "2.1 Setup", "3. Methods", + "4 Results"] + assert _titles(merged[0]["nodes"]) == ["2.1 Setup"] + assert not merged[1].get("nodes") + + +def test_new_sections_still_graft(): + entries = [ + {"title": "Introduction", "level": 1, "page": 2}, + {"title": "Methods", "level": 1, "page": 5}, + ] + detected = [_node("Abstract", 1), + _node("Data Collection", 6)] + merged = merge_bookmark_tree(detected, entries, n_pages=9) + assert _titles(merged) == ["Abstract", "Introduction", "Methods", + "Data Collection"] + assert _titles(merged[2]["nodes"]) == ["Data Collection"]