diff --git a/setup.py b/setup.py index abb11ae75..6cd996412 100755 --- a/setup.py +++ b/setup.py @@ -665,6 +665,7 @@ def build(): ret.append( (f'{g_root}/src/_table_spans.py', to_dir) ) ret.append( (f'{g_root}/src/_table_union.py', to_dir) ) ret.append( (f'{g_root}/src/_table_headers.py', to_dir) ) + ret.append( (f'{g_root}/src/_table_sections.py', to_dir) ) ret.append( (f'{g_root}/src/utils.py', to_dir) ) ret.append( (f'{g_root}/src/_wxcolors.py', to_dir) ) ret.append( (f'{g_root}/src/_apply_pages.py', to_dir) ) diff --git a/src/_table_sections.py b/src/_table_sections.py new file mode 100644 index 000000000..8780f6c92 --- /dev/null +++ b/src/_table_sections.py @@ -0,0 +1,198 @@ +""" +Copyright (C) 2023 Artifex Software, Inc. + +This file is part of PyMuPDF. + +PyMuPDF is free software: you can redistribute it and/or modify it under the +terms of the GNU Affero General Public License as published by the Free +Software Foundation, either version 3 of the License, or (at your option) +any later version. + +PyMuPDF is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +details. + +You should have received a copy of the GNU Affero General Public License +along with MuPDF. If not, see + +Alternative licensing terms are available from the licensor. +For commercial licensing, see or contact +Artifex Software, Inc., 39 Mesa Street, Suite 108A, San Francisco, +CA 94129, USA, for further information. + +--------------------------------------------------------------------- +PyMuPDF centered section-title placements. +""" + +"""Whole-table specialization of the existing H6 centered-title extent rule. + +Only proven, bold single-title/repeated-label blocks enter this policy. Existing +native words/drawings/characters are supplied by the refine call; no extraction, +layout decision, inference, GT or filename is consulted. Partial extents abstain. +""" +from copy import copy + + +def centered_extent(title, bounds): + """H6's existing 5%-of-interval-width test, restricted to the whole row.""" + x0, x1 = title + left, right = bounds + if not (left <= x0 < x1 <= right and x1 - x0 < right - left): + return False + return abs((x0 + x1 - left - right) * 0.5) <= 0.05 * (right - left) + + +def interior_vertical(paths, band, *, Rect, is_line_like, min_length): + """True physical interior partitions at the title band, not TGIF slots. + + Keep native segment extents: the existing border-line sets discard them. + Outer borders and underlines do not partition a title. Reuse the finder's + existing simulated-line predicate and minimum length, without new cutoffs. + """ + x0, y0, x1, y1 = band + for path in paths: + for item in path['items']: + if item[0] == 'l': + a, b = item[1:3] + if a.x != b.x: + continue + segments = [(a.x, min(a.y, b.y), max(a.y, b.y))] + elif item[0] == 're': + r = Rect(item[1]) + if path.get('type') in ('s', 'fs'): + segments = [(r.x0, r.y0, r.y1), (r.x1, r.y0, r.y1)] + elif is_line_like(r, min_length) and r.width < r.height: + segments = [((r.x0 + r.x1) * .5, r.y0, r.y1)] + else: + continue + else: + continue + if any(x0 < x < x1 and high - low >= min_length + and low < y1 and y0 < high for x, low, high in segments): + return True + return False + + +def title_row_proof(grid, index, words, word_members, chars, char_members, + paths, *, width, section_like, Rect, is_line_like, + min_length): + row = grid[index] + occupied = [cell for cell in row if cell.text.strip()] + if len(occupied) != 1 or not section_like([cell.text for cell in row]): + return None, 'not_single_section_title' + if not row or any(cell.bbox is None or cell.rowspan != 1 for cell in row): + return None, 'nonflat_title_row' + if any(ri + cell.rowspan > index for ri, earlier in enumerate(grid[:index]) + for cell in earlier): + return None, 'incoming_rowspan' + if sum(cell.colspan for cell in row) != width: + return None, 'incomplete_title_slots' + boxes = [Rect(cell.bbox) for cell in row] + if len({(r.y0, r.y1) for r in boxes}) != 1: + return None, 'unequal_title_bands' + band = (min(r.x0 for r in boxes), boxes[0].y0, + max(r.x1 for r in boxes), boxes[0].y1) + if interior_vertical(paths, band, Rect=Rect, is_line_like=is_line_like, + min_length=min_length): + return None, 'physical_vertical_partition' + title = occupied[0] + selected = tuple(word_members(Rect(title.bbox))) + whole_row = tuple(word_members(Rect(band))) + if not selected or selected != whole_row: + return None, 'competing_or_unproved_title_words' + actual = tuple(char_members(Rect(title.bbox))) + if not actual or not all(chars[i].get('bold', False) for i in actual): + return None, 'section_emphasis_not_proven' + extent = min(words[i][0] for i in selected), max(words[i][2] for i in selected) + if not centered_extent(extent, (band[0], band[2])): + return None, 'not_centered_over_whole_row' + return {'band': band, 'text': title.text, 'word_indices': selected, + 'width': width, 'word_extent': extent}, 'centered_title' + + +def aligned_blocks(rows, depth, grid, *, repeat_labels, prove): + cuts = tuple(repeat_labels(rows, depth)) + if not cuts or grid is None: + return cuts, (), 'no_repeated_label_blocks' + normalized = [tuple(text.casefold() for text in row) for row in rows] + signature = normalized[cuts[0]] + first = next(i for i, row in enumerate(normalized) if row == signature) + proofs = [] + for index, end in zip([first, *cuts], [*(cut - 1 for cut in cuts), len(rows)]): + if index == 0: + return cuts, (), 'untitled_repeat' + if not any(any(text.strip() for text in row) for row in rows[index + 1:end]): + return cuts, (), 'no_record_after_labels' + proof, reason = prove(index - 1) + if proof is None: + return cuts, (), reason + proofs.append(proof) + return tuple(cut - 1 for cut in cuts), tuple(proofs), 'centered_section_aligned' + + +def merge_title_row(grid, proofs, *, width, Rect, word_members): + """Return actual placements, preserving all body cells and source fields.""" + if not grid: + return grid, None + row = grid[0] + populated = [cell for cell in row if cell.text.strip()] + if len(populated) != 1 or any(cell.bbox is None or cell.rowspan != 1 for cell in row): + return grid, None + boxes = [Rect(cell.bbox) for cell in row] + band = (min(r.x0 for r in boxes), min(r.y0 for r in boxes), + max(r.x1 for r in boxes), max(r.y1 for r in boxes)) + title = populated[0] + proof = next((p for p in proofs if p['band'] == band and p['text'] == title.text + and p['width'] == width and tuple(word_members(Rect(band))) == p['word_indices']), None) + if proof is None or sum(cell.colspan for cell in row) != width: + return grid, None + merged = copy(title) + merged.bbox, merged.colspan, merged.rowspan = band, width, 1 + return [[merged], *grid[1:]], {'text': title.text, 'bbox': band, 'colspan': width, + 'word_indices': proof['word_indices']} + +import pymupdf +from pymupdf._table_headers import section_label_row_like +from pymupdf._table_refine import ( + _WordCenterIndex, _refine_word_candidates, _refine_word_in_rect, +) + + +def _section_word_members(page, words, rect): + return tuple(i for i, word in _refine_word_candidates(page, words, rect) + if word[4].strip() and _refine_word_in_rect(*word[:4], *tuple(rect))) + + +def section_repeat_plan(page, grid, rows, depth, cuts, chars, *, width, + is_line_like, min_length): + """Plan section cuts using materials already prepared by find_tables. + + Missing caches abstain; never re-extract page text or drawings. The plan is + scoped to the current refined parent and carries real page-word indices. + """ + words = getattr(page, '_refine_words_cache', None) + cache = getattr(page, '_table_drawings_cache', None) + if not cuts or grid is None or words is None or not cache: + return cuts, () + char_words = [(c['x0'], c['top'], c['x1'], c['bottom'], c['text']) for c in chars] + index = _WordCenterIndex(char_words) + def char_members(rect): + return tuple(i for i in index.candidates(rect) if chars[i]['text'].strip() + and _refine_word_in_rect(*char_words[i][:4], *tuple(rect))) + def prove(row_index): + return title_row_proof(grid, row_index, words, + lambda rect: _section_word_members(page, words, rect), chars, + char_members, cache[0], width=width, + section_like=section_label_row_like, Rect=pymupdf.Rect, + is_line_like=is_line_like, min_length=min_length) + aligned, proofs, _reason = aligned_blocks(rows, depth, grid, + repeat_labels=lambda _rows, _depth: cuts, prove=prove) + return aligned, proofs + + +def apply_section_title(page, grid, proofs, *, width): + """Apply the actual title placement after strict final span resolution.""" + merged, _event = merge_title_row(grid, proofs, width=width, Rect=pymupdf.Rect, + word_members=lambda rect: _section_word_members(page, page._refine_words_cache, rect)) + return merged diff --git a/src/_table_union.py b/src/_table_union.py index bd3a31003..53f3828e9 100644 --- a/src/_table_union.py +++ b/src/_table_union.py @@ -97,6 +97,28 @@ def _layout_table_grids(page): return grids +def _union_nested_candidate_indexes(candidates, containers=None): + """Rule grids inside one actual cell of another rule grid are cell text. + + Use only already-admitted native candidates. Containment by a large layout + box is not nesting: independent panels in an overmerged region must survive. + Equal boxes, mere overlaps and children crossing parent cells stay candidates. + """ + nested = set() + for parent_index in (range(len(candidates)) if containers is None else containers): + parent_bbox, parent_grid = candidates[parent_index] + cell_rects = None + for index, (bbox, _grid) in enumerate(candidates): + if index == parent_index or bbox == parent_bbox or bbox not in parent_bbox: + continue + if cell_rects is None: + cell_rects = [pymupdf.Rect(cell) for row in parent_grid + for cell in row if cell is not None] + if any(bbox in cell for cell in cell_rects): + nested.add(index) + return nested + + def _union_line_candidates(page, *, add_lines=None, add_boxes=None): """Line-based table candidates for the union stage as ``(bbox, grid)`` pairs. @@ -509,8 +531,10 @@ def _union_replace_append(existing, candidates, *, page, grid_ref, grid_ref_iou, grid_refs, consumed = {}, set() replacements = {} append_candidates = [] + # A validated grid-ref already owns the native cells containing these children. + nested_in_refs = _union_nested_candidate_indexes(candidates, consumed) for candidate_index, candidate in enumerate(candidates): - if candidate_index in consumed: + if candidate_index in consumed or candidate_index in nested_in_refs: continue owner_index, ambiguous = _union_find_owner(candidate[0], existing_bboxes) if owner_index is not None: @@ -534,7 +558,14 @@ def _union_replace_append(existing, candidates, *, page, grid_ref, grid_ref_iou, else: entries.append(entry) entries.extend(append_candidates) - return entries + # A ruled child becomes cell text only if its ruled container was selected. + # Keep the normal grid-ref gates and the original replacement decision: + # rejecting a raw parent proposal must not suppress an independent child. + selected_grids = {id(grid) for _bbox, grid in entries} + ruled = [entry for entry in candidates if id(entry[1]) in selected_grids] + nested = _union_nested_candidate_indexes(ruled) + nested_grids = {id(ruled[index][1]) for index in nested} + return [entry for entry in entries if id(entry[1]) not in nested_grids] def _find_tables_union(page, *, add_lines=None, add_boxes=None): diff --git a/src/table.py b/src/table.py index 8417f8515..d0b6bfb9f 100644 --- a/src/table.py +++ b/src/table.py @@ -3037,7 +3037,7 @@ def _refine_repeated_leading_header_cuts(rows, header_rows): Several tables printed one under another with no gap are detected as one grid, and the giveaway is that their shared header row recurs verbatim. The - signature is the first row that carries at least two non-empty cells in a + signature is the first row that carries at least two distinct non-empty labels in a grid at least two columns wide and is not purely numeric or punctuation; it must also be a leading row -- inside the header region the header rules found, or within the first three rows, because that finder is conservative and a @@ -3076,6 +3076,8 @@ def _refine_repeated_leading_header_cuts(rows, header_rows): nonempty = [text for text in signature if text] if len(nonempty) < 2: continue + if len(set(nonempty)) < 2: + continue # one repeated marker does not identify distinct columns if not any(char.isalpha() for text in nonempty for char in text): continue # a repeated all-numeric row is data, not a header repeats = [ @@ -3125,15 +3127,24 @@ def _refine_grid_tables(page, tab): # union) bounds the shaded-rectangle search. working = refine_grid_structure(page, grid, table_bbox=tab.bbox) model_grid, header_rows, body_start = _refine_model_grid(page, working) + section_titles = () if model_grid is None: body_start = 1 cuts = () else: # The model grid has one row per row of `working`, so its cut indices # index `working` directly. - cuts = _refine_repeated_leading_header_cuts( - _refine_placements_text_grid(model_grid), header_rows - ) + rows = _refine_placements_text_grid(model_grid) + cuts = _refine_repeated_leading_header_cuts(rows, header_rows) + if cuts: + from ._table_sections import section_repeat_plan + + cuts, section_titles = section_repeat_plan( + page, model_grid, rows, header_rows, cuts, CHARS._list(), + width=_refine_placement_grid_width(model_grid), + is_line_like=_is_line_like, + min_length=TableSettings().edge_min_length, + ) if cuts: boundaries = (0, *cuts, len(working)) segments = [working[start:end] for start, end in zip(boundaries, boundaries[1:])] @@ -3155,6 +3166,13 @@ def _refine_grid_tables(page, tab): # Build the tagged model on `segment` directly, not on the re-gridded # new_tab.cells, so placements match the refined grid. placements, region = _refine_build_placements(page, segment, start) + if section_titles: + from ._table_sections import apply_section_title + + placements = apply_section_title( + page, placements, section_titles, + width=_refine_placement_grid_width(placements), + ) new_tab.placements = placements new_tab.header_rows = region.top_header_rows new_tab.section_rows = region.section_header_rows diff --git a/tests/test_table_nested_candidates.py b/tests/test_table_nested_candidates.py new file mode 100644 index 000000000..a035ce810 --- /dev/null +++ b/tests/test_table_nested_candidates.py @@ -0,0 +1,74 @@ +"""Nested rule grids are cell text only under a retained native container.""" +import unittest + +import pymupdf +from pymupdf import _table_union as union + + +def entry(box, rows=2, cols=2): + x0, y0, x1, y1 = box + grid = [[(x0+(x1-x0)*c/cols, y0+(y1-y0)*r/rows, + x0+(x1-x0)*(c+1)/cols, y0+(y1-y0)*(r+1)/rows) + for c in range(cols)] for r in range(rows)] + return pymupdf.Rect(box), grid + + +class NestedCandidateTests(unittest.TestCase): + def setUp(self): + self.outer = entry((0, 0, 100, 100)) + self.left = entry((5, 5, 40, 40)) + self.right = entry((55, 55, 90, 90)) + self.args = dict(page=None, grid_ref=False, grid_ref_iou=.9, + span_mult_gate=False, span_mult_threshold=3) + + def test_single_actual_cell_and_order(self): + nested = union._union_nested_candidate_indexes + self.assertEqual(nested([self.outer, self.left, self.right]), {1, 2}) + self.assertEqual(nested([self.left, self.right, self.outer]), {0, 1}) + self.assertEqual(nested([self.outer, entry((0, 0, 50, 50))]), {1}) + self.assertEqual(nested([self.outer, self.left, entry((10, 10, 20, 20))]), {1, 2}) + + def test_bbox_alone_is_not_cell_nesting(self): + nested = union._union_nested_candidate_indexes + for child in [entry((25, 25, 75, 75)), entry((75, 75, 125, 125)), self.outer]: + with self.subTest(child=child[0]): + self.assertFalse(nested([self.outer, child])) + gap = (pymupdf.Rect(0, 0, 100, 100), [[None, (50, 0, 100, 100)]]) + self.assertFalse(nested([gap, self.left])) + self.assertFalse(nested([self.left, self.right])) + + def test_only_approved_containers_may_shadow_children(self): + nested = union._union_nested_candidate_indexes + candidates = [self.outer, self.left] + self.assertFalse(nested(candidates, set())) + self.assertEqual(nested(candidates, {0}), {1}) + + def test_retained_outer_is_not_rebuilt(self): + result = union._union_replace_append([], [self.outer, self.left, self.right], **self.args) + self.assertEqual(len(result), 1) + self.assertIs(result[0], self.outer) + self.assertEqual(len(result[0][1]), 2) + + def test_ambiguous_raw_container_does_not_remove_valid_child(self): + primary = entry((0, 0, 100, 30)) + low = entry((10, 60, 25, 75)) + result = union._union_replace_append([primary], [self.outer, low], **self.args) + self.assertEqual(len(result), 2) + self.assertIs(result[1], low) + + def test_internal_children_do_not_replace_approved_grid_ref(self): + result = union._union_replace_append([self.outer], [self.outer, self.left, self.right], + **dict(self.args, grid_ref=True)) + self.assertEqual(len(result), 1) + self.assertIs(result[0][1], self.outer[1]) + self.assertEqual(result[0][0], self.outer[0]) + + def test_layout_overmerge_does_not_make_independent_panels_nested(self): + result = union._union_replace_append([self.outer], [self.left, self.right], **self.args) + self.assertEqual(result, [self.left, self.right]) + self.assertEqual(union._union_replace_append([], [self.left, self.right], **self.args), + [self.left, self.right]) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_table_section_titles.py b/tests/test_table_section_titles.py new file mode 100644 index 000000000..1fffc149b --- /dev/null +++ b/tests/test_table_section_titles.py @@ -0,0 +1,128 @@ +"""Centered section titles preserve real slots, text ownership and body cells.""" +import unittest +from types import SimpleNamespace + +import pymupdf +from pymupdf import table +from pymupdf._table_headers import section_label_row_like +from pymupdf._table_refine import _refine_word_in_rect +from pymupdf._table_sections import ( + aligned_blocks, centered_extent, merge_title_row, section_repeat_plan, + title_row_proof, +) +from pymupdf._table_spans import SpanCell + + +class SectionTitleTests(unittest.TestCase): + words = [(42, 5, 68, 15, 'Section')] + chars = [dict(x0=44, top=5, x1=46, bottom=15, text='S', bold=True)] + + def grid(self): + row = [SpanCell((0, 0, 40, 20), '', 4, 1, 'th'), + SpanCell((40, 0, 60, 20), 'Section', 2, 1, 'th'), + SpanCell((60, 0, 110, 20), '', 5, 1, 'th')] + row[1].source_marker = object() + body = [SpanCell((i*10, 20, (i+1)*10, 40), str(i), 1, 1) for i in range(11)] + return [row, body] + + def members(self, rect): + return tuple(i for i, word in enumerate(self.words) + if _refine_word_in_rect(*word[:4], *tuple(rect))) + + def proof(self, grid, **changes): + kwargs = dict(words=self.words, word_members=self.members, + chars=self.chars, char_members=lambda rect: (0,), paths=[], + width=11, section_like=section_label_row_like, + Rect=pymupdf.Rect, is_line_like=table._is_line_like, min_length=3) + kwargs.update(changes) + return title_row_proof(grid, 0, **kwargs) + + def test_real_colspan_geometry_source_and_body(self): + grid = self.grid() + proof, _ = self.proof(grid) + self.assertIsNotNone(proof) + merged, event = merge_title_row(grid, [proof], width=11, Rect=pymupdf.Rect, + word_members=self.members) + self.assertEqual(merged[0][0].colspan, 11) + self.assertEqual(merged[0][0].bbox, (0, 0, 110, 20)) + self.assertIs(merged[0][0].source_marker, grid[0][1].source_marker) + self.assertIs(merged[1], grid[1]) + self.assertEqual(grid[0][1].colspan, 2) + self.assertEqual(event['word_indices'], (0,)) + + def test_centered_extent_boundary(self): + for title, expected in [((42, 68), True), ((45.5, 75.5), True), + ((45.6, 75.6), False), ((5, 25), False), + ((-1, 50), False), ((0, 110), False)]: + with self.subTest(title=title): + self.assertEqual(centered_extent(title, (0, 110)), expected) + + def test_competing_content_and_rowspan_keep_original(self): + for change in ['other_text', 'rowspan', 'slots', 'unequal_band']: + with self.subTest(change=change): + grid = self.grid() + if change == 'other_text': + grid[0][0].text = 'Other' + elif change == 'rowspan': + grid[0][0].rowspan = 2 + elif change == 'slots': + grid[0][0].colspan = 3 + else: + grid[0][0].bbox = (0, 1, 40, 20) + self.assertIsNone(self.proof(grid)[0]) + self.assertIsNone(self.proof(self.grid(), chars=[dict(self.chars[0], bold=False)])[0]) + self.assertIsNone(self.proof(self.grid(), + word_members=lambda r: (0,) if tuple(r) == (40, 0, 60, 20) else (0, 1))[0]) + + def test_native_partitions_are_scoped_to_title_band(self): + def line(x, y0, y1): + return dict(type='s', items=[('l', pymupdf.Point(x, y0), pymupdf.Point(x, y1))]) + cases = [([line(30, 0, 20)], False), + ([line(0, 0, 20), line(110, 0, 20)], True), + ([line(30, 20, 40)], True), + ([dict(type='f', items=[('re', pymupdf.Rect(29, 0, 31, 20), 1)])], False), + ([dict(type='f', items=[('re', pymupdf.Rect(0, 0, 110, 20), 1)])], True), + ([dict(type='s', items=[('re', pymupdf.Rect(30, 0, 80, 20), 1)])], False)] + for paths, expected in cases: + with self.subTest(paths=paths): + self.assertEqual(self.proof(self.grid(), paths=paths)[0] is not None, expected) + + def test_changed_source_or_title_does_not_merge(self): + grid = self.grid() + proof, _ = self.proof(grid) + self.assertIsNone(merge_title_row(grid, [proof], width=11, Rect=pymupdf.Rect, + word_members=lambda rect: (1,))[1]) + grid[0][1].text = 'Changed' + self.assertIsNone(merge_title_row(grid, [proof], width=11, Rect=pymupdf.Rect, + word_members=self.members)[1]) + + def test_all_blocks_proven_before_moving_cut(self): + rows = [['First', ''], ['A', 'B'], ['x', '1'], + ['Second', ''], ['A', 'B'], ['y', '2']] + result = aligned_blocks(rows, 2, [[]]*6, repeat_labels=lambda r, d: (4,), + prove=lambda index: ({'row': index}, 'ok')) + self.assertEqual(result[0], (3,)) + self.assertEqual(len(result[1]), 2) + rejected = aligned_blocks(rows, 2, [[]]*6, repeat_labels=lambda r, d: (4,), + prove=lambda index: (None, 'unproved')) + self.assertEqual(rejected[0], (4,)) + self.assertFalse(rejected[1]) + + def test_missing_cached_material_does_not_extract(self): + page = SimpleNamespace() + self.assertEqual(section_repeat_plan(page, self.grid(), [], 1, (4,), [], + width=11, is_line_like=table._is_line_like, min_length=3), ((4,), ())) + + def test_repeated_labels_require_two_distinct_types(self): + cut = table._refine_repeated_leading_header_cuts + for labels, expected in [(['A', 'A', 'A'], ()), + (['A', 'A', 'B'], (2,)), + (['(omitted)', '(omitted)', ''], ()), + (['(Actual)', '(Budget)', ''], (2,))]: + with self.subTest(labels=labels): + rows = [labels, ['x', '1', 'y'], labels, ['z', '2', 'w']] + self.assertEqual(cut(rows, 1), expected) + + +if __name__ == '__main__': + unittest.main()