Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -665,6 +665,7 @@ def build():
ret.append( (f'{g_root}/src/_table_spans.py', to_dir) )
ret.append( (f'{g_root}/src/_table_union.py', to_dir) )
ret.append( (f'{g_root}/src/_table_headers.py', to_dir) )
ret.append( (f'{g_root}/src/_table_sections.py', to_dir) )
ret.append( (f'{g_root}/src/utils.py', to_dir) )
ret.append( (f'{g_root}/src/_wxcolors.py', to_dir) )
ret.append( (f'{g_root}/src/_apply_pages.py', to_dir) )
Expand Down
198 changes: 198 additions & 0 deletions src/_table_sections.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,198 @@
"""
Copyright (C) 2023 Artifex Software, Inc.

This file is part of PyMuPDF.

PyMuPDF is free software: you can redistribute it and/or modify it under the
terms of the GNU Affero General Public License as published by the Free
Software Foundation, either version 3 of the License, or (at your option)
any later version.

PyMuPDF is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
details.

You should have received a copy of the GNU Affero General Public License
along with MuPDF. If not, see <https://www.gnu.org/licenses/agpl-3.0.en.html>

Alternative licensing terms are available from the licensor.
For commercial licensing, see <https://www.artifex.com/> or contact
Artifex Software, Inc., 39 Mesa Street, Suite 108A, San Francisco,
CA 94129, USA, for further information.

---------------------------------------------------------------------
PyMuPDF centered section-title placements.
"""

"""Whole-table specialization of the existing H6 centered-title extent rule.

Only proven, bold single-title/repeated-label blocks enter this policy. Existing
native words/drawings/characters are supplied by the refine call; no extraction,
layout decision, inference, GT or filename is consulted. Partial extents abstain.
"""
from copy import copy


def centered_extent(title, bounds):
"""H6's existing 5%-of-interval-width test, restricted to the whole row."""
x0, x1 = title
left, right = bounds
if not (left <= x0 < x1 <= right and x1 - x0 < right - left):
return False
return abs((x0 + x1 - left - right) * 0.5) <= 0.05 * (right - left)


def interior_vertical(paths, band, *, Rect, is_line_like, min_length):
"""True physical interior partitions at the title band, not TGIF slots.

Keep native segment extents: the existing border-line sets discard them.
Outer borders and underlines do not partition a title. Reuse the finder's
existing simulated-line predicate and minimum length, without new cutoffs.
"""
x0, y0, x1, y1 = band
for path in paths:
for item in path['items']:
if item[0] == 'l':
a, b = item[1:3]
if a.x != b.x:
continue
segments = [(a.x, min(a.y, b.y), max(a.y, b.y))]
elif item[0] == 're':
r = Rect(item[1])
if path.get('type') in ('s', 'fs'):
segments = [(r.x0, r.y0, r.y1), (r.x1, r.y0, r.y1)]
elif is_line_like(r, min_length) and r.width < r.height:
segments = [((r.x0 + r.x1) * .5, r.y0, r.y1)]
else:
continue
else:
continue
if any(x0 < x < x1 and high - low >= min_length
and low < y1 and y0 < high for x, low, high in segments):
return True
return False


def title_row_proof(grid, index, words, word_members, chars, char_members,
paths, *, width, section_like, Rect, is_line_like,
min_length):
row = grid[index]
occupied = [cell for cell in row if cell.text.strip()]
if len(occupied) != 1 or not section_like([cell.text for cell in row]):
return None, 'not_single_section_title'
if not row or any(cell.bbox is None or cell.rowspan != 1 for cell in row):
return None, 'nonflat_title_row'
if any(ri + cell.rowspan > index for ri, earlier in enumerate(grid[:index])
for cell in earlier):
return None, 'incoming_rowspan'
if sum(cell.colspan for cell in row) != width:
return None, 'incomplete_title_slots'
boxes = [Rect(cell.bbox) for cell in row]
if len({(r.y0, r.y1) for r in boxes}) != 1:
return None, 'unequal_title_bands'
band = (min(r.x0 for r in boxes), boxes[0].y0,
max(r.x1 for r in boxes), boxes[0].y1)
if interior_vertical(paths, band, Rect=Rect, is_line_like=is_line_like,
min_length=min_length):
return None, 'physical_vertical_partition'
title = occupied[0]
selected = tuple(word_members(Rect(title.bbox)))
whole_row = tuple(word_members(Rect(band)))
if not selected or selected != whole_row:
return None, 'competing_or_unproved_title_words'
actual = tuple(char_members(Rect(title.bbox)))
if not actual or not all(chars[i].get('bold', False) for i in actual):
return None, 'section_emphasis_not_proven'
extent = min(words[i][0] for i in selected), max(words[i][2] for i in selected)
if not centered_extent(extent, (band[0], band[2])):
return None, 'not_centered_over_whole_row'
return {'band': band, 'text': title.text, 'word_indices': selected,
'width': width, 'word_extent': extent}, 'centered_title'


def aligned_blocks(rows, depth, grid, *, repeat_labels, prove):
cuts = tuple(repeat_labels(rows, depth))
if not cuts or grid is None:
return cuts, (), 'no_repeated_label_blocks'
normalized = [tuple(text.casefold() for text in row) for row in rows]
signature = normalized[cuts[0]]
first = next(i for i, row in enumerate(normalized) if row == signature)
proofs = []
for index, end in zip([first, *cuts], [*(cut - 1 for cut in cuts), len(rows)]):
if index == 0:
return cuts, (), 'untitled_repeat'
if not any(any(text.strip() for text in row) for row in rows[index + 1:end]):
return cuts, (), 'no_record_after_labels'
proof, reason = prove(index - 1)
if proof is None:
return cuts, (), reason
proofs.append(proof)
return tuple(cut - 1 for cut in cuts), tuple(proofs), 'centered_section_aligned'


def merge_title_row(grid, proofs, *, width, Rect, word_members):
"""Return actual placements, preserving all body cells and source fields."""
if not grid:
return grid, None
row = grid[0]
populated = [cell for cell in row if cell.text.strip()]
if len(populated) != 1 or any(cell.bbox is None or cell.rowspan != 1 for cell in row):
return grid, None
boxes = [Rect(cell.bbox) for cell in row]
band = (min(r.x0 for r in boxes), min(r.y0 for r in boxes),
max(r.x1 for r in boxes), max(r.y1 for r in boxes))
title = populated[0]
proof = next((p for p in proofs if p['band'] == band and p['text'] == title.text
and p['width'] == width and tuple(word_members(Rect(band))) == p['word_indices']), None)
if proof is None or sum(cell.colspan for cell in row) != width:
return grid, None
merged = copy(title)
merged.bbox, merged.colspan, merged.rowspan = band, width, 1
return [[merged], *grid[1:]], {'text': title.text, 'bbox': band, 'colspan': width,
'word_indices': proof['word_indices']}

import pymupdf
from pymupdf._table_headers import section_label_row_like
from pymupdf._table_refine import (
_WordCenterIndex, _refine_word_candidates, _refine_word_in_rect,
)


def _section_word_members(page, words, rect):
return tuple(i for i, word in _refine_word_candidates(page, words, rect)
if word[4].strip() and _refine_word_in_rect(*word[:4], *tuple(rect)))


def section_repeat_plan(page, grid, rows, depth, cuts, chars, *, width,
is_line_like, min_length):
"""Plan section cuts using materials already prepared by find_tables.

Missing caches abstain; never re-extract page text or drawings. The plan is
scoped to the current refined parent and carries real page-word indices.
"""
words = getattr(page, '_refine_words_cache', None)
cache = getattr(page, '_table_drawings_cache', None)
if not cuts or grid is None or words is None or not cache:
return cuts, ()
char_words = [(c['x0'], c['top'], c['x1'], c['bottom'], c['text']) for c in chars]
index = _WordCenterIndex(char_words)
def char_members(rect):
return tuple(i for i in index.candidates(rect) if chars[i]['text'].strip()
and _refine_word_in_rect(*char_words[i][:4], *tuple(rect)))
def prove(row_index):
return title_row_proof(grid, row_index, words,
lambda rect: _section_word_members(page, words, rect), chars,
char_members, cache[0], width=width,
section_like=section_label_row_like, Rect=pymupdf.Rect,
is_line_like=is_line_like, min_length=min_length)
aligned, proofs, _reason = aligned_blocks(rows, depth, grid,
repeat_labels=lambda _rows, _depth: cuts, prove=prove)
return aligned, proofs


def apply_section_title(page, grid, proofs, *, width):
"""Apply the actual title placement after strict final span resolution."""
merged, _event = merge_title_row(grid, proofs, width=width, Rect=pymupdf.Rect,
word_members=lambda rect: _section_word_members(page, page._refine_words_cache, rect))
return merged
35 changes: 33 additions & 2 deletions src/_table_union.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,28 @@ def _layout_table_grids(page):
return grids


def _union_nested_candidate_indexes(candidates, containers=None):
"""Rule grids inside one actual cell of another rule grid are cell text.

Use only already-admitted native candidates. Containment by a large layout
box is not nesting: independent panels in an overmerged region must survive.
Equal boxes, mere overlaps and children crossing parent cells stay candidates.
"""
nested = set()
for parent_index in (range(len(candidates)) if containers is None else containers):
parent_bbox, parent_grid = candidates[parent_index]
cell_rects = None
for index, (bbox, _grid) in enumerate(candidates):
if index == parent_index or bbox == parent_bbox or bbox not in parent_bbox:
continue
if cell_rects is None:
cell_rects = [pymupdf.Rect(cell) for row in parent_grid
for cell in row if cell is not None]
if any(bbox in cell for cell in cell_rects):
nested.add(index)
return nested


def _union_line_candidates(page, *, add_lines=None, add_boxes=None):
"""Line-based table candidates for the union stage as ``(bbox, grid)`` pairs.

Expand Down Expand Up @@ -509,8 +531,10 @@ def _union_replace_append(existing, candidates, *, page, grid_ref, grid_ref_iou,
grid_refs, consumed = {}, set()
replacements = {}
append_candidates = []
# A validated grid-ref already owns the native cells containing these children.
nested_in_refs = _union_nested_candidate_indexes(candidates, consumed)
for candidate_index, candidate in enumerate(candidates):
if candidate_index in consumed:
if candidate_index in consumed or candidate_index in nested_in_refs:
continue
owner_index, ambiguous = _union_find_owner(candidate[0], existing_bboxes)
if owner_index is not None:
Expand All @@ -534,7 +558,14 @@ def _union_replace_append(existing, candidates, *, page, grid_ref, grid_ref_iou,
else:
entries.append(entry)
entries.extend(append_candidates)
return entries
# A ruled child becomes cell text only if its ruled container was selected.
# Keep the normal grid-ref gates and the original replacement decision:
# rejecting a raw parent proposal must not suppress an independent child.
selected_grids = {id(grid) for _bbox, grid in entries}
ruled = [entry for entry in candidates if id(entry[1]) in selected_grids]
nested = _union_nested_candidate_indexes(ruled)
nested_grids = {id(ruled[index][1]) for index in nested}
return [entry for entry in entries if id(entry[1]) not in nested_grids]


def _find_tables_union(page, *, add_lines=None, add_boxes=None):
Expand Down
26 changes: 22 additions & 4 deletions src/table.py
Original file line number Diff line number Diff line change
Expand Up @@ -3037,7 +3037,7 @@ def _refine_repeated_leading_header_cuts(rows, header_rows):

Several tables printed one under another with no gap are detected as one
grid, and the giveaway is that their shared header row recurs verbatim. The
signature is the first row that carries at least two non-empty cells in a
signature is the first row that carries at least two distinct non-empty labels in a
grid at least two columns wide and is not purely numeric or punctuation; it
must also be a leading row -- inside the header region the header rules found,
or within the first three rows, because that finder is conservative and a
Expand Down Expand Up @@ -3076,6 +3076,8 @@ def _refine_repeated_leading_header_cuts(rows, header_rows):
nonempty = [text for text in signature if text]
if len(nonempty) < 2:
continue
if len(set(nonempty)) < 2:
continue # one repeated marker does not identify distinct columns
if not any(char.isalpha() for text in nonempty for char in text):
continue # a repeated all-numeric row is data, not a header
repeats = [
Expand Down Expand Up @@ -3125,15 +3127,24 @@ def _refine_grid_tables(page, tab):
# union) bounds the shaded-rectangle search.
working = refine_grid_structure(page, grid, table_bbox=tab.bbox)
model_grid, header_rows, body_start = _refine_model_grid(page, working)
section_titles = ()
if model_grid is None:
body_start = 1
cuts = ()
else:
# The model grid has one row per row of `working`, so its cut indices
# index `working` directly.
cuts = _refine_repeated_leading_header_cuts(
_refine_placements_text_grid(model_grid), header_rows
)
rows = _refine_placements_text_grid(model_grid)
cuts = _refine_repeated_leading_header_cuts(rows, header_rows)
if cuts:
from ._table_sections import section_repeat_plan

cuts, section_titles = section_repeat_plan(
page, model_grid, rows, header_rows, cuts, CHARS._list(),
width=_refine_placement_grid_width(model_grid),
is_line_like=_is_line_like,
min_length=TableSettings().edge_min_length,
)
if cuts:
boundaries = (0, *cuts, len(working))
segments = [working[start:end] for start, end in zip(boundaries, boundaries[1:])]
Expand All @@ -3155,6 +3166,13 @@ def _refine_grid_tables(page, tab):
# Build the tagged model on `segment` directly, not on the re-gridded
# new_tab.cells, so placements match the refined grid.
placements, region = _refine_build_placements(page, segment, start)
if section_titles:
from ._table_sections import apply_section_title

placements = apply_section_title(
page, placements, section_titles,
width=_refine_placement_grid_width(placements),
)
new_tab.placements = placements
new_tab.header_rows = region.top_header_rows
new_tab.section_rows = region.section_header_rows
Expand Down
74 changes: 74 additions & 0 deletions tests/test_table_nested_candidates.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
"""Nested rule grids are cell text only under a retained native container."""
import unittest

import pymupdf
from pymupdf import _table_union as union


def entry(box, rows=2, cols=2):
x0, y0, x1, y1 = box
grid = [[(x0+(x1-x0)*c/cols, y0+(y1-y0)*r/rows,
x0+(x1-x0)*(c+1)/cols, y0+(y1-y0)*(r+1)/rows)
for c in range(cols)] for r in range(rows)]
return pymupdf.Rect(box), grid


class NestedCandidateTests(unittest.TestCase):
def setUp(self):
self.outer = entry((0, 0, 100, 100))
self.left = entry((5, 5, 40, 40))
self.right = entry((55, 55, 90, 90))
self.args = dict(page=None, grid_ref=False, grid_ref_iou=.9,
span_mult_gate=False, span_mult_threshold=3)

def test_single_actual_cell_and_order(self):
nested = union._union_nested_candidate_indexes
self.assertEqual(nested([self.outer, self.left, self.right]), {1, 2})
self.assertEqual(nested([self.left, self.right, self.outer]), {0, 1})
self.assertEqual(nested([self.outer, entry((0, 0, 50, 50))]), {1})
self.assertEqual(nested([self.outer, self.left, entry((10, 10, 20, 20))]), {1, 2})

def test_bbox_alone_is_not_cell_nesting(self):
nested = union._union_nested_candidate_indexes
for child in [entry((25, 25, 75, 75)), entry((75, 75, 125, 125)), self.outer]:
with self.subTest(child=child[0]):
self.assertFalse(nested([self.outer, child]))
gap = (pymupdf.Rect(0, 0, 100, 100), [[None, (50, 0, 100, 100)]])
self.assertFalse(nested([gap, self.left]))
self.assertFalse(nested([self.left, self.right]))

def test_only_approved_containers_may_shadow_children(self):
nested = union._union_nested_candidate_indexes
candidates = [self.outer, self.left]
self.assertFalse(nested(candidates, set()))
self.assertEqual(nested(candidates, {0}), {1})

def test_retained_outer_is_not_rebuilt(self):
result = union._union_replace_append([], [self.outer, self.left, self.right], **self.args)
self.assertEqual(len(result), 1)
self.assertIs(result[0], self.outer)
self.assertEqual(len(result[0][1]), 2)

def test_ambiguous_raw_container_does_not_remove_valid_child(self):
primary = entry((0, 0, 100, 30))
low = entry((10, 60, 25, 75))
result = union._union_replace_append([primary], [self.outer, low], **self.args)
self.assertEqual(len(result), 2)
self.assertIs(result[1], low)

def test_internal_children_do_not_replace_approved_grid_ref(self):
result = union._union_replace_append([self.outer], [self.outer, self.left, self.right],
**dict(self.args, grid_ref=True))
self.assertEqual(len(result), 1)
self.assertIs(result[0][1], self.outer[1])
self.assertEqual(result[0][0], self.outer[0])

def test_layout_overmerge_does_not_make_independent_panels_nested(self):
result = union._union_replace_append([self.outer], [self.left, self.right], **self.args)
self.assertEqual(result, [self.left, self.right])
self.assertEqual(union._union_replace_append([], [self.left, self.right], **self.args),
[self.left, self.right])


if __name__ == '__main__':
unittest.main()
Loading
Loading