diff --git a/docs/source/modules/io.rst b/docs/source/modules/io.rst index f57e42ba15..b137041210 100644 --- a/docs/source/modules/io.rst +++ b/docs/source/modules/io.rst @@ -177,3 +177,47 @@ page, so exporting one page to several formats orders it only once. .. autofunction:: doctr.io.exporters.predictions_in_reading_order .. autofunction:: doctr.io.exporters.to_json_safe + +Figures +------- + +When the predictor runs with ``detect_layout=True``, the figures found by the layout model take part in the +reading order and are materialized by the Markdown, AsciiDoc and HTML exports. How they are materialized is +controlled by the ``images`` argument, which accepts either an image mode or a configured +:class:`FigureEncoder`: + +* ``'placeholder'`` (the default): a comment marks where a figure was detected, without touching the pixels +* ``'none'``: the figures are dropped entirely +* ``'embedded'``: each figure is cropped out of the page and inlined as a base64 data URI +* ``'referenced'``: each crop is written next to the export and referenced by a relative path + +.. code:: python + + from doctr.io import DocumentFile, FigureEncoder + from doctr.models import ocr_predictor + + predictor = ocr_predictor(pretrained=True, detect_layout=True) + doc = predictor(DocumentFile.from_pdf("report.pdf")) + + # A self-contained Markdown file + markdown = doc.export_as_markdown(images="embedded") + # ... or one that points at the crops on disk + markdown = doc.export_as_markdown(images=FigureEncoder("referenced", image_dir="assets", path_prefix="assets/")) + +A caption detected next to a figure becomes its alternative text (and its ``
`` in HTML) as soon as +the export carries the pixels. Plain text and the hOCR export never inline an image: hOCR positions each figure +as an ``ocr_photo`` area instead. Pages restored from a JSON export carry no pixels, so their figures fall back +to a placeholder. + +.. autoclass:: FigureEncoder + :members: resolve, source, enabled, materializes, materializes_on + +.. autofunction:: crop_layout_region + +.. autofunction:: encode_crop + +.. autofunction:: picture_regions + +.. autofunction:: is_picture_region + +.. autofunction:: is_picture_label diff --git a/docs/source/using_doctr/using_models.rst b/docs/source/using_doctr/using_models.rst index 8209e1737d..eb50011ce3 100644 --- a/docs/source/using_doctr/using_models.rst +++ b/docs/source/using_doctr/using_models.rst @@ -474,6 +474,21 @@ In addition to running the :py:meth:`layout_predictor `. +The figures found by the layout model also take part in the reading order, and the Markdown / AsciiDoc / HTML exports can materialize them, either inlined as base64 data URIs or written next to the export (see :ref:`Figures` for the details): + +.. code:: python3 + + # A self-contained Markdown file, figures included + markdown = result.export_as_markdown(images="embedded") + + # ... or one referencing the crops written to an `assets` directory + from doctr.io import FigureEncoder + + encoder = FigureEncoder("referenced", image_dir="assets", path_prefix="assets/") + markdown = result.export_as_markdown(images=encoder) + +By default (``images="placeholder"``) a comment marks the position of every detected figure, and ``images="none"`` drops them entirely. + Running the predictors on GPU ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/doctr/io/__init__.py b/doctr/io/__init__.py index be8e911358..1744c3b3d8 100644 --- a/doctr/io/__init__.py +++ b/doctr/io/__init__.py @@ -1,5 +1,6 @@ from .elements import * from .exporters import * +from .figures import * from .html import * from .image import * from .pdf import * diff --git a/doctr/io/exporters.py b/doctr/io/exporters.py index a4a4e7caf8..7247bd564d 100644 --- a/doctr/io/exporters.py +++ b/doctr/io/exporters.py @@ -12,10 +12,11 @@ import numpy as np import doctr +from doctr.io.figures import FigureEncoder, is_picture_label, picture_regions from doctr.utils.common_types import BoundingBox if TYPE_CHECKING: # pragma: no cover - from doctr.io.elements import Block, KIEPage, Line, Page, Table + from doctr.io.elements import Block, KIEPage, LayoutElement, Line, Page, Table __all__ = [ "AsciiDocExporter", @@ -92,14 +93,16 @@ def _covering_region_indices(geoms: list[Any], region_geoms: list[Any], min_cove def _reading_order_signature(page: "Page", direction: str) -> tuple[Any, ...]: """A cheap structural fingerprint of a page, used to invalidate the reading-order cache. - Covers the requested direction and the identity (plus line count) of every block and table, so - replacing or re-grouping the page content invalidates the cache. In-place edits to a `Line`'s words - are not detected; callers mutating a page that deeply should drop `_reading_order_cache` themselves. + Covers the requested direction and the identity (plus line count) of every block, table and layout + region, so replacing or re-grouping the page content invalidates the cache. In-place edits to a + `Line`'s words are not detected; callers mutating a page that deeply should drop + `_reading_order_cache` themselves. """ return ( direction, tuple((id(block), len(block.lines)) for block in page.blocks), tuple(id(table) for table in getattr(page, "tables", ()) or ()), + tuple(id(region) for region in getattr(page, "layout", ()) or ()), ) @@ -112,7 +115,7 @@ def _store_reading_order(page: "Page", signature: tuple[Any, ...], result: tuple def page_reading_order(page: "Page", direction: str = "auto") -> tuple[list[Any], list[str | None], str]: - """Linearize the content of a page (blocks & tables) in reading order. + """Linearize the content of a page (blocks, tables & figures) in reading order. The result is memoized on the page: every exporter calls this, so a page exported to several formats (or built with `keep_reading_order=True` and then exported) orders its content once. @@ -122,10 +125,10 @@ def page_reading_order(page: "Page", direction: str = "auto") -> tuple[list[Any] direction: reading direction, one of 'auto', 'ltr', 'rtl', 'ttb-rtl' or 'ttb-ltr' Returns: - a tuple with the ordered items (blocks & tables), their layout label (None without layout) and the - effective reading direction + a tuple with the ordered items (blocks, tables & picture regions), their layout label (None without + layout) and the effective reading direction """ - from doctr.io.elements import Block, Table + from doctr.io.elements import Block, LayoutElement, Table from doctr.models.reading_order import ( ReadingOrderPredictor, assign_layout_labels, @@ -147,7 +150,10 @@ def page_reading_order(page: "Page", direction: str = "auto") -> tuple[list[Any] region_labels = [region.type for region in page.layout] lines = [line for block in page.blocks for line in block.lines] - elements: list[Any] = [*lines, *page.tables] + # Figures take part in the ordering itself: `sort_reading_order` treats them as floats (never merged + # with their neighbors) and attaches the surrounding captions to them + figures = picture_regions(page) + elements: list[Any] = [*lines, *page.tables, *figures] if len(elements) == 0: _store_reading_order(page, signature, ([], [], direction)) return [], [], direction @@ -162,7 +168,10 @@ def page_reading_order(page: "Page", direction: str = "auto") -> tuple[list[Any] elt_labels: list[str | None] = [None] * len(elements) if len(region_geoms) > 0: elt_labels = assign_layout_labels(elt_geoms, region_geoms, region_labels) - elt_labels = ["Table" if isinstance(elt, Table) else label for elt, label in zip(elements, elt_labels)] + elt_labels = [ + "Table" if isinstance(elt, Table) else elt.type if isinstance(elt, LayoutElement) else label + for elt, label in zip(elements, elt_labels) + ] segments = resolve_reading_segments(elt_geoms, direction=direction, labels=elt_labels) items = [] @@ -184,9 +193,10 @@ def _claim_artefacts(block_lines: list[Any]) -> list[Any]: for segment in segments: first = elements[segment[0]] seg_label = elt_labels[segment[0]] - if isinstance(first, Table): + if isinstance(first, (Table, LayoutElement)): + # Floats are never merged with their neighbors, so the segment holds this item alone items.append(first) - labels.append("Table") + labels.append("Table" if isinstance(first, Table) else seg_label) open_list_region = None continue if normalize_layout_label(seg_label) in _LIST_LABELS: @@ -279,14 +289,18 @@ class _PageTextExporter: """Shared logic of the reading-order-aware text exporters. Subclasses define the format specifics: heading prefixes (per normalized layout label), the bullet - prefix, character escaping, line finalization (neutralizing markers a line must not start with) and the - table rendering. + prefix, character escaping, line finalization (neutralizing markers a line must not start with), the + table rendering and the figure rendering. """ headings: ClassVar[dict[str, str]] = {} bullet: ClassVar[str] = "- " block_break: ClassVar[str] = "\n\n" page_break: ClassVar[str] = "\n\n" + # Whether the format can carry an image at all (plain text cannot, and drops the figures silently) + supports_figures: ClassVar[bool] = False + # Marks a detected figure whose pixels are not materialized + figure_placeholder: ClassVar[str] = "" def escape_text(self, text: str) -> str: """Escape the characters carrying a structural meaning in the target format""" @@ -300,6 +314,19 @@ def render_table(self, table: "Table", escape: bool = True) -> str: """Render a recognized table in the target format""" raise NotImplementedError + def render_figure(self, source: str | None, caption: str | None = None) -> str: + """Render a figure detected by the layout model in the target format. + + Args: + source: the image source (a data URI or a relative path), or None when the pixels were not + materialized, in which case the placeholder is emitted + caption: the caption detected next to the figure, if any + + Returns: + the figure markup, or an empty string when the format cannot carry it + """ + return self.figure_placeholder + def class_header(self, class_name: str, escape: bool = True) -> str: """Render the header of a detection class in a KIE export""" raise NotImplementedError @@ -309,6 +336,43 @@ def _line_text(self, line: "Line", direction: str, escape: bool) -> str: text = " ".join(word.render() for word in ordered_line_words(line, direction)) return self.escape_text(text) if escape else text + def _block_lines(self, block: "Block", direction: str, escape: bool, auto: bool) -> list[str]: + """The non-empty rendered lines of a block, in reading order.""" + lines = [self._line_text(line, _line_render_direction(line, direction, auto), escape) for line in block.lines] + return [line for line in lines if line.strip()] + + def _figure_markup( + self, + page: "Page", + items: list[Any], + labels: list[str | None], + index: int, + encoder: FigureEncoder, + figure_count: int, + direction: str, + escape: bool, + auto: bool, + ) -> tuple[str, int]: + """Render the figure at `items[index]`, absorbing its caption when it carries the pixels. + + The caption is only consumed when the figure actually has a source: a placeholder cannot display + it, so it has to stay in the text flow as a regular paragraph. + + Returns: + the figure markup and the index of the next item to process + """ + from doctr.models.reading_order import normalize_layout_label + + source = encoder.source(page, items[index], figure_count) + index += 1 + caption = None + if source is not None and index < len(items) and normalize_layout_label(labels[index]) == "caption": + caption_lines = self._block_lines(items[index], direction, escape, auto) + if caption_lines: + caption = " ".join(caption_lines) + index += 1 + return self.render_figure(source, caption), index + def export_page( self, page: "Page", @@ -316,6 +380,7 @@ def export_page( escape: bool = True, include_furniture: bool = True, block_break: str | None = None, + images: "str | FigureEncoder | None" = "placeholder", ) -> str: """Export a page, with its content sorted in reading order. @@ -325,36 +390,58 @@ def export_page( escape: whether the characters or markers carrying a structural meaning should be neutralized include_furniture: whether page headers, page footers and footnotes should be included block_break: the string inserted between two blocks (the format-specific default when None) + images: how the figures detected by the layout model are materialized, either an image mode + ('none', 'placeholder', 'embedded' or 'referenced') or a configured + :class:`~doctr.io.FigureEncoder` Returns: the exported page as a string """ - from doctr.io.elements import Table + from doctr.io.elements import LayoutElement, Table from doctr.models.reading_order import layout_label_role, normalize_layout_label auto = direction == "auto" + encoder = FigureEncoder.resolve(images) + # The text detected inside a figure is only redundant once the figure carries its own pixels + drop_figure_text = self.supports_figures and encoder.materializes_on(page) items, labels, direction = page_reading_order(page, direction) parts: list[str] = [] list_group: list[str] = [] + figure_count = 0 def _flush_list() -> None: if list_group: parts.append("\n".join(list_group)) list_group.clear() - for item, label in zip(items, labels): + index = 0 + while index < len(items): + item, label = items[index], labels[index] if not include_furniture and layout_label_role(label) in ("header", "footer", "footnote"): + index += 1 + continue + if isinstance(item, LayoutElement): # a figure detected by the layout model + if not (encoder.enabled and self.supports_figures): + index += 1 + continue + _flush_list() + figure_count += 1 + rendered, index = self._figure_markup( + page, items, labels, index, encoder, figure_count, direction, escape, auto + ) + if rendered: + parts.append(rendered) continue + index += 1 if isinstance(item, Table): _flush_list() rendered = self.render_table(item, escape=escape) if rendered: parts.append(rendered) continue - item_lines = [ - self._line_text(line, _line_render_direction(line, direction, auto), escape) for line in item.lines - ] - item_lines = [line for line in item_lines if line.strip()] + if drop_figure_text and is_picture_label(label): + continue # this text is inside a figure, and already visible in the emitted image + item_lines = self._block_lines(item, direction, escape, auto) if len(item_lines) == 0: continue norm_label = normalize_layout_label(label) @@ -443,6 +530,8 @@ class MarkdownExporter(_PageTextExporter): headings: ClassVar[dict[str, str]] = {"title": "# ", "section_header": "## "} bullet: ClassVar[str] = "- " page_break: ClassVar[str] = "\n\n---\n\n" + supports_figures: ClassVar[bool] = True + figure_placeholder: ClassVar[str] = "" def escape_text(self, text: str) -> str: return "".join(f"\\{char}" if char in _MD_SPECIAL_CHARS else char for char in text) @@ -467,6 +556,14 @@ def _cell(value: str) -> str: separator = "| " + " | ".join("---" for _ in grid[0]) + " |" return "\n".join([rows[0], separator, *rows[1:]]) + def render_figure(self, source: str | None, caption: str | None = None) -> str: + """Render a figure as an image, using its caption as the alternative text""" + if source is None: + return self.figure_placeholder + # The alternative text sits inside a link label: only the delimiters have to be neutralized + alt = (caption or "").replace("[", "\\[").replace("]", "\\]") + return f"![{alt}]({source})" + def class_header(self, class_name: str, escape: bool = True) -> str: return f"**{self.escape_text(class_name) if escape else class_name}**" @@ -481,6 +578,8 @@ class AsciiDocExporter(_PageTextExporter): headings: ClassVar[dict[str, str]] = {"title": "== ", "section_header": "=== "} bullet: ClassVar[str] = "* " page_break: ClassVar[str] = "\n\n<<<\n\n" + supports_figures: ClassVar[bool] = True + figure_placeholder: ClassVar[str] = "// image" def escape_text(self, text: str) -> str: return "".join(f"\\{char}" if char in _ADOC_SPECIAL_CHARS else char for char in text) @@ -505,6 +604,15 @@ def _row(row: list[str]) -> str: return "\n".join(["|===", _row(grid[0]), "", *[_row(row) for row in grid[1:]], "|==="]) + def render_figure(self, source: str | None, caption: str | None = None) -> str: + """Render a figure as a block image macro, titled with its caption""" + if source is None: + return self.figure_placeholder + title = f".{caption}\n" if caption else "" + # The alternative text sits between the brackets of the macro: only those have to be neutralized + alt = (caption or "").replace("[", "\\[").replace("]", "\\]") + return f"{title}image::{source}[{alt}]" + def class_header(self, class_name: str, escape: bool = True) -> str: return f"*{self.escape_text(class_name) if escape else class_name}*" @@ -528,6 +636,8 @@ class HTMLExporter(_PageTextExporter): headings: ClassVar[dict[str, str]] = {"title": "h1", "section_header": "h2"} block_break: ClassVar[str] = "\n" page_break: ClassVar[str] = "\n
\n" + supports_figures: ClassVar[bool] = True + figure_placeholder: ClassVar[str] = "" def escape_text(self, text: str) -> str: return _html_escape(text, quote=False) @@ -539,33 +649,53 @@ def export_page( escape: bool = True, include_furniture: bool = True, block_break: str | None = None, + images: "str | FigureEncoder | None" = "placeholder", ) -> str: - from doctr.io.elements import Table + from doctr.io.elements import LayoutElement, Table from doctr.models.reading_order import layout_label_role, normalize_layout_label auto = direction == "auto" + encoder = FigureEncoder.resolve(images) + # The text detected inside a figure is only redundant once the figure carries its own pixels + drop_figure_text = self.supports_figures and encoder.materializes_on(page) items, labels, direction = page_reading_order(page, direction) parts: list[str] = [] list_group: list[str] = [] + figure_count = 0 def _flush_list() -> None: if list_group: parts.append("") list_group.clear() - for item, label in zip(items, labels): + index = 0 + while index < len(items): + item, label = items[index], labels[index] if not include_furniture and layout_label_role(label) in ("header", "footer", "footnote"): + index += 1 continue + if isinstance(item, LayoutElement): # a figure detected by the layout model + if not (encoder.enabled and self.supports_figures): + index += 1 + continue + _flush_list() + figure_count += 1 + rendered, index = self._figure_markup( + page, items, labels, index, encoder, figure_count, direction, escape, auto + ) + if rendered: + parts.append(rendered) + continue + index += 1 if isinstance(item, Table): _flush_list() rendered = self.render_table(item, escape=escape) if rendered: parts.append(rendered) continue - item_lines = [ - self._line_text(line, _line_render_direction(line, direction, auto), escape) for line in item.lines - ] - item_lines = [line for line in item_lines if line.strip()] + if drop_figure_text and is_picture_label(label): + continue # this text is inside a figure, and already visible in the emitted image + item_lines = self._block_lines(item, direction, escape, auto) if len(item_lines) == 0: continue norm_label = normalize_layout_label(label) @@ -581,6 +711,16 @@ def _flush_list() -> None: _flush_list() return (self.block_break if block_break is None else block_break).join(parts) + def render_figure(self, source: str | None, caption: str | None = None) -> str: + """Render a figure as a `
` element, with its caption as a `
`""" + if source is None: + return self.figure_placeholder + # The caption reaches this point already escaped (`escape=True`), but it lands in an attribute + # value, where the double quote also has to be neutralized + alt = (caption or "").replace('"', """) + figcaption = f"\n
{caption}
" if caption else "" + return f'
{alt}{figcaption}
' + def render_table(self, table: "Table", escape: bool = True) -> str: """Render a table as an HTML table (first row used as header)""" grid = table.to_grid() @@ -664,7 +804,7 @@ class XMLExporter: >>> xml_bytes, xml_tree = XMLExporter().export_page(page) """ - ocr_capabilities: ClassVar[str] = "ocr_page ocr_carea ocr_par ocr_line ocrx_word" + ocr_capabilities: ClassVar[str] = "ocr_page ocr_carea ocr_par ocr_line ocrx_word ocr_photo" def _new_document(self, file_title: str, language: str) -> tuple[ETElement, ETElement]: """Create the hOCR root element with its , returning the root and its element.""" @@ -742,6 +882,36 @@ def _add_table( cell_span.text = cell.value return table_count + 1 + def _add_figure( + self, page_div: ETElement, region: "LayoutElement", width: int, height: int, figure_count: int + ) -> int: + """Serialize a figure detected by the layout model as an hOCR `ocr_photo` area. + + The pixels stay in the page image: hOCR describes the region, it does not carry it. + + Args: + page_div: the `ocr_page` element the figure is appended to + region: the picture region to serialize + width: the page width in pixels + height: the page height in pixels + figure_count: the 1-based index of the figure on the page + + Returns: + the index of the next figure + """ + if len(region.geometry) != 2: + raise TypeError("XML export is only available for straight bounding boxes for now.") + SubElement( + page_div, + "div", + attrib={ + "class": "ocr_photo", + "id": f"figure_{figure_count}", + "title": _hocr_bbox(region.geometry, width, height), # type: ignore[arg-type] + }, + ) + return figure_count + 1 + def export_page( self, page: "Page", @@ -752,6 +922,9 @@ def export_page( ) -> tuple[bytes, ET.ElementTree]: """Export a page as hOCR XML, with its content sorted in reading order. + The figures detected by the layout model are serialized as `ocr_photo` areas, positioned but + without their pixels. + Args: page: the page to export file_title: the title of the XML file @@ -763,12 +936,13 @@ def export_page( Returns: a tuple of the XML byte string, and its ElementTree """ - from doctr.io.elements import Table + from doctr.io.elements import LayoutElement, Table block_count: int = 1 line_count: int = 1 word_count: int = 1 table_count: int = 1 + figure_count: int = 1 height, width = page.dimensions page_hocr, body = self._new_document(file_title, _resolve_hocr_language(page.language)) page_div = SubElement( @@ -784,12 +958,15 @@ def export_page( if reading_order: items, _, direction = page_reading_order(page, direction) else: - items = [*page.blocks, *page.tables] + items = [*page.blocks, *page.tables, *picture_regions(page)] # iterate over the blocks / lines / words and create the XML elements line by line with the attributes for item in items: if isinstance(item, Table): table_count = self._add_table(page_div, item, width, height, table_count, dpi=dpi) continue + if isinstance(item, LayoutElement): + figure_count = self._add_figure(page_div, item, width, height, figure_count) + continue block = item if len(block.geometry) != 2: raise TypeError("XML export is only available for straight bounding boxes for now.") @@ -980,11 +1157,12 @@ def export(self, reading_order: bool = True) -> dict[str, Any]: Returns: a JSON-serializable dict """ - from doctr.io.elements import Element, Table + from doctr.io.elements import Block, Element export_dict = Element.export(cast("Element", self)) if reading_order: - blocks = [item for item in page_reading_order(cast("Page", self))[0] if not isinstance(item, Table)] + # Tables and figures have their own keys in the export: only the blocks are re-serialized here + blocks = [item for item in page_reading_order(cast("Page", self))[0] if isinstance(item, Block)] if blocks: # an empty linearization (no line on the page) leaves the stored blocks untouched export_dict["blocks"] = [block.export() for block in blocks] return export_dict @@ -1012,33 +1190,52 @@ def export_as_xml( cast("Page", self), file_title=file_title, direction=direction, reading_order=reading_order, dpi=dpi ) - def items_in_reading_order(self, direction: str = "auto") -> list["Block | Table"]: - """Return the content of the page (blocks & tables) sorted in reading order. + def items_in_reading_order(self, direction: str = "auto") -> list["Block | Table | LayoutElement"]: + """Return the content of the page (blocks, tables & figures) sorted in reading order. Args: direction: reading direction, one of 'auto', 'ltr', 'rtl', 'ttb-rtl' or 'ttb-ltr' Returns: - list of blocks & tables in reading order + list of blocks, tables & picture regions in reading order """ return page_reading_order(cast("Page", self), direction)[0] - def export_as_markdown(self, direction: str = "auto", escape: bool = True, include_furniture: bool = True) -> str: + def export_as_markdown( + self, + direction: str = "auto", + escape: bool = True, + include_furniture: bool = True, + images: "str | FigureEncoder | None" = "placeholder", + ) -> str: """Export the page as Markdown, with its content sorted in reading order. Args: direction: reading direction, one of 'auto', 'ltr', 'rtl', 'ttb-rtl' or 'ttb-ltr' escape: whether the characters carrying a structural meaning in Markdown should be escaped include_furniture: whether page headers, page footers and footnotes should be included + images: how the figures detected by the layout model are materialized, either an image mode + ('none', 'placeholder', 'embedded' or 'referenced') or a configured + :class:`~doctr.io.FigureEncoder` Returns: a Markdown string """ return MarkdownExporter().export_page( - cast("Page", self), direction=direction, escape=escape, include_furniture=include_furniture + cast("Page", self), + direction=direction, + escape=escape, + include_furniture=include_furniture, + images=images, ) - def export_as_asciidoc(self, direction: str = "auto", escape: bool = True, include_furniture: bool = True) -> str: + def export_as_asciidoc( + self, + direction: str = "auto", + escape: bool = True, + include_furniture: bool = True, + images: "str | FigureEncoder | None" = "placeholder", + ) -> str: """Export the page as AsciiDoc, with its content sorted in reading order. Args: @@ -1046,25 +1243,42 @@ def export_as_asciidoc(self, direction: str = "auto", escape: bool = True, inclu escape: whether the characters and line markers carrying a structural meaning in AsciiDoc should be escaped include_furniture: whether page headers, page footers and footnotes should be included + images: how the figures detected by the layout model are materialized, either an image mode + ('none', 'placeholder', 'embedded' or 'referenced') or a configured + :class:`~doctr.io.FigureEncoder` Returns: an AsciiDoc string """ return AsciiDocExporter().export_page( - cast("Page", self), direction=direction, escape=escape, include_furniture=include_furniture + cast("Page", self), + direction=direction, + escape=escape, + include_furniture=include_furniture, + images=images, ) - def export_as_html(self, direction: str = "auto", include_furniture: bool = True) -> str: + def export_as_html( + self, + direction: str = "auto", + include_furniture: bool = True, + images: "str | FigureEncoder | None" = "placeholder", + ) -> str: """Export the page as semantic HTML, with its content sorted in reading order. Args: direction: reading direction, one of 'auto', 'ltr', 'rtl', 'ttb-rtl' or 'ttb-ltr' include_furniture: whether page headers, page footers and footnotes should be included + images: how the figures detected by the layout model are materialized, either an image mode + ('none', 'placeholder', 'embedded' or 'referenced') or a configured + :class:`~doctr.io.FigureEncoder` Returns: an HTML string """ - return HTMLExporter().export_page(cast("Page", self), direction=direction, include_furniture=include_furniture) + return HTMLExporter().export_page( + cast("Page", self), direction=direction, include_furniture=include_furniture, images=images + ) def export_as(self, format: str, **kwargs: Any) -> Any: """Export the page in the requested format. @@ -1263,7 +1477,8 @@ def export_as_markdown(self, page_break: str = "\n\n---\n\n", **kwargs: Any) -> Args: page_break: the string inserted between two pages (a thematic break by default) - **kwargs: additional keyword arguments passed to the `Page.export_as_markdown` method + **kwargs: additional keyword arguments passed to the `Page.export_as_markdown` method, among which + `images` to control how the detected figures are materialized Returns: a Markdown string @@ -1275,7 +1490,8 @@ def export_as_asciidoc(self, page_break: str = "\n\n<<<\n\n", **kwargs: Any) -> Args: page_break: the string inserted between two pages (an AsciiDoc page break by default) - **kwargs: additional keyword arguments passed to the `Page.export_as_asciidoc` method + **kwargs: additional keyword arguments passed to the `Page.export_as_asciidoc` method, among which + `images` to control how the detected figures are materialized Returns: an AsciiDoc string @@ -1287,7 +1503,8 @@ def export_as_html(self, page_break: str = "
", **kwargs: Any) -> str: Args: page_break: the HTML snippet inserted between two pages - **kwargs: additional keyword arguments passed to the page export + **kwargs: additional keyword arguments passed to the page export, among which `images` to + control how the detected figures are materialized Returns: an HTML string diff --git a/doctr/io/figures.py b/doctr/io/figures.py new file mode 100644 index 0000000000..1d1e6b1ecc --- /dev/null +++ b/doctr/io/figures.py @@ -0,0 +1,268 @@ +# Copyright (C) 2021-2026, Mindee. + +# This program is licensed under the Apache License 2.0. +# See LICENSE or go to for full license details. + +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import cv2 +import numpy as np + +from doctr.utils.geometry import extract_crops, extract_rcrops + +if TYPE_CHECKING: # pragma: no cover + from doctr.io.elements import LayoutElement, Page + +__all__ = [ + "IMAGE_FORMATS", + "IMAGE_MODES", + "FigureEncoder", + "crop_layout_region", + "encode_crop", + "is_picture_label", + "is_picture_region", + "picture_regions", +] + +# How the figures detected by the layout model are materialized in the Markdown / AsciiDoc / HTML exports +IMAGE_MODES = ("none", "placeholder", "embedded", "referenced") +IMAGE_FORMATS = ("png", "jpg", "jpeg", "webp") + + +def is_picture_label(label: str | None) -> bool: + """Whether a layout label denotes a figure (as opposed to a table or a text region). + + Args: + label: the layout label to inspect (e.g. a DocLayNet class such as 'Picture' or 'Table') + + Returns: + True for the float labels that are not tables ('Picture', 'Figure', 'Chart', ...) + """ + from doctr.models.reading_order import layout_label_role, normalize_layout_label + + return layout_label_role(label) == "float" and normalize_layout_label(label) != "table" + + +def is_picture_region(region: "LayoutElement") -> bool: + """Whether a layout region is a figure (as opposed to a table or a text region). + + Args: + region: the layout region to inspect + + Returns: + True for the float regions that are not tables ('Picture', 'Figure', 'Chart', ...) + """ + return is_picture_label(getattr(region, "type", None)) + + +def picture_regions(page: "Page") -> list["LayoutElement"]: + """The figure regions detected on a page, in the order the layout model returned them. + + Args: + page: the page to inspect + + Returns: + the list of picture regions (empty when the page carries no layout) + """ + return [region for region in (getattr(page, "layout", None) or []) if is_picture_region(region)] + + +def _pad_geometry(points: np.ndarray, padding: float) -> np.ndarray: + """Grow a geometry around its center by a relative margin, and clip it back to the page.""" + if padding == 0: + return points + center = points.mean(axis=0, keepdims=True) + return np.clip(center + (points - center) * (1 + 2 * padding), 0, 1) + + +def crop_layout_region( + page_img: np.ndarray | None, + geometry: Any, + padding: float = 0.0, +) -> np.ndarray | None: + """Crop the pixels of a layout region out of its page. + + Straight regions are sliced out of the page, rotated ones are de-rotated with a warp (the layout + polygons are reading-oriented, exactly like the detection ones). + + Args: + page_img: the page image, as stored on `Page.page`. An empty array (a page restored from a + JSON export) or None yields None. + geometry: the region geometry, either a straight ((xmin, ymin), (xmax, ymax)) box or a (4, 2) + polygon, with coordinates relative to the page size + padding: relative margin added around the region on each side (0.05 grows it by 5%) + + Returns: + the cropped image, or None when the page carries no pixels or the region is degenerate (empty or + smaller than 2x2 pixels) + """ + if page_img is None or page_img.size == 0: + return None + points = np.asarray(geometry, dtype=np.float32).reshape(-1, 2) + if points.shape[0] not in (2, 4): + return None + points = _pad_geometry(points, padding) + if points.shape[0] == 2: # straight box + box = np.array( + [[points[:, 0].min(), points[:, 1].min(), points[:, 0].max(), points[:, 1].max()]], dtype=np.float32 + ) + crops = extract_crops(page_img, box) + else: # rotated polygon + crops = extract_rcrops(page_img, points[None, ...].astype(np.float32)) + if len(crops) == 0 or crops[0].size == 0 or min(crops[0].shape[:2]) < 2: + return None # a collapsed region would otherwise yield a 1-pixel image + return crops[0] + + +def encode_crop(crop: np.ndarray, image_format: str = "png", quality: int = 95) -> bytes: + """Encode a crop into an image file format. + + Args: + crop: the RGB crop to encode (docTR pages are RGB, OpenCV expects BGR) + image_format: one of 'png', 'jpg'/'jpeg' or 'webp' + quality: the encoding quality of the lossy formats ('jpg'/'jpeg' and 'webp') + + Returns: + the encoded image bytes + """ + if image_format not in IMAGE_FORMATS: + raise ValueError(f"unsupported image format '{image_format}', should be one of {list(IMAGE_FORMATS)}") + extension = ".jpg" if image_format in ("jpg", "jpeg") else f".{image_format}" + params: list[int] = [] + if extension == ".jpg": + params = [int(cv2.IMWRITE_JPEG_QUALITY), int(quality)] + elif extension == ".webp": + params = [int(cv2.IMWRITE_WEBP_QUALITY), int(quality)] + array = cv2.cvtColor(crop, cv2.COLOR_RGB2BGR) if crop.ndim == 3 and crop.shape[2] == 3 else crop + success, buffer = cv2.imencode(extension, array, params) + if not success: # pragma: no cover + raise RuntimeError(f"failed to encode a figure crop as '{image_format}'") + return buffer.tobytes() + + +class FigureEncoder: + """Turns the figures detected by the layout model into an image source for the text exporters. + + Four modes are available: + + * ``none``: figures are dropped entirely, as they were before this was implemented + * ``placeholder`` (default): a format-specific comment marks where a figure was detected, without + touching the pixels + * ``embedded``: the crop is inlined as a base64 data URI, so the export stays a single file + * ``referenced``: the crop is written to ``image_dir`` and referenced by a relative path + + >>> from doctr.io import FigureEncoder + >>> markdown = page.export_as_markdown(images=FigureEncoder("referenced", image_dir="assets")) + + Args: + mode: one of 'none', 'placeholder', 'embedded' or 'referenced' + image_dir: the directory the crops are written to (required in 'referenced' mode) + path_prefix: prepended to the file names in 'referenced' mode, to match the location the export + is rendered from (e.g. 'assets/' when the Markdown file sits next to the `assets` directory) + image_format: one of 'png', 'jpg'/'jpeg' or 'webp' + quality: the encoding quality of the lossy formats + padding: relative margin added around each region, useful to catch the axis labels of a plot + """ + + def __init__( + self, + mode: str = "placeholder", + image_dir: str | Path | None = None, + path_prefix: str = "", + image_format: str = "png", + quality: int = 95, + padding: float = 0.0, + ) -> None: + if mode not in IMAGE_MODES: + raise ValueError(f"unsupported image mode '{mode}', should be one of {list(IMAGE_MODES)}") + if image_format not in IMAGE_FORMATS: + raise ValueError(f"unsupported image format '{image_format}', should be one of {list(IMAGE_FORMATS)}") + if mode == "referenced" and image_dir is None: + raise ValueError("an 'image_dir' is required to export the figures in 'referenced' mode") + self.mode = mode + self.image_dir = Path(image_dir) if image_dir is not None else None + self.path_prefix = path_prefix + self.image_format = image_format + self.quality = quality + self.padding = padding + # The files written so far, in emission order (empty unless the mode is 'referenced') + self.written: list[Path] = [] + + @classmethod + def resolve(cls, images: "str | FigureEncoder | None") -> "FigureEncoder": + """Build an encoder from the `images` argument of an export method. + + Args: + images: an image mode, an already configured encoder, or None (equivalent to 'none') + + Returns: + the encoder to use + """ + if isinstance(images, FigureEncoder): + return images + return cls(mode="none" if images is None else images) + + @property + def enabled(self) -> bool: + """Whether the figures should appear in the export at all""" + return self.mode != "none" + + @property + def materializes(self) -> bool: + """Whether the encoder carries the pixels of the figures (as opposed to marking their position)""" + return self.mode in ("embedded", "referenced") + + def materializes_on(self, page: "Page") -> bool: + """Whether the figures of this page will actually carry their pixels. + + The exporters use this to decide whether the text detected inside a figure is redundant: it is + already visible in the emitted image, but it would be lost with a mere placeholder. A page + restored from a JSON export carries no pixels, so its inner text must be kept. + + Args: + page: the page about to be exported + + Returns: + True when the mode carries the pixels and the page still has an image + """ + page_img = getattr(page, "page", None) + return self.materializes and page_img is not None and page_img.size > 0 + + def source(self, page: "Page", region: "LayoutElement", index: int) -> str | None: + """Resolve the image source of a figure. + + Args: + page: the page the figure belongs to + region: the picture region to encode + index: the 1-based index of the figure on the page, used to name the file + + Returns: + a data URI, a relative path, or None when the pixels are unavailable (which happens in the + 'none' and 'placeholder' modes, and on pages restored from a JSON export) + """ + if self.mode in ("none", "placeholder"): + return None + crop = crop_layout_region(getattr(page, "page", None), region.geometry, self.padding) + if crop is None: + return None + payload = encode_crop(crop, self.image_format, self.quality) + mime = "jpeg" if self.image_format in ("jpg", "jpeg") else self.image_format + if self.mode == "embedded": + from base64 import b64encode + + return f"data:image/{mime};base64,{b64encode(payload).decode('ascii')}" + extension = "jpg" if mime == "jpeg" else mime + name = f"page{getattr(page, 'page_idx', 0) + 1}_figure{index}.{extension}" + assert self.image_dir is not None # guaranteed by __init__ in 'referenced' mode + self.image_dir.mkdir(parents=True, exist_ok=True) + path = self.image_dir / name + path.write_bytes(payload) + self.written.append(path) + return f"{self.path_prefix}{name}" + + def __repr__(self) -> str: + return ( + f"{self.__class__.__name__}(mode='{self.mode}', image_dir={self.image_dir}, " + f"image_format='{self.image_format}')" + ) diff --git a/tests/common/test_io_exporters.py b/tests/common/test_io_exporters.py index 0a33bba07d..106e5740b0 100644 --- a/tests/common/test_io_exporters.py +++ b/tests/common/test_io_exporters.py @@ -1,5 +1,6 @@ import json +import cv2 import numpy as np import pytest @@ -15,6 +16,7 @@ page_reading_order, to_json_safe, ) +from doctr.io.figures import FigureEncoder def _word_at(text, x0, y0, x1, y1): @@ -873,3 +875,138 @@ def test_every_format_is_reachable_from_a_document(): assert kie_doc.export_as_html() == f"{kie_html}
{kie_html}" assert kie_page.export_as_html() == f"

{CLASS_NAME}

\n
    \n
  • value
    • \n
" assert kie_page.export_as_html(direction="rtl") == kie_page.export_as_html() + + +def _figure_page(): + """A title, a paragraph, a figure with a caption underneath, and a closing paragraph.""" + image = np.zeros((1000, 800, 3), dtype=np.uint8) + image[300:600, 100:700] = (255, 0, 0) + lines = [ + _line_at("Annual Report", 0.2, 0.04, 0.8, 0.08), + _line_at("Revenue grew steadily", 0.1, 0.12, 0.9, 0.16), + _line_at("axis label", 0.15, 0.45, 0.35, 0.48), # text detected inside the figure + _line_at("Figure 1 quarterly revenue", 0.2, 0.63, 0.8, 0.66), + _line_at("The trend continued", 0.1, 0.72, 0.9, 0.76), + ] + layout = [ + elements.LayoutElement("Title", 0.99, ((0.15, 0.03), (0.85, 0.09))), + elements.LayoutElement("Text", 0.98, ((0.08, 0.11), (0.92, 0.17))), + elements.LayoutElement("Picture", 0.97, ((0.12, 0.30), (0.88, 0.60))), + elements.LayoutElement("Caption", 0.96, ((0.18, 0.62), (0.82, 0.67))), + elements.LayoutElement("Text", 0.98, ((0.08, 0.71), (0.92, 0.77))), + ] + return elements.Page(image, [elements.Block(lines=lines)], 0, (1000, 800), layout=layout) + + +def test_page_reading_order_with_figures(): + page = _figure_page() + items, labels, _ = page_reading_order(page) + kinds = [type(item).__name__ for item in items] + assert kinds.count("LayoutElement") == 1 + figure_idx = kinds.index("LayoutElement") + assert labels[figure_idx] == "Picture" + # The figure is read after the paragraph above it and before its caption + rendered = [item.render(line_break=" ") if isinstance(item, elements.Block) else None for item in items] + assert rendered.index("Revenue grew steadily") < figure_idx + assert figure_idx < rendered.index("Figure 1 quarterly revenue") + # A page without layout keeps yielding blocks only + plain = elements.Page(np.zeros((10, 10, 3), dtype=np.uint8), page.blocks, 0, (1000, 800)) + assert all(isinstance(item, elements.Block) for item in plain.items_in_reading_order()) + + +def test_page_export_markdown_images(): + page = _figure_page() + # Placeholder is the default: the position of the figure is marked, the text around it is untouched + markdown = page.export_as_markdown() + assert "" in markdown + assert markdown.index("Revenue grew steadily") < markdown.index("") + assert markdown.index("") < markdown.index("Figure 1 quarterly revenue") + assert "axis label" in markdown # nothing carries those pixels, so the text is kept + + # 'none' restores the pre-figure output + assert "" not in page.export_as_markdown(images="none") + assert "Figure 1 quarterly revenue" in page.export_as_markdown(images="none") + + # 'embedded' inlines the crop, absorbs the caption as the alternative text and drops the inner text + embedded = page.export_as_markdown(images="embedded") + assert "![Figure 1 quarterly revenue](data:image/png;base64," in embedded + assert "axis label" not in embedded + assert embedded.count("Figure 1 quarterly revenue") == 1 # the caption is not repeated as a paragraph + jpeg = FigureEncoder("embedded", image_format="jpeg") + assert "data:image/jpeg;base64," in page.export_as_markdown(images=jpeg) + + +def test_page_export_referenced_images(tmp_path): + page = _figure_page() + encoder = FigureEncoder("referenced", image_dir=tmp_path, path_prefix="assets/") + markdown = page.export_as_markdown(images=encoder) + assert "![Figure 1 quarterly revenue](assets/page1_figure1.png)" in markdown + assert encoder.written == [tmp_path / "page1_figure1.png"] + # The crop holds the figure pixels, not the whole page + + crop = cv2.imread(str(encoder.written[0])) + assert crop.shape[0] < page.dimensions[0] and crop.shape[1] < page.dimensions[1] + assert crop[..., 2].mean() > 200 # OpenCV reads BGR: the red rectangle lands on the last channel + + +def test_page_export_asciidoc_and_html_images(): + page = _figure_page() + assert "// image" in page.export_as_asciidoc() + assert "" in page.export_as_html() + + asciidoc = page.export_as_asciidoc(images="embedded") + assert ".Figure 1 quarterly revenue\nimage::data:image/png;base64," in asciidoc + + html = page.export_as_html(images="embedded") + assert '
Figure 1 quarterly revenue
" in html + assert 'alt="Figure 1 quarterly revenue"' in html + + # A figure without a caption gets an empty alternative text rather than an invented one + page.layout = [region for region in page.layout if region.type != "Caption"] + page._reading_order_cache = None + assert 'alt=""' in page.export_as_html(images="embedded") + + +def test_page_export_images_without_page_image(): + # A page restored from a JSON export carries no pixels: the figures degrade to a placeholder, and the + # text detected inside them is kept, since nothing else would carry it + restored = elements.Page.from_dict(_figure_page().export()) + markdown = restored.export_as_markdown(images="embedded") + assert "" in markdown + assert "axis label" in markdown + assert "Figure 1 quarterly revenue" in markdown + + +def test_text_and_json_exports_ignore_figures(): + page = _figure_page() + # Plain text has no image syntax: the figures are dropped + assert "image" not in page.render() + assert page.render().startswith("Annual Report") + assert TextExporter().export_page(page, images="embedded") == page.render() + # The figures have their own key in the JSON export and must not leak into the blocks + exported = page.export() + assert len(exported["blocks"]) == 5 + assert len(exported["layout"]) == 5 + json.dumps(exported) + + +def test_page_export_as_xml_figures(): + page = _figure_page() + xml_bytes, _ = page.export_as_xml() + xml = xml_bytes.decode() + assert 'class="ocr_photo" id="figure_1"' in xml + assert 'title="bbox 96 300 704 600"' in xml + assert "ocr_photo" in xml.split('name="ocr-capabilities" content="')[1].split('"')[0] + # The raw order also carries the figures + assert 'class="ocr_photo"' in page.export_as_xml(reading_order=False)[0].decode() + + +def test_document_export_passes_images_through(tmp_path): + page = _figure_page() + doc = elements.Document([page, page]) + assert doc.export_as_markdown().count("") == 2 + assert doc.export_as_markdown(images="none").count("") == 0 + encoder = FigureEncoder("referenced", image_dir=tmp_path) + doc.export_as_html(images=encoder) + assert len(encoder.written) == 2 diff --git a/tests/common/test_io_figures.py b/tests/common/test_io_figures.py new file mode 100644 index 0000000000..64401a4f9a --- /dev/null +++ b/tests/common/test_io_figures.py @@ -0,0 +1,125 @@ +import cv2 +import numpy as np +import pytest + +from doctr.io import elements +from doctr.io.figures import ( + FigureEncoder, + crop_layout_region, + encode_crop, + is_picture_label, + is_picture_region, + picture_regions, +) + + +def _page_image(): + """A dark page with a bright rectangle where the figure sits (relative box (0.1, 0.2) - (0.5, 0.6))""" + image = np.zeros((100, 200, 3), dtype=np.uint8) + image[20:60, 20:100] = (255, 0, 0) + return image + + +@pytest.mark.parametrize( + "label, expected", + [ + ("Picture", True), + ("picture", True), + ("Figure", True), + ("Chart", True), + ("Table", False), + ("Text", False), + ("Caption", False), + (None, False), + ], +) +def test_is_picture_label(label, expected): + assert is_picture_label(label) is expected + assert is_picture_region(elements.LayoutElement(label or "Text", 0.9, ((0, 0), (1, 1)))) is expected + + +def test_picture_regions(): + layout = [ + elements.LayoutElement("Text", 0.9, ((0.0, 0.0), (1.0, 0.1))), + elements.LayoutElement("Picture", 0.9, ((0.1, 0.2), (0.5, 0.6))), + elements.LayoutElement("Table", 0.9, ((0.1, 0.7), (0.9, 0.9))), + ] + page = elements.Page(_page_image(), [], 0, (100, 200), layout=layout) + assert [region.type for region in picture_regions(page)] == ["Picture"] + # A page without layout has no figure + assert picture_regions(elements.Page(_page_image(), [], 0, (100, 200))) == [] + + +def test_crop_layout_region(): + image = _page_image() + crop = crop_layout_region(image, ((0.1, 0.2), (0.5, 0.6))) + assert crop.shape[:2] == (41, 81) # the crop bounds are inclusive + assert (crop[:, :, 0] == 255).mean() > 0.9 + # Padding grows the region, so the dark background creeps in + padded = crop_layout_region(image, ((0.1, 0.2), (0.5, 0.6)), padding=0.25) + assert padded.shape[0] > crop.shape[0] and padded.shape[1] > crop.shape[1] + assert (padded[:, :, 0] == 255).mean() < (crop[:, :, 0] == 255).mean() + # Rotated polygons are de-rotated by a warp + polygon = np.array([[0.1, 0.2], [0.5, 0.2], [0.5, 0.6], [0.1, 0.6]], dtype=np.float32) + assert crop_layout_region(image, polygon).shape[:2] == (40, 80) + # A page without pixels, or a degenerate region, yields nothing + assert crop_layout_region(None, ((0.1, 0.2), (0.5, 0.6))) is None + assert crop_layout_region(np.zeros((0, 0, 3), dtype=np.uint8), ((0.1, 0.2), (0.5, 0.6))) is None + assert crop_layout_region(image, ((0.5, 0.5), (0.5, 0.5))) is None + + +@pytest.mark.parametrize("image_format", ["png", "jpg", "jpeg", "webp"]) +def test_encode_crop(image_format): + crop = _page_image()[20:60, 20:100] + payload = encode_crop(crop, image_format=image_format, quality=95) + assert isinstance(payload, bytes) and len(payload) > 0 + decoded = cv2.imdecode(np.frombuffer(payload, dtype=np.uint8), cv2.IMREAD_COLOR) + assert decoded.shape == crop.shape + # docTR pages are RGB: the red rectangle must survive the RGB -> BGR -> file -> BGR round trip + assert decoded[..., 2].mean() > decoded[..., 0].mean() + + with pytest.raises(ValueError): + encode_crop(crop, image_format="gif") + + +def test_figure_encoder_validation(tmp_path): + with pytest.raises(ValueError): + FigureEncoder(mode="inline") + with pytest.raises(ValueError): + FigureEncoder(mode="embedded", image_format="gif") + with pytest.raises(ValueError): # 'referenced' needs somewhere to write + FigureEncoder(mode="referenced") + FigureEncoder(mode="referenced", image_dir=tmp_path) + + # `resolve` accepts a mode, an encoder, or None + assert FigureEncoder.resolve("embedded").mode == "embedded" + assert FigureEncoder.resolve(None).mode == "none" + encoder = FigureEncoder("placeholder") + assert FigureEncoder.resolve(encoder) is encoder + assert "placeholder" in repr(encoder) + + +def test_figure_encoder_modes(tmp_path): + region = elements.LayoutElement("Picture", 0.9, ((0.1, 0.2), (0.5, 0.6))) + page = elements.Page(_page_image(), [], 0, (100, 200), layout=[region]) + + assert FigureEncoder("none").source(page, region, 1) is None + assert not FigureEncoder("none").enabled + assert FigureEncoder("placeholder").source(page, region, 1) is None + assert FigureEncoder("placeholder").enabled and not FigureEncoder("placeholder").materializes + + embedded = FigureEncoder("embedded").source(page, region, 1) + assert embedded.startswith("data:image/png;base64,") + assert FigureEncoder("embedded", image_format="jpg").source(page, region, 1).startswith("data:image/jpeg;base64,") + + encoder = FigureEncoder("referenced", image_dir=tmp_path / "assets", path_prefix="assets/") + assert encoder.source(page, region, 3) == "assets/page1_figure3.png" + assert encoder.written == [tmp_path / "assets" / "page1_figure3.png"] + assert encoder.written[0].read_bytes()[:4] == b"\x89PNG" + + # A page restored from a JSON export carries no pixels: the figures degrade to a placeholder + restored = elements.Page.from_dict(page.export()) + assert FigureEncoder("embedded").source(restored, region, 1) is None + assert FigureEncoder("embedded").materializes + assert not FigureEncoder("embedded").materializes_on(restored) + assert FigureEncoder("embedded").materializes_on(page)