gh-161114: Speed up pprint on deeply nested containers

Lib/pprint.py

@@ -198,9 +198,9 @@ def _format(self, object, stream, indent, allowance, context, level):
             self._recursive = True
             self._readable = False
             return
-        rep = self._repr(object, context, level)
         max_width = self._width - indent - prefix_len - allowance
-        if len(rep) > max_width:
+        rep = self._repr_if_fits(object, context, level, max_width)
+        if rep is None or len(rep) > max_width:
             p = self._dispatch.get(type(object).__repr__, None)
             # Lazy import to improve module import time
             from dataclasses import is_dataclass
@@ -220,6 +220,8 @@ def _format(self, object, stream, indent, allowance, context, level):
                 self._pprint_dataclass(object, stream, indent, allowance, context, level + 1)
                 del context[objid]
                 return
+            if rep is None:
+                rep = self._repr(object, context, level)
         stream.write(rep)
 
     def _format_block_start(self, start_str, indent):
@@ -626,8 +628,8 @@ def _format_items(self, items, stream, indent, allowance, context, level):
                 max_width -= allowance
                 width -= allowance
             if self._compact:
-                rep = self._repr(ent, context, level)
-                w = len(rep) + 2
+                rep = self._repr_if_fits(ent, context, level, max_width - 2)
+                w = max_width + 1 if rep is None else len(rep) + 2
                 if width < w:
                     width = max_width
                     if delim:
@@ -655,6 +657,33 @@ def _repr(self, object, context, level):
             self._recursive = True
         return repr
 
+    # Set by _repr_if_fits() for the duration of its _repr() call and
+    # consumed by _safe_repr(), which raises _TooWide as soon as the
+    # representation of a container grows beyond it.  Finding out that an
+    # object does not fit on one line thus costs no more than the width of
+    # the line, instead of the size of the object, for each level of nesting.
+    # It is an attribute, not an argument, because subclasses override
+    # format() with its existing signature.
+    _max_width = None
+
+    def _repr_if_fits(self, object, context, level, max_width):
+        # Return the repr of object, or None if building it was cut short
+        # because it cannot fit in max_width.  A completed repr is returned
+        # even when it is too wide, so that _format() does not have to
+        # build it again when it writes it.
+        if type(self).format is not PrettyPrinter.format:
+            # An override may shorten what PrettyPrinter.format() returns,
+            # so only the complete result can tell whether it fits.
+            return self._repr(object, context, level)
+        saved = self._max_width
+        self._max_width = max_width
+        try:
+            return self._repr(object, context, level)
+        except _TooWide:
+            return None
+        finally:
+            self._max_width = saved
+
     def format(self, object, context, maxlevels, level):
         """Format object for a specific context, returning a string
         and flags indicating whether the representation is 'readable'
@@ -852,6 +881,12 @@ def _safe_repr(self, object, context, maxlevels, level):
             components = []
             append = components.append
             level += 1
+            max_width = self._max_width
+            width = 0
+            if max_width is not None and 4 * len(object) > max_width:
+                # Every item costs at least "k: v, ": do not bother
+                # formatting (and sorting) them.
+                raise _TooWide
             if self._sort_dicts:
                 items = sorted(object.items(), key=_safe_tuple)
             else:
@@ -865,6 +900,10 @@ def _safe_repr(self, object, context, maxlevels, level):
                 readable = readable and kreadable and vreadable
                 if krecur or vrecur:
                     recursive = True
+                if max_width is not None:
+                    width += len(krepr) + len(vrepr) + 4
+                    if width > max_width:
+                        raise _TooWide
             del context[objid]
             rep = "{%s}" % ", ".join(components)
             if is_frozendict:
@@ -894,6 +933,8 @@ def _safe_repr(self, object, context, maxlevels, level):
             components = []
             append = components.append
             level += 1
+            max_width = self._max_width
+            width = 0
             for o in object:
                 orepr, oreadable, orecur = self.format(
                     o, context, maxlevels, level)
@@ -902,6 +943,10 @@ def _safe_repr(self, object, context, maxlevels, level):
                     readable = False
                 if orecur:
                     recursive = True
+                if max_width is not None:
+                    width += len(orepr) + 2
+                    if width > max_width:
+                        raise _TooWide
             del context[objid]
             return format % ", ".join(components), readable, recursive
 
@@ -923,6 +968,9 @@ def _safe_repr(self, object, context, maxlevels, level):
                 #  We have a view that somehow has lost its type's _mapping, raise
                 #  an error by calling repr() instead of failing cryptically later
                 return repr(object), True, False
+            max_width = self._max_width
+            if max_width is not None and 2 * len(object) > max_width:
+                raise _TooWide
             if self._sort_dicts:
                 object = sorted(object, key=key)
             context[objid] = 1
@@ -931,13 +979,18 @@ def _safe_repr(self, object, context, maxlevels, level):
             components = []
             append = components.append
             level += 1
+            width = 0
             for val in object:
                 vrepr, vreadable, vrecur = self.format(
                     val, context, maxlevels, level)
                 append(vrepr)
                 readable = readable and vreadable
                 if vrecur:
                     recursive = True
+                if max_width is not None:
+                    width += len(vrepr) + 2
+                    if width > max_width:
+                        raise _TooWide
             del context[objid]
             return typ.__name__ + '([%s])' % ", ".join(components), readable, recursive
 
@@ -954,6 +1007,13 @@ def _recursion(object):
             % (type(object).__name__, id(object)))
 
 
+class _TooWide(BaseException):
+    """Raised by _safe_repr() when the representation it is building is
+    already wider than PrettyPrinter._max_width.  It is not an Exception,
+    so that "except Exception" handlers it passes through do not catch it.
+    """
+
+
 def _wrap_bytes_repr(object, width, allowance):
     current = b''
     last = len(object) // 4 * 4

Lib/test/test_pprint.py

@@ -1039,6 +1039,61 @@ def test_depth(self):
         self.assertEqual(pprint.pformat(nested_dict, depth=1), lv1_dict)
         self.assertEqual(pprint.pformat(nested_list, depth=1), lv1_list)
 
+    def test_deeply_nested_items_not_formatted_per_level(self):
+        # The items of the innermost container must not be formatted again
+        # for each of the containers around it, which would make the time
+        # proportional to the size of the data times its nesting depth.
+        class Item(int):
+            calls = 0
+            def __repr__(self):
+                Item.calls += 1
+                return int.__repr__(self)
+
+        depth = 50
+        size = 1000
+        width = 80
+        for wrap in (
+            lambda o: [o],
+            lambda o: (o,),
+            lambda o: {'k': o},
+            lambda o: frozendict({'k': o}),
+            lambda o: {'k': o}.values(),
+        ):
+            for items in ([Item(i) for i in range(size)],
+                          {Item(i): Item(i) for i in range(size)}):
+                for kwargs in {}, {'compact': True}, {'expand': True}:
+                    nested = items
+                    for i in range(depth):
+                        nested = wrap(nested)
+                    with self.subTest(type=type(nested).__name__,
+                                      items=type(items).__name__, **kwargs):
+                        Item.calls = 0
+                        pprint.pformat(nested, width=width, **kwargs)
+                        # Each level may format a line's worth of items to
+                        # find out that the container does not fit on one
+                        # line, and every item is formatted a few times once
+                        # the innermost container is laid out.
+                        self.assertLess(Item.calls,
+                                        depth * width + 4 * len(items))
+
+    def test_format_override_shortening_repr(self):
+        # A format() override may shorten what PrettyPrinter.format()
+        # returns, so whether that fits on one line cannot be decided before
+        # it is complete.
+        class ShortPrettyPrinter(pprint.PrettyPrinter):
+            def format(self, object, context, maxlevels, level):
+                rep, readable, recursive = super().format(
+                    object, context, maxlevels, level)
+                if len(rep) > 20:
+                    rep = rep[:17] + '...'
+                return rep, False, recursive
+
+        printer = ShortPrettyPrinter(width=30)
+        self.assertEqual(printer.pformat({'a': list(range(100))}),
+                         "{'a': [0, 1, 2, 3...")
+        self.assertEqual(printer.pformat([list(range(100))] * 3),
+                         "[[0, 1, 2, 3, 4, ...")
+
     def test_sort_unorderable_values(self):
         # Issue 3976:  sorted pprints fail for unorderable values.
         n = 20

Misc/NEWS.d/next/Library/2026-10-22-05-21-01.gh-issue-161114.9r-JQu.rst

@@ -0,0 +1,4 @@
+Improve performance of :func:`pprint.pformat`, :func:`pprint.pprint` and
+:class:`pprint.PrettyPrinter` for deeply nested containers that do not fit on
+one line. Formatting time no longer grows with the size of the data multiplied
+by its nesting depth.