Fix quadratic-time parsing of consecutive reference definitions

docs/changelog.md

@@ -16,6 +16,8 @@ See the [Contributing Guide](contributing.md) for details.
 
 ### Fixed
 
+* Fix quadratic-time parsing of many consecutive reference definitions, and of
+  many consecutive footnote definitions in the `footnotes` extension.
 * Support automatic links with URL schemes other than HTTP and FTP, such as
   `<ssh://example.org>` (#1244).
 * Fix quadratic-time id generation in the `toc` extension when many headers

markdown/blockprocessors.py

@@ -588,16 +588,29 @@ def run(self, parent: etree.Element, blocks: list[str]) -> bool:
         block = blocks.pop(0)
         m = self.RE.search(block)
         if m:
-            id = m.group(1).strip().lower()
-            link = m.group(2).lstrip('<').rstrip('>')
-            title = m.group(5) or m.group(6)
-            self.parser.md.references[id] = (link, title)
-            if block[m.end():].strip():
+            start = m.start()
+            end = m.end()
+            matches = [m]
+            # Collect the definitions on the following lines so they are all
+            # recorded in this pass rather than one per pass through the block
+            # processors. A definition which spans lines is left for the next
+            # pass, as another processor may claim part of it.
+            for m in self.RE.finditer(block, end):
+                if block[end:m.start()].strip('\n') or '\n' in m.group():
+                    break
+                matches.append(m)
+                end = m.end()
+            for m in matches:
+                id = m.group(1).strip().lower()
+                link = m.group(2).lstrip('<').rstrip('>')
+                title = m.group(5) or m.group(6)
+                self.parser.md.references[id] = (link, title)
+            if block[end:].strip():
                 # Add any content after match back to blocks as separate block
-                blocks.insert(0, block[m.end():].lstrip('\n'))
-            if block[:m.start()].strip():
+                blocks.insert(0, block[end:].lstrip('\n'))
+            if block[:start].strip():
                 # Add any content before match back to blocks as separate block
-                blocks.insert(0, block[:m.start()].rstrip('\n'))
+                blocks.insert(0, block[:start].rstrip('\n'))
             return True
         # No match. Restore block.
         blocks.insert(0, block)

markdown/extensions/footnotes.py

@@ -261,6 +261,16 @@ def run(self, parent: etree.Element, blocks: list[str]) -> bool:
 
         m = self.RE.search(block)
         if m:
+            preceding = block[:m.start()]
+            # Record the definitions which are directly followed by another
+            # definition in this pass rather than one per pass through the block
+            # processors.
+            for m2 in self.RE.finditer(block, m.end()):
+                if block[m.end():m2.start()].strip('\n'):
+                    break
+                self.footnotes.setFootnote(m.group(1), m.group(2).rstrip())
+                m = m2
+
             id = m.group(1)
             fn_blocks = [m.group(2)]
 
@@ -284,9 +294,9 @@ def run(self, parent: etree.Element, blocks: list[str]) -> bool:
             footnote = "\n\n".join(fn_blocks)
             self.footnotes.setFootnote(id, footnote.rstrip())
 
-            if block[:m.start()].strip():
+            if preceding.strip():
                 # Add any content before match back to blocks as separate block
-                blocks.insert(0, block[:m.start()].rstrip('\n'))
+                blocks.insert(0, preceding.rstrip('\n'))
             return True
         # No match. Restore block.
         blocks.insert(0, block)

tests/test_syntax/blocks/test_reference_links.py

@@ -60,3 +60,32 @@ def test_malformed_reference_with_long_run_of_spaces(self):
         self.assertMarkdownRenders(
             text, f'<p>{text}</p>', expected_attrs={'references': {}}
         )
+
+    def test_reference_definitions_before_setext_underline(self):
+        """
+        A setext underline after consecutive reference definitions is a paragraph,
+        and does not turn the last definition into a header.
+        """
+        self.assertMarkdownRenders(
+            '[a]: http://example.com/a\n[b]: http://example.com/b\n===',
+            '<p>===</p>',
+            expected_attrs={
+                'references': {'a': ('http://example.com/a', None), 'b': ('http://example.com/b', None)}
+            }
+        )
+
+    def test_many_reference_definitions(self):
+        """
+        Consecutive reference definitions should be consumed together rather
+        than returning the remainder of the block to be parsed once per
+        definition.
+        """
+        count = 10000
+        text = ''.join(f'[id{i}]: http://example.com/{i}\n' for i in range(count))
+        self.assertMarkdownRenders(
+            text,
+            '',
+            expected_attrs={
+                'references': {f'id{i}': (f'http://example.com/{i}', None) for i in range(count)}
+            }
+        )

tests/test_syntax/extensions/test_footnotes.py

@@ -265,6 +265,29 @@ def test_footnote_multple_complex_no_blank_line_between(self):
             '</div>'
         )
 
+    def test_many_footnote_definitions(self):
+        """
+        Consecutive footnote definitions should be consumed together rather
+        than returning the remainder of the block to be parsed once per
+        definition.
+        """
+        count = 5000
+        self.assertMarkdownRenders(
+            ''.join(f'[^{i}]: Footnote {i}\n' for i in range(count)),
+            '<div class="footnote">\n'
+            '<hr />\n'
+            '<ol>\n'
+            + ''.join(
+                f'<li id="fn:{i}">\n'
+                f'<p>Footnote {i}&#160;<a class="footnote-backref" href="#fnref:{i}"'
+                f' title="Jump back to footnote {i + 1} in the text">&#8617;</a></p>\n'
+                '</li>\n'
+                for i in range(count)
+            )
+            + '</ol>\n'
+            '</div>'
+        )
+
     def test_backlink_text(self):
         """Test back-link configuration."""