From 5cd3e1684d7a9a661d5ccf13b708975643a81510 Mon Sep 17 00:00:00 2001 From: Naaz nuria Date: Tue, 27 Oct 2026 14:17:25 +0530 Subject: [PATCH 1/3] Fix quadratic-time parsing of consecutive reference definitions ReferenceProcessor and FootnoteBlockProcessor record one definition per pass and return the rest of the block to be parsed again, so every earlier block processor rescans the shrinking block once per definition. Record a run of definitions in a single pass, leaving the last one for the next pass when other content follows it so the earlier processors still see the same blocks they do today. Add regression tests and a changelog entry. --- docs/changelog.md | 2 ++ markdown/blockprocessors.py | 34 ++++++++++++++----- markdown/extensions/footnotes.py | 28 ++++++++++++--- .../blocks/test_reference_links.py | 27 +++++++++++++++ .../test_syntax/extensions/test_footnotes.py | 23 +++++++++++++ 5 files changed, 102 insertions(+), 12 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 76350a27..e0ce5ae5 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -16,6 +16,8 @@ See the [Contributing Guide](contributing.md) for details. ### Fixed +* Fix quadratic-time parsing of many consecutive reference definitions, and of + many consecutive footnote definitions in the `footnotes` extension. * Support automatic links with URL schemes other than HTTP and FTP, such as `` (#1244). * Fix quadratic-time id generation in the `toc` extension when many headers diff --git a/markdown/blockprocessors.py b/markdown/blockprocessors.py index 728bf8f9..868f2227 100644 --- a/markdown/blockprocessors.py +++ b/markdown/blockprocessors.py @@ -588,16 +588,34 @@ def run(self, parent: etree.Element, blocks: list[str]) -> bool: block = blocks.pop(0) m = self.RE.search(block) if m: - id = m.group(1).strip().lower() - link = m.group(2).lstrip('<').rstrip('>') - title = m.group(5) or m.group(6) - self.parser.md.references[id] = (link, title) - if block[m.end():].strip(): + start = m.start() + end = m.end() + matches = [m] + # Collect the definitions on the following lines so they are all + # recorded in this pass rather than one per pass through the block + # processors. A definition which is indented or spans lines is left + # for the next pass, as is the last one when other content follows + # it, as another processor may claim them (an indented code block or + # a setext header, for example). + for m in self.RE.finditer(block, end): + if (block[end:m.start()].strip('\n') or '\n' in m.group() + or block.startswith(' ' * self.tab_length, m.start())): + break + matches.append(m) + end = m.end() + if len(matches) > 1 and block[end:].strip(): + end = matches.pop().start() + for m in matches: + id = m.group(1).strip().lower() + link = m.group(2).lstrip('<').rstrip('>') + title = m.group(5) or m.group(6) + self.parser.md.references[id] = (link, title) + if block[end:].strip(): # Add any content after match back to blocks as separate block - blocks.insert(0, block[m.end():].lstrip('\n')) - if block[:m.start()].strip(): + blocks.insert(0, block[end:].lstrip('\n')) + if block[:start].strip(): # Add any content before match back to blocks as separate block - blocks.insert(0, block[:m.start()].rstrip('\n')) + blocks.insert(0, block[:start].rstrip('\n')) return True # No match. Restore block. blocks.insert(0, block) diff --git a/markdown/extensions/footnotes.py b/markdown/extensions/footnotes.py index 46bb93c3..d0648d82 100644 --- a/markdown/extensions/footnotes.py +++ b/markdown/extensions/footnotes.py @@ -261,12 +261,32 @@ def run(self, parent: etree.Element, blocks: list[str]) -> bool: m = self.RE.search(block) if m: + first = m + preceding = block[:m.start()] + therest = block[m.end():].lstrip('\n') + m2 = self.RE.search(therest) + # Record the definitions which are directly followed by another + # definition in this pass rather than one per pass through the block + # processors. The last one is left for the next pass when other + # content follows it, as another processor may claim that block (an + # indented code block or a setext header, for example). + while m2 and m2.start() == 0 and not therest.startswith(' ' * self.tab_length): + self.footnotes.setFootnote(m.group(1), m.group(2).rstrip()) + block, m = therest, m2 + therest = block[m.end():].lstrip('\n') + m2 = self.RE.search(therest) + if m is not first and therest.strip(): + # Add back to blocks everything from beginning of match forward for next iteration. + blocks.insert(0, block) + if preceding.strip(): + # Add any content before match back to blocks as separate block + blocks.insert(0, preceding.rstrip('\n')) + return True + id = m.group(1) fn_blocks = [m.group(2)] # Handle rest of block - therest = block[m.end():].lstrip('\n') - m2 = self.RE.search(therest) if m2: # Another footnote exists in the rest of this block. # Any content before match is continuation of this footnote, which may be lazily indented. @@ -284,9 +304,9 @@ def run(self, parent: etree.Element, blocks: list[str]) -> bool: footnote = "\n\n".join(fn_blocks) self.footnotes.setFootnote(id, footnote.rstrip()) - if block[:m.start()].strip(): + if preceding.strip(): # Add any content before match back to blocks as separate block - blocks.insert(0, block[:m.start()].rstrip('\n')) + blocks.insert(0, preceding.rstrip('\n')) return True # No match. Restore block. blocks.insert(0, block) diff --git a/tests/test_syntax/blocks/test_reference_links.py b/tests/test_syntax/blocks/test_reference_links.py index 8bd67521..983df42e 100644 --- a/tests/test_syntax/blocks/test_reference_links.py +++ b/tests/test_syntax/blocks/test_reference_links.py @@ -60,3 +60,30 @@ def test_malformed_reference_with_long_run_of_spaces(self): self.assertMarkdownRenders( text, f'

{text}

', expected_attrs={'references': {}} ) + + def test_reference_definition_before_setext_header(self): + """ + A reference definition followed by a setext underline is a header, even + when it is preceded by another definition. + """ + self.assertMarkdownRenders( + '[a]: http://example.com/a\n[b]: http://example.com/b\n===', + '

[b]: http://example.com/b

', + expected_attrs={'references': {'a': ('http://example.com/a', None)}} + ) + + def test_many_reference_definitions(self): + """ + Consecutive reference definitions should be consumed together rather + than returning the remainder of the block to be parsed once per + definition. + """ + count = 10000 + text = ''.join(f'[id{i}]: http://example.com/{i}\n' for i in range(count)) + self.assertMarkdownRenders( + text, + '', + expected_attrs={ + 'references': {f'id{i}': (f'http://example.com/{i}', None) for i in range(count)} + } + ) diff --git a/tests/test_syntax/extensions/test_footnotes.py b/tests/test_syntax/extensions/test_footnotes.py index 896e1fcd..f7fc0b96 100644 --- a/tests/test_syntax/extensions/test_footnotes.py +++ b/tests/test_syntax/extensions/test_footnotes.py @@ -265,6 +265,29 @@ def test_footnote_multple_complex_no_blank_line_between(self): '' ) + def test_many_footnote_definitions(self): + """ + Consecutive footnote definitions should be consumed together rather + than returning the remainder of the block to be parsed once per + definition. + """ + count = 5000 + self.assertMarkdownRenders( + ''.join(f'[^{i}]: Footnote {i}\n' for i in range(count)), + '
\n' + '
\n' + '
    \n' + + ''.join( + f'
  1. \n' + f'

    Footnote {i} ↩

    \n' + '
  2. \n' + for i in range(count) + ) + + '
\n' + '
' + ) + def test_backlink_text(self): """Test back-link configuration.""" From 12b6975575733480a6a427fe79f303a077d5b931 Mon Sep 17 00:00:00 2001 From: Naaz nuria Date: Wed, 28 Oct 2026 09:55:51 +0530 Subject: [PATCH 2/3] Record all consecutive definitions in one pass Use finditer for footnote definitions as for reference definitions, and no longer leave the last definition of a run, or an indented one, for the next pass. A setext underline after consecutive reference definitions is now a paragraph rather than turning the last definition into a header. --- markdown/blockprocessors.py | 11 +++------ markdown/extensions/footnotes.py | 24 ++++++------------- .../blocks/test_reference_links.py | 12 ++++++---- 3 files changed, 17 insertions(+), 30 deletions(-) diff --git a/markdown/blockprocessors.py b/markdown/blockprocessors.py index 868f2227..87c298ed 100644 --- a/markdown/blockprocessors.py +++ b/markdown/blockprocessors.py @@ -593,18 +593,13 @@ def run(self, parent: etree.Element, blocks: list[str]) -> bool: matches = [m] # Collect the definitions on the following lines so they are all # recorded in this pass rather than one per pass through the block - # processors. A definition which is indented or spans lines is left - # for the next pass, as is the last one when other content follows - # it, as another processor may claim them (an indented code block or - # a setext header, for example). + # processors. A definition which spans lines is left for the next + # pass, as another processor may claim part of it. for m in self.RE.finditer(block, end): - if (block[end:m.start()].strip('\n') or '\n' in m.group() - or block.startswith(' ' * self.tab_length, m.start())): + if block[end:m.start()].strip('\n') or '\n' in m.group(): break matches.append(m) end = m.end() - if len(matches) > 1 and block[end:].strip(): - end = matches.pop().start() for m in matches: id = m.group(1).strip().lower() link = m.group(2).lstrip('<').rstrip('>') diff --git a/markdown/extensions/footnotes.py b/markdown/extensions/footnotes.py index d0648d82..cc425216 100644 --- a/markdown/extensions/footnotes.py +++ b/markdown/extensions/footnotes.py @@ -261,32 +261,22 @@ def run(self, parent: etree.Element, blocks: list[str]) -> bool: m = self.RE.search(block) if m: - first = m preceding = block[:m.start()] - therest = block[m.end():].lstrip('\n') - m2 = self.RE.search(therest) # Record the definitions which are directly followed by another # definition in this pass rather than one per pass through the block - # processors. The last one is left for the next pass when other - # content follows it, as another processor may claim that block (an - # indented code block or a setext header, for example). - while m2 and m2.start() == 0 and not therest.startswith(' ' * self.tab_length): + # processors. + for m2 in self.RE.finditer(block, m.end()): + if block[m.end():m2.start()].strip('\n'): + break self.footnotes.setFootnote(m.group(1), m.group(2).rstrip()) - block, m = therest, m2 - therest = block[m.end():].lstrip('\n') - m2 = self.RE.search(therest) - if m is not first and therest.strip(): - # Add back to blocks everything from beginning of match forward for next iteration. - blocks.insert(0, block) - if preceding.strip(): - # Add any content before match back to blocks as separate block - blocks.insert(0, preceding.rstrip('\n')) - return True + m = m2 id = m.group(1) fn_blocks = [m.group(2)] # Handle rest of block + therest = block[m.end():].lstrip('\n') + m2 = self.RE.search(therest) if m2: # Another footnote exists in the rest of this block. # Any content before match is continuation of this footnote, which may be lazily indented. diff --git a/tests/test_syntax/blocks/test_reference_links.py b/tests/test_syntax/blocks/test_reference_links.py index 983df42e..a17a5a8a 100644 --- a/tests/test_syntax/blocks/test_reference_links.py +++ b/tests/test_syntax/blocks/test_reference_links.py @@ -61,15 +61,17 @@ def test_malformed_reference_with_long_run_of_spaces(self): text, f'

{text}

', expected_attrs={'references': {}} ) - def test_reference_definition_before_setext_header(self): + def test_reference_definitions_before_setext_underline(self): """ - A reference definition followed by a setext underline is a header, even - when it is preceded by another definition. + A setext underline after consecutive reference definitions is a paragraph, + and does not turn the last definition into a header. """ self.assertMarkdownRenders( '[a]: http://example.com/a\n[b]: http://example.com/b\n===', - '

[b]: http://example.com/b

', - expected_attrs={'references': {'a': ('http://example.com/a', None)}} + '

===

', + expected_attrs={ + 'references': {'a': ('http://example.com/a', None), 'b': ('http://example.com/b', None)} + } ) def test_many_reference_definitions(self): From 2e8e7bc3e93a4240b7930f378aeda6d95c86abc0 Mon Sep 17 00:00:00 2001 From: Naaz nuria Date: Thu, 29 Oct 2026 10:04:37 +0530 Subject: [PATCH 3/3] Merge reference definition loops and add tests Record each reference definition as it is matched rather than collecting them first, and add tests covering runs of reference and footnote definitions next to split definitions, paragraphs, headers, rules, blockquotes and code blocks. --- markdown/blockprocessors.py | 20 ++- .../blocks/test_reference_links.py | 122 +++++++++++++++++ .../test_syntax/extensions/test_footnotes.py | 128 ++++++++++++++++++ 3 files changed, 258 insertions(+), 12 deletions(-) diff --git a/markdown/blockprocessors.py b/markdown/blockprocessors.py index 87c298ed..d3574045 100644 --- a/markdown/blockprocessors.py +++ b/markdown/blockprocessors.py @@ -588,23 +588,19 @@ def run(self, parent: etree.Element, blocks: list[str]) -> bool: block = blocks.pop(0) m = self.RE.search(block) if m: - start = m.start() - end = m.end() - matches = [m] - # Collect the definitions on the following lines so they are all - # recorded in this pass rather than one per pass through the block - # processors. A definition which spans lines is left for the next - # pass, as another processor may claim part of it. - for m in self.RE.finditer(block, end): - if block[end:m.start()].strip('\n') or '\n' in m.group(): + start = end = m.start() + # Record the definitions on the following lines in this pass rather + # than one per pass through the block processors. A definition after + # the first which spans lines is left for the next pass, as another + # processor may claim part of it. + for m in self.RE.finditer(block, start): + if m.start() > start and (block[end:m.start()].strip('\n') or '\n' in m.group()): break - matches.append(m) - end = m.end() - for m in matches: id = m.group(1).strip().lower() link = m.group(2).lstrip('<').rstrip('>') title = m.group(5) or m.group(6) self.parser.md.references[id] = (link, title) + end = m.end() if block[end:].strip(): # Add any content after match back to blocks as separate block blocks.insert(0, block[end:].lstrip('\n')) diff --git a/tests/test_syntax/blocks/test_reference_links.py b/tests/test_syntax/blocks/test_reference_links.py index a17a5a8a..5c9884b0 100644 --- a/tests/test_syntax/blocks/test_reference_links.py +++ b/tests/test_syntax/blocks/test_reference_links.py @@ -74,6 +74,128 @@ def test_reference_definitions_before_setext_underline(self): } ) + def test_reference_definitions_before_split_definition(self): + """ + A definition split across lines after other definitions is left for the + next pass, so a horizontal rule on its second line is not taken as the link. + """ + self.assertMarkdownRenders( + '[a]: http://example.com/a\n[b]:\n***', + '

[b]:

\n
', + expected_attrs={'references': {'a': ('http://example.com/a', None)}} + ) + + def test_reference_definitions_before_definition_split_across_lines(self): + self.assertMarkdownRenders( + '[a]: http://example.com/a\n[b]:\nhttp://example.com/b', + '', + expected_attrs={ + 'references': {'a': ('http://example.com/a', None), 'b': ('http://example.com/b', None)} + } + ) + + def test_reference_definitions_title_on_own_line(self): + self.assertMarkdownRenders( + '[a]: http://example.com/a\n[b]: http://example.com/b\n"Title"', + '', + expected_attrs={ + 'references': {'a': ('http://example.com/a', None), 'b': ('http://example.com/b', 'Title')} + } + ) + + def test_reference_definitions_before_paragraph(self): + self.assertMarkdownRenders( + '[a]: http://example.com/a\n[b]: http://example.com/b\nSome text', + '

Some text

', + expected_attrs={ + 'references': {'a': ('http://example.com/a', None), 'b': ('http://example.com/b', None)} + } + ) + + def test_paragraph_before_reference_definitions(self): + self.assertMarkdownRenders( + 'Some text\n[a]: http://example.com/a\n[b]: http://example.com/b', + '

Some text

', + expected_attrs={ + 'references': {'a': ('http://example.com/a', None), 'b': ('http://example.com/b', None)} + } + ) + + def test_reference_definitions_separated_by_paragraph(self): + self.assertMarkdownRenders( + '[a]: http://example.com/a\nSome text\n[b]: http://example.com/b', + '

Some text

', + expected_attrs={ + 'references': {'a': ('http://example.com/a', None), 'b': ('http://example.com/b', None)} + } + ) + + def test_indented_reference_definitions(self): + self.assertMarkdownRenders( + '[a]: http://example.com/a\n [b]: http://example.com/b', + '', + expected_attrs={ + 'references': {'a': ('http://example.com/a', None), 'b': ('http://example.com/b', None)} + } + ) + + def test_indented_reference_definitions_short_tab_length(self): + """ + A definition indented by `tab_length` spaces directly after another + definition is a definition, not an indented code block. + """ + self.assertMarkdownRenders( + '[a]: http://example.com/a\n [b]: http://example.com/b', + '', + expected_attrs={ + 'references': {'a': ('http://example.com/a', None), 'b': ('http://example.com/b', None)} + }, + tab_length=2 + ) + + def test_duplicate_reference_definitions(self): + self.assertMarkdownRenders( + '[a]: http://example.com/a\n[a]: http://example.com/b', + '', + expected_attrs={'references': {'a': ('http://example.com/b', None)}} + ) + + def test_reference_definitions_before_hash_header(self): + self.assertMarkdownRenders( + '[a]: http://example.com/a\n[b]: http://example.com/b\n# Header', + '

Header

', + expected_attrs={ + 'references': {'a': ('http://example.com/a', None), 'b': ('http://example.com/b', None)} + } + ) + + def test_reference_definitions_before_hr(self): + self.assertMarkdownRenders( + '[a]: http://example.com/a\n[b]: http://example.com/b\n***', + '
', + expected_attrs={ + 'references': {'a': ('http://example.com/a', None), 'b': ('http://example.com/b', None)} + } + ) + + def test_reference_definitions_before_blockquote(self): + self.assertMarkdownRenders( + '[a]: http://example.com/a\n[b]: http://example.com/b\n> quote', + '
\n

quote

\n
', + expected_attrs={ + 'references': {'a': ('http://example.com/a', None), 'b': ('http://example.com/b', None)} + } + ) + + def test_reference_definitions_before_code_block(self): + self.assertMarkdownRenders( + '[a]: http://example.com/a\n[b]: http://example.com/b\n code', + '
code\n
', + expected_attrs={ + 'references': {'a': ('http://example.com/a', None), 'b': ('http://example.com/b', None)} + } + ) + def test_many_reference_definitions(self): """ Consecutive reference definitions should be consumed together rather diff --git a/tests/test_syntax/extensions/test_footnotes.py b/tests/test_syntax/extensions/test_footnotes.py index f7fc0b96..5cd1037d 100644 --- a/tests/test_syntax/extensions/test_footnotes.py +++ b/tests/test_syntax/extensions/test_footnotes.py @@ -265,6 +265,134 @@ def test_footnote_multple_complex_no_blank_line_between(self): '' ) + def test_consecutive_footnotes_last_multi_line(self): + self.assertMarkdownRenders( + self.dedent( + """ + [^1]: Footnote 1 + [^2]: Footnote 2 + line 2 + """ + ), + '
\n' + '
\n' + '
    \n' + '
  1. \n' + '

    Footnote 1 ↩

    \n' + '
  2. \n' + '
  3. \n' + '

    Footnote 2\nline 2 ↩

    \n' + '
  4. \n' + '
\n' + '
' + ) + + def test_consecutive_footnotes_last_multi_line_lazy_indent(self): + self.assertMarkdownRenders( + self.dedent( + """ + [^1]: Footnote 1 + [^2]: Footnote 2 + line 2 + """ + ), + '
\n' + '
\n' + '
    \n' + '
  1. \n' + '

    Footnote 1 ↩

    \n' + '
  2. \n' + '
  3. \n' + '

    Footnote 2\nline 2 ↩

    \n' + '
  4. \n' + '
\n' + '
' + ) + + def test_consecutive_footnotes_last_multi_paragraph(self): + self.assertMarkdownRenders( + self.dedent( + """ + [^1]: Footnote 1 + [^2]: Footnote 2 + + paragraph 2 + """ + ), + '
\n' + '
\n' + '
    \n' + '
  1. \n' + '

    Footnote 1 ↩

    \n' + '
  2. \n' + '
  3. \n' + '

    Footnote 2

    \n' + '

    paragraph 2 ↩

    \n' + '
  4. \n' + '
\n' + '
' + ) + + def test_paragraph_before_consecutive_footnotes(self): + self.assertMarkdownRenders( + self.dedent( + """ + paragraph[^1] + [^1]: Footnote 1 + [^2]: Footnote 2 + """ + ), + '

paragraph1

\n' + '
\n' + '
\n' + '
    \n' + '
  1. \n' + '

    Footnote 1 ↩

    \n' + '
  2. \n' + '
  3. \n' + '

    Footnote 2 ↩

    \n' + '
  4. \n' + '
\n' + '
' + ) + + def test_consecutive_footnotes_before_setext_underline(self): + """ + A setext underline after consecutive footnotes is a lazily indented line + of the last footnote, and does not turn that definition into a header. + """ + self.assertMarkdownRenders( + self.dedent( + """ + [^1]: Footnote 1 + [^2]: Footnote 2 + === + """ + ), + '
\n' + '
\n' + '
    \n' + '
  1. \n' + '

    Footnote 1 ↩

    \n' + '
  2. \n' + '
  3. \n' + '

    Footnote 2

    \n' + '

    ↩

    \n' + '
  4. \n' + '
\n' + '
' + ) + def test_many_footnote_definitions(self): """ Consecutive footnote definitions should be consumed together rather