From cd2bb8e163f515df557be1a8815024d40b5c41fd Mon Sep 17 00:00:00 2001 From: facelessuser Date: Mon, 7 Sep 2026 13:32:47 -0600 Subject: [PATCH 01/12] Rewrite emphasis handling - This is a complete rewrite of how emphasis handling is done. - Drop use of multiple regex patterns in run in multiple passes and instead evaluate delimiters, nested or otherwise, and build up HTML elements. - Try to consume tokens as much as possible until a full element is constructed (with children if any). - If an outer set of tokens cannot be resolved, but one or more sub tokens can, render the first sub token span and cache the remaining ones for subsequent reentry and render those until the cache is exhausted. - Two tests results were updated to match new behavior. --- markdown/extensions/legacy_em.py | 28 +- markdown/inlinepatterns.py | 531 +++++++++++++++------- tests/misc/underscores.html | 6 - tests/misc/underscores.txt | 11 - tests/test_syntax/inline/test_emphasis.py | 33 +- 5 files changed, 400 insertions(+), 209 deletions(-) delete mode 100644 tests/misc/underscores.html delete mode 100644 tests/misc/underscores.txt diff --git a/markdown/extensions/legacy_em.py b/markdown/extensions/legacy_em.py index 39efe9a73..a78ebb402 100644 --- a/markdown/extensions/legacy_em.py +++ b/markdown/extensions/legacy_em.py @@ -14,29 +14,7 @@ from __future__ import annotations from . import Extension -from ..inlinepatterns import UnderscoreProcessor, EmStrongItem, EM_STRONG2_RE, STRONG_EM2_RE -import re - -# _emphasis_ -EMPHASIS_RE = r'(_)([^_]+)\1' - -# __strong__ -STRONG_RE = r'(_{2})(.+?)\1' - -# __strong_em___ -STRONG_EM_RE = r'(_)\1(?!\1)([^_]+?)\1(?!\1)(.+?)\1{3}' - - -class LegacyUnderscoreProcessor(UnderscoreProcessor): - """Emphasis processor for handling strong and em matches inside underscores.""" - - PATTERNS = [ - EmStrongItem(re.compile(EM_STRONG2_RE, re.DOTALL | re.UNICODE), 'double', 'strong,em'), - EmStrongItem(re.compile(STRONG_EM2_RE, re.DOTALL | re.UNICODE), 'double', 'em,strong'), - EmStrongItem(re.compile(STRONG_EM_RE, re.DOTALL | re.UNICODE), 'double2', 'strong,em'), - EmStrongItem(re.compile(STRONG_RE, re.DOTALL | re.UNICODE), 'single', 'strong'), - EmStrongItem(re.compile(EMPHASIS_RE, re.DOTALL | re.UNICODE), 'single', 'em') - ] +from ..inlinepatterns import DelimiterProcessor class LegacyEmExtension(Extension): @@ -50,8 +28,8 @@ def extendMarkdown(self, md): | [`LegacyUnderscoreProcessor`][markdown.extensions.legacy_em.LegacyUnderscoreProcessor] | [`inlinepatterns`][markdown.inlinepatterns.build_inlinepatterns] | `em_strong2` | `50` | """ - # flake8: noqa: E501 48-50 - md.inlinePatterns.register(LegacyUnderscoreProcessor(r'_'), 'em_strong2', 50) + # flake8: noqa: E501 27-29 + md.inlinePatterns.register(DelimiterProcessor(r'_', 'strong,em'), 'em_strong2', 50) def makeExtension(**kwargs): # pragma: no cover diff --git a/markdown/inlinepatterns.py b/markdown/inlinepatterns.py index 0f3533b2e..02a05aecf 100644 --- a/markdown/inlinepatterns.py +++ b/markdown/inlinepatterns.py @@ -41,7 +41,8 @@ from __future__ import annotations from . import util -from typing import TYPE_CHECKING, Any, Collection, NamedTuple +from typing import TYPE_CHECKING, Any, Collection, NamedTuple, cast +from collections import deque import re import xml.etree.ElementTree as etree from html import entities @@ -89,9 +90,8 @@ def build_inlinepatterns(md: Markdown, **kwargs: Any) -> util.Registry[InlinePro inlinePatterns.register(SubstituteTagInlineProcessor(LINE_BREAK_RE, 'br'), 'linebreak', 100) inlinePatterns.register(HtmlInlineProcessor(HTML_RE, md), 'html', 90) inlinePatterns.register(HtmlInlineProcessor(ENTITY_RE, md), 'entity', 80) - inlinePatterns.register(SimpleTextInlineProcessor(NOT_STRONG_RE), 'not_strong', 70) - inlinePatterns.register(AsteriskProcessor(r'\*'), 'em_strong', 60) - inlinePatterns.register(UnderscoreProcessor(r'_'), 'em_strong2', 50) + inlinePatterns.register(DelimiterProcessor('*', 'strong,em'), 'em_strong', 60) + inlinePatterns.register(DelimiterProcessor('_', 'strong,em', smart=True), 'em_strong2', 50) return inlinePatterns @@ -107,36 +107,6 @@ def build_inlinepatterns(md: Markdown, **kwargs: Any) -> util.Registry[InlinePro ESCAPE_RE = r'\\(.)' """ Match a backslash escaped character (`\\<` or `\\*`). """ -EMPHASIS_RE = r'(\*)([^\*]+)\1' -""" Match emphasis with an asterisk (`*emphasis*`). """ - -STRONG_RE = r'(\*{2})(.+?)\1' -""" Match strong with an asterisk (`**strong**`). """ - -SMART_STRONG_RE = r'(?)` or `[text](url "title")`). """ @@ -149,9 +119,6 @@ def build_inlinepatterns(md: Markdown, **kwargs: Any) -> util.Registry[InlinePro IMAGE_REFERENCE_RE = IMAGE_LINK_RE """ Match start of image reference (`![alt text][2]`). """ -NOT_STRONG_RE = r'((^|(?<=\s))(\*{1,3}|_{1,3})(?=\s|$))' -""" Match a stand-alone `*` or `_`. """ - AUTOLINK_RE = r'<((?:[Ff]|[Hh][Tt])[Tt][Pp][Ss]?://[^<>]*)>' """ Match an automatic link (``). """ @@ -592,151 +559,383 @@ def _unescape(m: re.Match[str]) -> str: return RE.sub(_unescape, text) -class AsteriskProcessor(InlineProcessor): - """Emphasis processor for handling strong and em matches inside asterisks.""" - - PATTERNS = [ - EmStrongItem(re.compile(EM_STRONG_RE, re.DOTALL | re.UNICODE), 'double', 'strong,em'), - EmStrongItem(re.compile(STRONG_EM_RE, re.DOTALL | re.UNICODE), 'double', 'em,strong'), - EmStrongItem(re.compile(STRONG_EM3_RE, re.DOTALL | re.UNICODE), 'double2', 'strong,em'), - EmStrongItem(re.compile(STRONG_RE, re.DOTALL | re.UNICODE), 'single', 'strong'), - EmStrongItem(re.compile(EMPHASIS_RE, re.DOTALL | re.UNICODE), 'single', 'em') - ] - """ The various strong and emphasis patterns handled by this processor. """ +class DelimiterProcessor(InlineProcessor): + """Processor for handling complex nested patterns such as strong and em matches.""" - def build_single(self, m: re.Match[str], tag: str, idx: int) -> etree.Element: - """Return single tag.""" - el1 = etree.Element(tag) - text = m.group(2) - self.parse_sub_patterns(text, el1, None, idx) - return el1 - - def build_double(self, m: re.Match[str], tags: str, idx: int) -> etree.Element: - """Return double tag.""" - - tag1, tag2 = tags.split(",") - el1 = etree.Element(tag1) - el2 = etree.Element(tag2) - text = m.group(2) - self.parse_sub_patterns(text, el2, None, idx) - el1.append(el2) - if len(m.groups()) == 3: - text = m.group(3) - self.parse_sub_patterns(text, el1, el2, idx) - return el1 - - def build_double2(self, m: re.Match[str], tags: str, idx: int) -> etree.Element: - """Return double tags (variant 2): `text text`.""" - - tag1, tag2 = tags.split(",") - el1 = etree.Element(tag1) - el2 = etree.Element(tag2) - text = m.group(2) - self.parse_sub_patterns(text, el1, None, idx) - text = m.group(3) - el1.append(el2) - self.parse_sub_patterns(text, el2, None, idx) - return el1 - - def parse_sub_patterns( - self, data: str, parent: etree.Element, last: etree.Element | None, idx: int + def __init__( + self, + token: str, + tags: str, + md: Markdown | None = None, + smart: bool = False, + double: bool = False ) -> None: """ - Parses sub patterns. - - `data`: text to evaluate. + Initialize. - `parent`: Parent to attach text and sub elements to. - - `last`: Last appended child to parent. Can also be None if parent has no children. + Arguments: + token: A single character token. + tags: A tag or two tags seprated by comma. When two are specified, the first will be the + one that takes double tokens. + md: the Markdown object + smart: Enable intelligent word logic. + double: If only one tag is specified, indicate whether it requires double tokens. - `idx`: Current pattern index that was used to evaluate the parent. """ - offset = 0 - pos = 0 - - length = len(data) - while pos < length: - # Find the start of potential emphasis or strong tokens - if self.compiled_re.match(data, pos): - matched = False - # See if the we can match an emphasis/strong pattern - for index, item in enumerate(self.PATTERNS): - # Only evaluate patterns that are after what was used on the parent - if index <= idx: - continue - m = item.pattern.match(data, pos) - if m: - # Append child nodes to parent - # Text nodes should be appended to the last - # child if present, and if not, it should - # be added as the parent's text node. - text = data[offset:m.start(0)] - if text: - if last is not None: - last.tail = text - else: - parent.text = text - el = self.build_element(m, item.builder, item.tags, index) - parent.append(el) - last = el - # Move our position past the matched hunk - offset = pos = m.end(0) - matched = True - if not matched: - # We matched nothing, move on to the next character - pos += 1 + # Cache info + self.regions: list[tuple[int, int, int, int, int]] = [] + self.stack: deque[tuple[int, int, int]] = deque() + self.cache_index = 0 + self.cache_pos = 0 + + self.smart = smart + self.tags = tags.split(',') + self.double = len(tags) != 2 and double + super().__init__(self._build_patterns(token), md) + + def _build_patterns(self, token: str) -> str: + """Build regular expression patterns.""" + + # Build up patterns + self.token = token + etoken = re.escape(token) + avoid_start = fr'(?:(?<=_)|(?(?(?{avoid_start}{etoken}{{1,3}}(?![\s{etoken}])(?!$)) + ''', + flags=re.UNICODE + ) + elif self.double: + self.boundary = re.compile( + fr'''(?x) + (?P(?(?{avoid_start}{etoken}{{2}}(?![\s{etoken}])(?!$)) + ''', + flags=re.UNICODE + ) else: - # Increment position as no potential emphasis start was found. - pos += 1 - - # Append any leftover text as a text node. - text = data[offset:] - if text: - if last is not None: - last.tail = text + # This case is not currently used + self.boundary = re.compile( + fr'''(?x) + (?P(?(?{avoid_start}{etoken}{{1}}(?![\s{etoken}])(?!$)) + ''', + flags=re.UNICODE + ) + # Patterns for "dumb" cases. + else: + if len(self.tags) == 2: + self.boundary = re.compile( + fr'''(?x)(?: + (?P(?(?{etoken}{{1,3}}(?![\s{etoken}])(?!$)) + )''', + flags=re.UNICODE + ) + elif self.double: + self.boundary = re.compile( + fr'''(?x) + (?P(?(?{etoken}{{2}}(?![\s{etoken}])(?!$)) + ''', + flags=re.UNICODE + ) else: - parent.text = text - - def build_element(self, m: re.Match[str], builder: str, tags: str, index: int) -> etree.Element: + self.boundary = re.compile( + fr'''(?x) + (?P(?(?{etoken}{{1}}(?![\s{etoken}])(?!$)) + ''', + flags=re.UNICODE + ) + + return fr'{etoken}' + + def _build_element( + self, + data: str, + start: int = 0, + offset: int = 0 + ) -> tuple[etree.Element, int]: """Element builder.""" - if builder == 'double2': - return self.build_double2(m, tags, index) - elif builder == 'double': - return self.build_double(m, tags, index) + regions = self.regions + el: etree.Element | None = None + last: Any = None + previous: Any = None + greater: Any = None + lesser: Any = None + + triple = set() + outer: list[etree.Element] = [] + outer_r: list[tuple[int, int, int, int, int]] = [] + + if len(self.tags) == 2: + greater, lesser = self.tags + elif self.double: + greater = self.tags[0] + lesser = None else: - return self.build_single(m, tags, index) - - def handleMatch(self, m: re.Match[str], data: str) -> tuple[etree.Element | None, int | None, int | None]: - """Parse patterns.""" - - el = None - start = None - end = None - - for index, item in enumerate(self.PATTERNS): - m1 = item.pattern.match(data, m.start(0)) - if m1: - start = m1.start(0) - end = m1.end(0) - el = self.build_element(m1, item.builder, item.tags, index) + lesser = self.tags[0] + greater = None + + # Iterate regions creating the elements they represent + end = len(regions) + idx = 0 + for idx, i in enumerate(range(start, end), 1): + r = regions[i] + # Not contained within region + if idx and r[0] > regions[start][3]: + idx -= 1 break - return el, start, end + # Get the appropriate element(s) + if r[4] == 3: + el1 = etree.Element(greater) + el2 = etree.Element(lesser) + elif r[4] == 2: + el1 = etree.Element(greater) + el2 = None + else: + el1 = etree.Element(lesser) + el2 = None + + # Populate the elements with their text + if idx > 1: + if last.text is None: + if previous[2] < r[0]: + last.text = data[previous[1]+offset:previous[2]+offset] + else: + last.text = data[previous[1]+offset:r[0]+offset] + if last is not outer[-1] and last.tail is None: + if r[0] < outer_r[-1][3]: + last.tail = data[previous[3]+offset:r[0]+offset] + else: + last.tail = data[previous[3]+offset:outer_r[-1][2]+offset] + outer[-1].tail = data[outer_r[-1][3]+offset:r[0]+offset] + + # First element + if el is None: + el = el1 + last = el + outer.append(el) + outer_r.append(r) + + # Subsequent elements + else: + # Is the current outer element no longer wrapping this one? + while len(outer_r) > 1 and r[3] > outer_r[-1][3]: + outer.pop() + outer_r.pop() + # Double nested element (triple token) + if outer[-1] in triple: + outer[-1][-1].append(el1) + + # Non-nested + else: + outer[-1].append(el1) + + # Is this element wrapping the next? + if i + 1 < end: + if r[3] > regions[i + 1][3]: + outer.append(el1) + outer_r.append(r) + + # Track the last element we parsed. + last = el1 + + # Nest secondary element if there is one. + # Track triple tokens (double elements) + # so we can identify quickly and properly nest. + if el2 is not None: + el1.append(el2) + last = el2 + triple.add(el1) + + # Track the previous region. + previous = r + + # Populate remaining elements with their text + while outer: + if last.text is None: + last.text = data[previous[1]+offset:previous[2]+offset] + if last.tail is None and last is not outer[-1]: + last.tail = data[previous[3]+offset:outer_r[-1][2]+offset] + last = outer.pop() + previous = outer_r.pop() + + return cast('etree.Element', el), idx + + def get_cached_result(self, pos: int, data: str) -> tuple[etree.Element, int, int]: + """Get a cached result.""" + + stack = self.stack + regions = self.regions + + # Process the next region(s) in the cache + offset = pos - self.cache_pos + start, end = regions[self.cache_index][0], regions[self.cache_index][3] + el, count = self._build_element(data, self.cache_index, offset) + + # Determine next offset + self.cache_index += count + if self.cache_index < len(regions): + self.cache_pos = regions[self.cache_index][0] + while stack: + entry = stack.popleft() + if entry[0] > end: + if entry[0] < self.cache_pos: + self.cache_pos = entry[0] + break + + # Nothing left to process + else: + regions.clear() + stack.clear() + self.cache_index = 0 + self.cache_pos = 0 + + # Whether element is valid or not, we'll advance past the end + return el, start + offset, end + offset + + def handleMatch( # type: ignore[override] + self, + m: re.Match[str], + data: str + ) -> tuple[etree.Element | None, int | None, int | None]: + """Parse delimiter pattern.""" + + # Do we have entries we haven't returned yet? + if self.regions: + return self.get_cached_result(m.start(0), data) + + # If token is not an opening, quit + m2 = self.boundary.match(data, m.start(0)) + if m2 is None or m2.lastgroup[0] == 'e': # type: ignore[index] + if m2 is not None: + m = m2 + # Advance past the full length of the delimiter found + return None, m.start(0), m.end(0) + + # Get the stack and regions + stack = self.stack + regions = self.regions + + # Data offset + offset = m2.end(0) + # Stack of opening delimiters + stack.append((m2.start(0), offset, len(m2.group(0)))) + + # Pair tokens until the stack is empty or we can no longer find tokens. + while stack: + m2 = self.boundary.search(data, offset) + if m2 is None: + break + offset = m2.end(0) + + # Get current and last delimiter size + current = len(m2.group(0)) + last = stack[-1][-1] + + # Some delimiters may be ambiguous and look like both a start or an end + is_start = m2.lastgroup[0] != 'e' # type: ignore[index] + is_end = not is_start or m2.lastgroup[0] != 's' # type: ignore[index] + ambiguous = is_start and is_end + + # Find closing tokens + # Looking for: + # - `*em*` + # - `**strong**` + # - `***strong,em***` + # - `*em**` + # - `*em***` + # - `**strong***` + # + # Avoid ambiguous tokens that could be a start or an end. + # Consume starts until the end token is fully consumed. + # If we don't consume the entire end, see if next rule consumes it. + if is_end and ((not ambiguous and current > last) or (current == last)): + is_start = False + + # Consume previous tokens until the delimiter is consumed + s = m2.start(0) + while current and last <= current: + delimiter = stack.pop() + + # Build up region for pair and adjust accounting. + regions.append((delimiter[0], delimiter[1], s, s + delimiter[-1], delimiter[-1])) + s += delimiter[-1] + current -= delimiter[-1] + if not stack: + is_end = False + break + last = stack[-1][-1] + + # Do we still have more to consume? + is_end = current and stack and last > current + + # Looking for: + # - `***em*` + # - `***strong**` + # - `**em*` + if is_end and (last == 3 or not ambiguous) and last > current: + is_start = False + delimiter = stack.pop() + new = last - current + regions.append((delimiter[0] + new, delimiter[1], m2.start(0), offset, current)) + stack.append((delimiter[0], delimiter[0] + new, new)) + + # Find opening tokens + if is_start: + # Looking for: + # - `*em ...*` + # - `**strong ...*` + # - `***em ...*` + stack.append((m2.start(0), m2.end(0), current)) + + # Build the HTML elements + if regions: + # Regions may be out of order. + regions.sort(key=lambda x: x[0]) + start, end = regions[0][0], regions[0][3] + el, count = self._build_element(data) + + # Cache unprocessed regions to avoid repeated searches + if count < len(regions): + self.cache_index = count + self.cache_pos = self.regions[count][0] + while stack: + entry = stack.popleft() + if entry[0] > end: + if entry[0] < self.cache_pos: + self.cache_pos = entry[0] + break + else: + # Cleanup + stack.clear() + regions.clear() -class UnderscoreProcessor(AsteriskProcessor): - """Emphasis processor for handling strong and em matches inside underscores.""" + return el, start, end - PATTERNS = [ - EmStrongItem(re.compile(EM_STRONG2_RE, re.DOTALL | re.UNICODE), 'double', 'strong,em'), - EmStrongItem(re.compile(STRONG_EM2_RE, re.DOTALL | re.UNICODE), 'double', 'em,strong'), - EmStrongItem(re.compile(SMART_STRONG_EM_RE, re.DOTALL | re.UNICODE), 'double2', 'strong,em'), - EmStrongItem(re.compile(SMART_STRONG_RE, re.DOTALL | re.UNICODE), 'single', 'strong'), - EmStrongItem(re.compile(SMART_EMPHASIS_RE, re.DOTALL | re.UNICODE), 'single', 'em') - ] - """ The various strong and emphasis patterns handled by this processor. """ + # We failed to pair any valid start/end delimiters, avoid the parsed range next pass. + start = m.start(0) + end = stack[-1][1] if stack else m.end(0) + stack.clear() + return None, start, end class LinkInlineProcessor(InlineProcessor): diff --git a/tests/misc/underscores.html b/tests/misc/underscores.html deleted file mode 100644 index 72d51b8b5..000000000 --- a/tests/misc/underscores.html +++ /dev/null @@ -1,6 +0,0 @@ -

THIS_SHOULD_STAY_AS_IS

-

Here is some emphasis, ok?

-

Ok, at least this should work.

-

THIS__SHOULD__STAY

-

Here is some strong stuff.

-

THISSHOULDSTAY?

\ No newline at end of file diff --git a/tests/misc/underscores.txt b/tests/misc/underscores.txt deleted file mode 100644 index 3c7f4bdd9..000000000 --- a/tests/misc/underscores.txt +++ /dev/null @@ -1,11 +0,0 @@ -THIS_SHOULD_STAY_AS_IS - -Here is some _emphasis_, ok? - -Ok, at least _this_ should work. - -THIS__SHOULD__STAY - -Here is some __strong__ stuff. - -THIS___SHOULD___STAY? diff --git a/tests/test_syntax/inline/test_emphasis.py b/tests/test_syntax/inline/test_emphasis.py index 6e96ea32c..a7b3c56fa 100644 --- a/tests/test_syntax/inline/test_emphasis.py +++ b/tests/test_syntax/inline/test_emphasis.py @@ -20,6 +20,7 @@ """ from markdown.test_tools import TestCase +import textwrap class TestNotEmphasis(TestCase): @@ -147,7 +148,7 @@ def test_complex_emphasis_smart_underscore(self): def test_complex_emphasis_smart_underscore_mid_word(self): self.assertMarkdownRenders( 'This is text __bold_italic bold___ with more text', - '

This is text __bold_italic bold___ with more text

' + '

This is text bold_italic bold_ with more text

' ) def test_nested_emphasis(self): @@ -191,3 +192,33 @@ def test_link_emphasis_inner_outer(self): '**[**text**](url)**', '

text

' ) + + def test_underscore_legacy(self): + + self.assertMarkdownRenders( + textwrap.dedent( + """ + THIS_SHOULD_STAY_AS_IS + + Here is some _emphasis_, ok? + + Ok, at least _this_ should work. + + THIS__SHOULD__STAY + + Here is some __strong__ stuff. + + THIS___SHOULD___STAY? + """ + ), + textwrap.dedent( + """ +

THIS_SHOULD_STAY_AS_IS

+

Here is some emphasis, ok?

+

Ok, at least this should work.

+

THIS__SHOULD__STAY

+

Here is some strong stuff.

+

THIS___SHOULD___STAY?

+ """ + ).strip() + ) From 59fdb1d6d58915a1844b2c49a6408a1dd7a60a16 Mon Sep 17 00:00:00 2001 From: facelessuser Date: Tue, 8 Sep 2026 06:44:37 -0600 Subject: [PATCH 02/12] Remove outdated comment --- markdown/inlinepatterns.py | 1 - 1 file changed, 1 deletion(-) diff --git a/markdown/inlinepatterns.py b/markdown/inlinepatterns.py index 02a05aecf..723aa983d 100644 --- a/markdown/inlinepatterns.py +++ b/markdown/inlinepatterns.py @@ -809,7 +809,6 @@ def get_cached_result(self, pos: int, data: str) -> tuple[etree.Element, int, in self.cache_index = 0 self.cache_pos = 0 - # Whether element is valid or not, we'll advance past the end return el, start + offset, end + offset def handleMatch( # type: ignore[override] From aa2ed07f8e3fe1254bdb5c0d89917fa13ab2ee1c Mon Sep 17 00:00:00 2001 From: facelessuser Date: Tue, 8 Sep 2026 06:59:59 -0600 Subject: [PATCH 03/12] If Markdown object is ever left in a bad state reset the stack This is a prevented measure to ensure the processor is always in a good state. This situation has never been observed, but if it did occur, this would allow the processor to reset its state and continue properly. Only extensions have a way to reset, processors don't. Nor do they have a way to detect when a reset would be needed. - Add a current time for each Markdown run. - Have DelimiterProcessor compare the each run time, and if it has changed, perform a reset of the stack. --- markdown/core.py | 3 +++ markdown/extensions/legacy_em.py | 2 +- markdown/inlinepatterns.py | 12 ++++++++++-- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/markdown/core.py b/markdown/core.py index 370cb7ec5..fd946d54f 100644 --- a/markdown/core.py +++ b/markdown/core.py @@ -20,6 +20,7 @@ from __future__ import annotations import codecs +import time import sys import logging import importlib @@ -106,6 +107,7 @@ def __init__(self, **kwargs): """ + self.last_run: float = 0.0 self.tab_length: int = kwargs.get('tab_length', 4) self.ESCAPED_CHARS: list[str] = [ @@ -267,6 +269,7 @@ def reset(self) -> Markdown: Called once upon creation of a class instance. Should be called manually between calls to [`Markdown.convert`][markdown.Markdown.convert]. """ + self.last_run = time.time() self.htmlStash.reset() self.references.clear() diff --git a/markdown/extensions/legacy_em.py b/markdown/extensions/legacy_em.py index a78ebb402..3a9b02e1d 100644 --- a/markdown/extensions/legacy_em.py +++ b/markdown/extensions/legacy_em.py @@ -29,7 +29,7 @@ def extendMarkdown(self, md): """ # flake8: noqa: E501 27-29 - md.inlinePatterns.register(DelimiterProcessor(r'_', 'strong,em'), 'em_strong2', 50) + md.inlinePatterns.register(DelimiterProcessor(r'_', 'strong,em', md), 'em_strong2', 50) def makeExtension(**kwargs): # pragma: no cover diff --git a/markdown/inlinepatterns.py b/markdown/inlinepatterns.py index 723aa983d..b78ef4212 100644 --- a/markdown/inlinepatterns.py +++ b/markdown/inlinepatterns.py @@ -90,8 +90,8 @@ def build_inlinepatterns(md: Markdown, **kwargs: Any) -> util.Registry[InlinePro inlinePatterns.register(SubstituteTagInlineProcessor(LINE_BREAK_RE, 'br'), 'linebreak', 100) inlinePatterns.register(HtmlInlineProcessor(HTML_RE, md), 'html', 90) inlinePatterns.register(HtmlInlineProcessor(ENTITY_RE, md), 'entity', 80) - inlinePatterns.register(DelimiterProcessor('*', 'strong,em'), 'em_strong', 60) - inlinePatterns.register(DelimiterProcessor('_', 'strong,em', smart=True), 'em_strong2', 50) + inlinePatterns.register(DelimiterProcessor('*', 'strong,em', md), 'em_strong', 60) + inlinePatterns.register(DelimiterProcessor('_', 'strong,em', md, smart=True), 'em_strong2', 50) return inlinePatterns @@ -589,6 +589,7 @@ def __init__( self.cache_index = 0 self.cache_pos = 0 + self.last_run = 0.0 self.smart = smart self.tags = tags.split(',') self.double = len(tags) != 2 and double @@ -818,6 +819,13 @@ def handleMatch( # type: ignore[override] ) -> tuple[etree.Element | None, int | None, int | None]: """Parse delimiter pattern.""" + # We are in a new run. Reset just in case we were somehow left in a bad state. + if self.md.last_run != self.last_run: + self.regions.clear() + self.stack.clear() + self.cache_index = 0 + self.cache_pos = 0 + # Do we have entries we haven't returned yet? if self.regions: return self.get_cached_result(m.start(0), data) From 99594f2a80dbc69ec37c645deca39a3ee022c610 Mon Sep 17 00:00:00 2001 From: facelessuser Date: Tue, 8 Sep 2026 07:25:18 -0600 Subject: [PATCH 04/12] Consolidate reset behavior and ensure we update our tracked run --- markdown/inlinepatterns.py | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/markdown/inlinepatterns.py b/markdown/inlinepatterns.py index b78ef4212..e62902691 100644 --- a/markdown/inlinepatterns.py +++ b/markdown/inlinepatterns.py @@ -583,18 +583,22 @@ def __init__( """ - # Cache info - self.regions: list[tuple[int, int, int, int, int]] = [] - self.stack: deque[tuple[int, int, int]] = deque() - self.cache_index = 0 - self.cache_pos = 0 - self.last_run = 0.0 self.smart = smart self.tags = tags.split(',') self.double = len(tags) != 2 and double + self.reset() super().__init__(self._build_patterns(token), md) + def reset(self): + """Rest.""" + + # Cache info + self.regions: list[tuple[int, int, int, int, int]] = [] + self.stack: deque[tuple[int, int, int]] = deque() + self.cache_index = 0 + self.cache_pos = 0 + def _build_patterns(self, token: str) -> str: """Build regular expression patterns.""" @@ -805,10 +809,7 @@ def get_cached_result(self, pos: int, data: str) -> tuple[etree.Element, int, in # Nothing left to process else: - regions.clear() - stack.clear() - self.cache_index = 0 - self.cache_pos = 0 + self.reset() return el, start + offset, end + offset @@ -821,10 +822,8 @@ def handleMatch( # type: ignore[override] # We are in a new run. Reset just in case we were somehow left in a bad state. if self.md.last_run != self.last_run: - self.regions.clear() - self.stack.clear() - self.cache_index = 0 - self.cache_pos = 0 + self.last_run = self.md.last_run + self.reset() # Do we have entries we haven't returned yet? if self.regions: From 19d247f6fc4b18e837dcc061af13f2cc626e036a Mon Sep 17 00:00:00 2001 From: facelessuser Date: Tue, 8 Sep 2026 07:27:37 -0600 Subject: [PATCH 05/12] Have other reset locations call the reset function as well --- markdown/inlinepatterns.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/markdown/inlinepatterns.py b/markdown/inlinepatterns.py index e62902691..6bebde8b1 100644 --- a/markdown/inlinepatterns.py +++ b/markdown/inlinepatterns.py @@ -932,15 +932,14 @@ def handleMatch( # type: ignore[override] break else: # Cleanup - stack.clear() - regions.clear() + self.reset() return el, start, end # We failed to pair any valid start/end delimiters, avoid the parsed range next pass. start = m.start(0) end = stack[-1][1] if stack else m.end(0) - stack.clear() + self.reset() return None, start, end From b82bd466af065e84d3997d20b07f555ed90d2640 Mon Sep 17 00:00:00 2001 From: facelessuser Date: Tue, 8 Sep 2026 08:04:49 -0600 Subject: [PATCH 06/12] Add return type --- markdown/inlinepatterns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/markdown/inlinepatterns.py b/markdown/inlinepatterns.py index 6bebde8b1..e25acf653 100644 --- a/markdown/inlinepatterns.py +++ b/markdown/inlinepatterns.py @@ -590,7 +590,7 @@ def __init__( self.reset() super().__init__(self._build_patterns(token), md) - def reset(self): + def reset(self) -> None: """Rest.""" # Cache info From dd24bcd44a5b50e84f52162931c4261ae38fb35d Mon Sep 17 00:00:00 2001 From: facelessuser Date: Tue, 8 Sep 2026 08:27:01 -0600 Subject: [PATCH 07/12] Fix link errors --- docs/extensions/api.md | 2 +- markdown/extensions/legacy_em.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/extensions/api.md b/docs/extensions/api.md index 6952bcd19..4822bbc2b 100644 --- a/docs/extensions/api.md +++ b/docs/extensions/api.md @@ -349,7 +349,7 @@ Here are some convenience functions and other examples: | Class | Kind | Description | | -------------------------------------------------------------------------------------|-----------|---------------------------------------------------------------| -| [`AsteriskProcessor`][markdown.inlinepatterns.AsteriskProcessor] | built-in | Emphasis processor for handling strong and em matches inside asterisks | +| [`DelimiterProcessor`][markdown.inlinepatterns.DelimiterProcessor] | built-in | Emphasis processor for handling strong and em matches | | [`WikiLinksInlineProcessor`][markdown.extensions.wikilinks.WikiLinksInlineProcessor] | extension | Link `[[article names]]` to wiki given in metadata | | [`FootnoteInlineProcessor`][markdown.extensions.footnotes.FootnoteInlineProcessor] | extension | Replaces footnote in text with link to footnote div at bottom | diff --git a/markdown/extensions/legacy_em.py b/markdown/extensions/legacy_em.py index 3a9b02e1d..e0edf05b4 100644 --- a/markdown/extensions/legacy_em.py +++ b/markdown/extensions/legacy_em.py @@ -23,9 +23,9 @@ class LegacyEmExtension(Extension): def extendMarkdown(self, md): """ Register the processor. - | Class Instance | Registry | Name | Priority | - | ------------------------------------------------------------- | ---------------------------------------------------------------- | ------ | :------: | - | [`LegacyUnderscoreProcessor`][markdown.extensions.legacy_em.LegacyUnderscoreProcessor] | [`inlinepatterns`][markdown.inlinepatterns.build_inlinepatterns] | `em_strong2` | `50` | + | Class Instance | Registry | Name | Priority | + | ------------------------------------------------------------------ | ---------------------------------------------------------------- | ------------ | :------: | + | [`DelimiterProcessor`][markdown.inlinepatterns.DelimiterProcessor] | [`inlinepatterns`][markdown.inlinepatterns.build_inlinepatterns] | `em_strong2` | `50` | """ # flake8: noqa: E501 27-29 From ee8182804fcb229248881afea57fc19e7522f25e Mon Sep 17 00:00:00 2001 From: facelessuser Date: Tue, 8 Sep 2026 08:30:52 -0600 Subject: [PATCH 08/12] Add a changelog item --- docs/changelog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.md b/docs/changelog.md index 913732761..b29385e2d 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -20,6 +20,7 @@ See the [Contributing Guide](contributing.md) for details. performance for repeated inline patterns (#1619). * Officially support Python 3.15 and drop support for Python 3.10 * Walk backtick runs in `BacktickInlineProcessor` without a regex (#1620). +* Complete rework of emphasis handling to imporove nested emphasis handling better (#1632). ### Fixed From 7f76bfb646d8a454058e48d899446335bd11cb78 Mon Sep 17 00:00:00 2001 From: facelessuser Date: Wed, 9 Sep 2026 10:03:55 -0600 Subject: [PATCH 09/12] Consolidate duplicate code --- markdown/inlinepatterns.py | 51 ++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 30 deletions(-) diff --git a/markdown/inlinepatterns.py b/markdown/inlinepatterns.py index e25acf653..93c49995b 100644 --- a/markdown/inlinepatterns.py +++ b/markdown/inlinepatterns.py @@ -785,32 +785,37 @@ def _build_element( return cast('etree.Element', el), idx - def get_cached_result(self, pos: int, data: str) -> tuple[etree.Element, int, int]: - """Get a cached result.""" - - stack = self.stack - regions = self.regions + def increment_next_position(self, end: int, count: int) -> None: + """ + Increment cache position to the next location that we can initiate an insertion. - # Process the next region(s) in the cache - offset = pos - self.cache_pos - start, end = regions[self.cache_index][0], regions[self.cache_index][3] - el, count = self._build_element(data, self.cache_index, offset) + Cache position should be the first match after our current replacement. + This gives us an anchor to calculate the new offset after insertion. + """ # Determine next offset self.cache_index += count - if self.cache_index < len(regions): - self.cache_pos = regions[self.cache_index][0] - while stack: - entry = stack.popleft() + if self.cache_index < len(self.regions): + self.cache_pos = self.regions[self.cache_index][0] + while self.stack: + entry = self.stack.popleft() if entry[0] > end: - if entry[0] < self.cache_pos: - self.cache_pos = entry[0] + self.cache_pos = entry[0] break # Nothing left to process else: self.reset() + def get_cached_result(self, pos: int, data: str) -> tuple[etree.Element, int, int]: + """Get a cached result.""" + + # Process the next region(s) in the cache + regions = self.regions + offset = pos - self.cache_pos + start, end = regions[self.cache_index][0], regions[self.cache_index][3] + el, count = self._build_element(data, self.cache_index, offset) + self.increment_next_position(end, count) return el, start + offset, end + offset def handleMatch( # type: ignore[override] @@ -919,21 +924,7 @@ def handleMatch( # type: ignore[override] regions.sort(key=lambda x: x[0]) start, end = regions[0][0], regions[0][3] el, count = self._build_element(data) - - # Cache unprocessed regions to avoid repeated searches - if count < len(regions): - self.cache_index = count - self.cache_pos = self.regions[count][0] - while stack: - entry = stack.popleft() - if entry[0] > end: - if entry[0] < self.cache_pos: - self.cache_pos = entry[0] - break - else: - # Cleanup - self.reset() - + self.increment_next_position(end, count) return el, start, end # We failed to pair any valid start/end delimiters, avoid the parsed range next pass. From cc8419279a8c57653ce24a3d12e11995503a1aca Mon Sep 17 00:00:00 2001 From: facelessuser Date: Thu, 10 Sep 2026 09:28:19 -0600 Subject: [PATCH 10/12] Fix some edge cases - Fix an issue in element building where we should have exited - When an ambiguous `***` is partially consumed by a smaller start, the remainder should be a non-ambiguous start. - Don't consume an ambiguous opening with an end that is smaller than the opening. - `***` are always consumed, even when ambiguous. --- markdown/inlinepatterns.py | 56 ++++++++++++-------- tests/test_syntax/inline/test_emphasis.py | 62 +++++++++++++++++++++++ 2 files changed, 98 insertions(+), 20 deletions(-) diff --git a/markdown/inlinepatterns.py b/markdown/inlinepatterns.py index 93c49995b..8790359f3 100644 --- a/markdown/inlinepatterns.py +++ b/markdown/inlinepatterns.py @@ -595,7 +595,7 @@ def reset(self) -> None: # Cache info self.regions: list[tuple[int, int, int, int, int]] = [] - self.stack: deque[tuple[int, int, int]] = deque() + self.stack: deque[tuple[int, int, bool, int]] = deque() self.cache_index = 0 self.cache_pos = 0 @@ -619,7 +619,7 @@ def _build_patterns(self, token: str) -> str: ''', flags=re.UNICODE ) - elif self.double: + elif self.double: # pragma: no cover self.boundary = re.compile( fr'''(?x) (?P(? str: ''', flags=re.UNICODE ) - else: + else: # pragma: no cover # This case is not currently used self.boundary = re.compile( fr'''(?x) @@ -649,7 +649,7 @@ def _build_patterns(self, token: str) -> str: )''', flags=re.UNICODE ) - elif self.double: + elif self.double: # pragma: no cover self.boundary = re.compile( fr'''(?x) (?P(? str: ''', flags=re.UNICODE ) - else: + else: # pragma: no cover self.boundary = re.compile( fr'''(?x) (?P(? regions[start][3]: + if idx and r[0] >= regions[start][3]: idx -= 1 break # Get the appropriate element(s) @@ -785,7 +785,7 @@ def _build_element( return cast('etree.Element', el), idx - def increment_next_position(self, end: int, count: int) -> None: + def increment_next_position(self, start: int, count: int) -> None: """ Increment cache position to the next location that we can initiate an insertion. @@ -799,7 +799,7 @@ def increment_next_position(self, end: int, count: int) -> None: self.cache_pos = self.regions[self.cache_index][0] while self.stack: entry = self.stack.popleft() - if entry[0] > end: + if entry[0] > start: self.cache_pos = entry[0] break @@ -815,7 +815,7 @@ def get_cached_result(self, pos: int, data: str) -> tuple[etree.Element, int, in offset = pos - self.cache_pos start, end = regions[self.cache_index][0], regions[self.cache_index][3] el, count = self._build_element(data, self.cache_index, offset) - self.increment_next_position(end, count) + self.increment_next_position(start, count) return el, start + offset, end + offset def handleMatch( # type: ignore[override] @@ -849,7 +849,8 @@ def handleMatch( # type: ignore[override] # Data offset offset = m2.end(0) # Stack of opening delimiters - stack.append((m2.start(0), offset, len(m2.group(0)))) + is_ambiguous = m2.lastgroup[0] != 's' # type: ignore[index] + stack.append((m2.start(0), offset, is_ambiguous, len(m2.group(0)))) # Pair tokens until the stack is empty or we can no longer find tokens. while stack: @@ -865,7 +866,7 @@ def handleMatch( # type: ignore[override] # Some delimiters may be ambiguous and look like both a start or an end is_start = m2.lastgroup[0] != 'e' # type: ignore[index] is_end = not is_start or m2.lastgroup[0] != 's' # type: ignore[index] - ambiguous = is_start and is_end + is_ambiguous = is_start and is_end # Find closing tokens # Looking for: @@ -879,11 +880,12 @@ def handleMatch( # type: ignore[override] # Avoid ambiguous tokens that could be a start or an end. # Consume starts until the end token is fully consumed. # If we don't consume the entire end, see if next rule consumes it. - if is_end and ((not ambiguous and current > last) or (current == last)): + if is_end and ((not is_ambiguous and current > last) or current in (last, 3)): is_start = False # Consume previous tokens until the delimiter is consumed s = m2.start(0) + original = current while current and last <= current: delimiter = stack.pop() @@ -896,19 +898,33 @@ def handleMatch( # type: ignore[override] break last = stack[-1][-1] + # Should remainder be treated as a new start? + if original == 3 and current and is_ambiguous: + self.stack.append((m2.start(0) + regions[-1][-1], m2.end(0), False, current)) + is_end = False + # Do we still have more to consume? - is_end = current and stack and last > current + else: + is_end = current and stack and last > current # Looking for: # - `***em*` # - `***strong**` # - `**em*` - if is_end and (last == 3 or not ambiguous) and last > current: - is_start = False + if is_end and (last == 3 or not is_ambiguous) and last > current: delimiter = stack.pop() + + # Don't pair with an ambiguous opening + while stack and delimiter[-1] != 3 and delimiter[2]: + delimiter = stack.pop() + last = delimiter[-1] + if delimiter[2]: + break + + is_start = False new = last - current regions.append((delimiter[0] + new, delimiter[1], m2.start(0), offset, current)) - stack.append((delimiter[0], delimiter[0] + new, new)) + stack.append((delimiter[0], delimiter[0] + new, False, new)) # Find opening tokens if is_start: @@ -916,7 +932,7 @@ def handleMatch( # type: ignore[override] # - `*em ...*` # - `**strong ...*` # - `***em ...*` - stack.append((m2.start(0), m2.end(0), current)) + stack.append((m2.start(0), m2.end(0), is_ambiguous, current)) # Build the HTML elements if regions: @@ -924,7 +940,7 @@ def handleMatch( # type: ignore[override] regions.sort(key=lambda x: x[0]) start, end = regions[0][0], regions[0][3] el, count = self._build_element(data) - self.increment_next_position(end, count) + self.increment_next_position(start, count) return el, start, end # We failed to pair any valid start/end delimiters, avoid the parsed range next pass. diff --git a/tests/test_syntax/inline/test_emphasis.py b/tests/test_syntax/inline/test_emphasis.py index a7b3c56fa..9909b7790 100644 --- a/tests/test_syntax/inline/test_emphasis.py +++ b/tests/test_syntax/inline/test_emphasis.py @@ -222,3 +222,65 @@ def test_underscore_legacy(self): """ ).strip() ) + + def test_advanced_nesting(self): + + self.maxDiff = None + + self.assertMarkdownRenders( + textwrap.dedent( + """ + **a*bc** + + *a**b**c**d**e**f* + + ***a**b*cd**e*f*** + + ***a**b*cd*e**f*** + + ***a**b*cd*e**f*g*h*** + + ***a***bc**d*e*** + + *a**b**c**d**e**f* + + *a**b***c**d***e**f* + + *a**b***c**d***e**f** + + __a _b c__ + + _a __b __c __d __e __f_ + + ___a __b _c d__ e_ f___ + + ___a __b _c d_ e__ f___ + + ___a __b _c d_ e__ f _g_ h___ + + ___a ___b c__ d_ e___ + + _a __b__ _c __d__ _e __f__ + """ + ), + textwrap.dedent( + """ +

*abc

+

abcde**f

+

abcdef

+

abcdef

+

abcdefgh

+

abcde

+

abcde**f

+

abcde**f

+

abcd*ef

+

_a b c

+

_a __b __c __d __e _f

+

a b c d e f

+

a b c d e f

+

a b c d e f g h

+

a b c d e

+

_a b _c d _e f

+ """ + ).strip() + ) From abe63fb007a30055f997cdfcc72827855dbfb4c0 Mon Sep 17 00:00:00 2001 From: facelessuser Date: Thu, 10 Sep 2026 10:33:29 -0600 Subject: [PATCH 11/12] Separate cache info defines from reset --- markdown/inlinepatterns.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/markdown/inlinepatterns.py b/markdown/inlinepatterns.py index 8790359f3..ac5e2f577 100644 --- a/markdown/inlinepatterns.py +++ b/markdown/inlinepatterns.py @@ -583,6 +583,13 @@ def __init__( """ + # Cache info + self.regions: list[tuple[int, int, int, int, int]] = [] + self.stack: deque[tuple[int, int, bool, int]] = deque() + self.cache_index = 0 + self.cache_pos = 0 + self.cache_legacy_pos = -1 + self.last_run = 0.0 self.smart = smart self.tags = tags.split(',') @@ -593,9 +600,8 @@ def __init__( def reset(self) -> None: """Rest.""" - # Cache info - self.regions: list[tuple[int, int, int, int, int]] = [] - self.stack: deque[tuple[int, int, bool, int]] = deque() + self.regions.clear() + self.stack.clear() self.cache_index = 0 self.cache_pos = 0 From 08d6519bc3cec170e1522a528bbef99ede9e4ab4 Mon Sep 17 00:00:00 2001 From: facelessuser Date: Thu, 10 Sep 2026 10:34:52 -0600 Subject: [PATCH 12/12] Fix lint --- markdown/inlinepatterns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/markdown/inlinepatterns.py b/markdown/inlinepatterns.py index ac5e2f577..049bd4631 100644 --- a/markdown/inlinepatterns.py +++ b/markdown/inlinepatterns.py @@ -855,7 +855,7 @@ def handleMatch( # type: ignore[override] # Data offset offset = m2.end(0) # Stack of opening delimiters - is_ambiguous = m2.lastgroup[0] != 's' # type: ignore[index] + is_ambiguous = m2.lastgroup[0] != 's' # type: ignore[index] stack.append((m2.start(0), offset, is_ambiguous, len(m2.group(0)))) # Pair tokens until the stack is empty or we can no longer find tokens. @@ -922,7 +922,7 @@ def handleMatch( # type: ignore[override] # Don't pair with an ambiguous opening while stack and delimiter[-1] != 3 and delimiter[2]: - delimiter = stack.pop() + delimiter = stack.pop() last = delimiter[-1] if delimiter[2]: break