From 5d16161b3cf6c51acdd4155d5992d00f19242045 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 11 Oct 2026 15:07:16 -0500 Subject: [PATCH 1/2] fix(sentence): leave an unclosed double backtick literal A `` with no '' after it held a LaTeX quote open to the end of the paragraph. Every later sentence stayed on its line. CommonMark reads an unmatched backtick run as literal text. The delimiter walk now counts the '' closers ahead and opens a quote only when one is left to close it. A closed ``quote'' still keeps its sentences together. --- src/sentence/unicode.rs | 33 ++++++++++++++++-- tests/integration.rs | 2 ++ tests/md_unclosed_double_backtick.rs | 50 ++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 3 deletions(-) create mode 100644 tests/md_unclosed_double_backtick.rs diff --git a/src/sentence/unicode.rs b/src/sentence/unicode.rs index 8ad73d69..06c06dda 100644 --- a/src/sentence/unicode.rs +++ b/src/sentence/unicode.rs @@ -3873,7 +3873,7 @@ fn glue_tick_capital_after_punct(result: &mut Vec, segment: &str) -> boo fn merge_splits_inside_delimiters(segments: Vec) -> Vec { let mut result: Vec = Vec::with_capacity(segments.len()); - let mut state = DelimState::default(); + let mut state = DelimState::for_text(&segments.concat()); for segment in segments { if state.is_inside() { @@ -3979,9 +3979,28 @@ pub struct DelimState { last_char: Option, /// When the previous chunk ended in `\`, the next `"` / `'` is escaped. pending_escape: bool, + /// LaTeX `''` closers still ahead in the text, when known. + /// A ```` with no `''` left to close it is literal, as an unmatched + /// backtick run is in CommonMark, so it opens no span. + latex_closers_ahead: Option, } impl DelimState { + /// A state that knows how many `''` closers `text` holds, so a + /// ```` with none left after it stays literal. + pub fn for_text(text: &str) -> Self { + let mut probe = DelimState { + latex_closers_ahead: Some(0), + ..DelimState::default() + }; + probe.feed(text); + let closers = -probe.latex_closers_ahead.unwrap_or(0); + DelimState { + latex_closers_ahead: Some(closers), + ..DelimState::default() + } + } + pub fn is_inside(&self) -> bool { self.ascii_double_open || self.ascii_single_open @@ -4023,12 +4042,20 @@ impl DelimState { self.last_char = Some('`'); continue; } - self.latex_quote_depth += 1; + if self + .latex_closers_ahead + .is_none_or(|ahead| ahead > self.latex_quote_depth) + { + self.latex_quote_depth += 1; + } self.last_char = Some('`'); continue; } if ch == '\'' && next == Some('\'') { let _ = iter.next(); + if let Some(ahead) = self.latex_closers_ahead.as_mut() { + *ahead -= 1; + } self.latex_quote_depth = (self.latex_quote_depth - 1).max(0); self.last_char = Some('\''); continue; @@ -4124,7 +4151,7 @@ pub fn newlines_respect_delimiter_spans(formatted: &str) -> bool { return true; } let (protected, _) = protect_inline_tokens(trimmed_end); - let mut state = DelimState::default(); + let mut state = DelimState::for_text(&protected); for line in protected.split('\n') { if state.is_inside() { return false; diff --git a/tests/integration.rs b/tests/integration.rs index 3c89e988..48be1a06 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -108,6 +108,8 @@ mod md_ref_link_interior_punct; mod md_shortcut_ref_label; #[path = "md_spanning_lrd_title.rs"] mod md_spanning_lrd_title; +#[path = "md_unclosed_double_backtick.rs"] +mod md_unclosed_double_backtick; #[path = "md_yaml_front_matter.rs"] mod md_yaml_front_matter; #[path = "org_angle_links.rs"] diff --git a/tests/md_unclosed_double_backtick.rs b/tests/md_unclosed_double_backtick.rs new file mode 100644 index 00000000..f65bb1d3 --- /dev/null +++ b/tests/md_unclosed_double_backtick.rs @@ -0,0 +1,50 @@ +//! A ```` with no `''` after it is literal, as an unmatched backtick run +//! is in CommonMark. It must not hold the rest of the paragraph on one line. +//! A closed LaTeX quote still keeps its sentences together. + +use snapper_fmt::format::Format; +use snapper_fmt::{FormatConfig, format_text}; + +fn cfg(format: Format) -> FormatConfig { + FormatConfig { + format, + max_width: 0, + ..Default::default() + } + .without_safety_backstops() +} + +#[test] +fn unclosed_double_backtick_does_not_stop_splits() { + let input = "Unclosed `` tick here. Next sentence. Third one.\n"; + let out = format_text(input, &cfg(Format::Markdown)).unwrap(); + assert_eq!( + out, "Unclosed `` tick here.\nNext sentence.\nThird one.\n", + "an unmatched `` must not open a span, got:\n{out}" + ); + assert_eq!(format_text(&out, &cfg(Format::Markdown)).unwrap(), out); +} + +#[test] +fn unclosed_double_backtick_after_a_closed_quote_still_splits() { + let input = "He said ``Stop. Now.'' and left. Then `` stray. Next one.\n"; + let out = format_text(input, &cfg(Format::Plaintext)).unwrap(); + assert!( + out.contains("``Stop. Now.''"), + "a closed LaTeX quote must stay one span, got:\n{out}" + ); + assert!( + out.contains("stray.\nNext one.\n"), + "the unmatched `` after it must not open a span, got:\n{out}" + ); +} + +#[test] +fn closed_latex_quote_stays_whole() { + let input = "He said ``Stop. Now.'' and left. Next sentence.\n"; + let out = format_text(input, &cfg(Format::Latex)).unwrap(); + assert!( + out.contains("``Stop. Now.'' and left.\nNext sentence.\n"), + "a closed LaTeX quote must stay one span, got:\n{out}" + ); +} From fca6264998c6f688e86b468ce2b49dc1d31faa16 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 11 Oct 2026 15:07:16 -0500 Subject: [PATCH 2/2] fix(markdown): keep a pipe-less line under a table as a row A GFM table runs to a blank line or to a block that interrupts a paragraph (GFM ex. 202). A line under the last row with no pipe is one more row. The parser read it as prose and split it, which added a row. The CLI render check then reverted that line with a warning. The table now takes such lines, as pulldown-cmark does. A list item, heading, fence, quote, HTML block or footnote ends it. A line indented less than the header ends it too, since it may belong to an outer container. The light-table test expected the split, so it now checks the row and a later paragraph. --- src/parser/markdown.rs | 34 ++++++++++++++-- tests/integration.rs | 2 + tests/md_gfm_table_lazy_rows.rs | 67 +++++++++++++++++++++++++++++++ tests/md_light_table_interrupt.rs | 11 ++++- 4 files changed, 109 insertions(+), 5 deletions(-) create mode 100644 tests/md_gfm_table_lazy_rows.rs diff --git a/src/parser/markdown.rs b/src/parser/markdown.rs index b334b277..210bf1e0 100644 --- a/src/parser/markdown.rs +++ b/src/parser/markdown.rs @@ -1342,9 +1342,10 @@ fn list_remainder_raw_end(lines: &[Line<'_>], start: usize, rest: &str) -> Optio if header.len() != dcells.len() || !dcells.iter().all(|c| is_gfm_delimiter_cell(c)) { return None; } + let header_indent = line_indent(delim.text); let mut end = start + 1; for (j, line) in lines.iter().enumerate().skip(start + 2) { - if !is_gfm_table_row(line.text) { + if !continues_gfm_table(line.text, header_indent) { break; } end = j; @@ -1490,7 +1491,7 @@ fn quoted_gfm_table_end(lines: &[Line<'_>], start: usize, depth: usize) -> Optio let Some(inner) = strip_quote_markers(line.text, depth) else { break; }; - if !is_gfm_table_row(inner) { + if !continues_gfm_table(inner, line_indent(header_inner)) { break; } end = j; @@ -1741,6 +1742,32 @@ fn is_gfm_table_row(line: &str) -> bool { gfm_table_cells(line).is_some() } +/// A line that continues a GFM table body although it has no pipe. +/// The table runs to a blank line or a block that interrupts a paragraph +/// (GFM ex. 202; pulldown `parse_table_row`), so `bar` under the last row +/// is one more row, not prose. +/// A line indented less than the header may belong to an outer container, +/// so it ends the table here and stays prose. +fn is_gfm_lazy_table_row(line: &str, min_indent: usize) -> bool { + let content = line.trim_start(); + if content.is_empty() || line_indent(line) < min_indent { + return false; + } + !(is_thematic_break(content) + || HEADING_RE.is_match(content) + || FENCED_CODE_RE.is_match(content) + || content.starts_with('>') + || LIST_ITEM_RE.is_match(content) + || html_block_kind(content).is_some_and(HtmlBlock::can_interrupt) + || is_footnote_definition(content)) +} + +/// True when `line` is the next row of a GFM table whose header is indented +/// by `header_indent`. +fn continues_gfm_table(line: &str, header_indent: usize) -> bool { + is_gfm_table_row(line) || is_gfm_lazy_table_row(line, header_indent) +} + /// Pandoc line block: a line that starts with `|` and a space, and does /// not end with a pipe. A flanking-pipe row is a table ([`TABLE_ROW_RE`]), /// not a line block. `|cell` (no space) is prose. @@ -1765,9 +1792,10 @@ fn gfm_table_end(lines: &[Line<'_>], start: usize) -> Option { if header.len() != dcells.len() || !dcells.iter().all(|c| is_gfm_delimiter_cell(c)) { return None; } + let header_indent = line_indent(lines[start].text); let mut end = start + 1; for (j, line) in lines.iter().enumerate().skip(start + 2) { - if !is_gfm_table_row(line.text) { + if !continues_gfm_table(line.text, header_indent) { break; } end = j; diff --git a/tests/integration.rs b/tests/integration.rs index 48be1a06..ade1b756 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -54,6 +54,8 @@ mod md_example_list; mod md_four_space_nested_list; #[path = "md_gfm_alerts.rs"] mod md_gfm_alerts; +#[path = "md_gfm_table_lazy_rows.rs"] +mod md_gfm_table_lazy_rows; #[path = "md_html_comment_indent.rs"] mod md_html_comment_indent; #[path = "md_html_type6_close.rs"] diff --git a/tests/md_gfm_table_lazy_rows.rs b/tests/md_gfm_table_lazy_rows.rs new file mode 100644 index 00000000..8a97bbbe --- /dev/null +++ b/tests/md_gfm_table_lazy_rows.rs @@ -0,0 +1,67 @@ +//! A GFM table runs to a blank line or a block that interrupts a paragraph +//! (GFM ex. 202). A line under the last row with no pipe is one more row, +//! so splitting it would add a row. A list, heading or outer line after the +//! table still splits. + +use snapper_fmt::format::Format; +use snapper_fmt::{FormatConfig, format_text}; + +fn md_cfg() -> FormatConfig { + FormatConfig { + format: Format::Markdown, + max_width: 0, + ..Default::default() + } + .without_safety_backstops() +} + +fn fmt(input: &str) -> String { + let out = format_text(input, &md_cfg()).unwrap(); + assert_eq!(format_text(&out, &md_cfg()).unwrap(), out, "not idempotent"); + out +} + +#[test] +fn line_under_the_last_row_stays_a_row() { + let input = "| a | b |\n|---|---|\n| x | y |\nAfter table. Next.\n\nOne here. Two here.\n"; + let out = fmt(input); + assert!( + out.contains("| x | y |\nAfter table. Next.\n"), + "a pipe-less line under the table is a row, got:\n{out}" + ); + assert!( + out.contains("One here.\nTwo here.\n"), + "the paragraph after the blank line must still split, got:\n{out}" + ); +} + +#[test] +fn list_or_heading_after_the_table_still_splits() { + let input = "| a | b |\n|---|---|\n| x | y |\n- Item one. Item two.\n\n\ + | a | b |\n|---|---|\n| x | y |\n# Head\nPara one. Para two.\n"; + let out = fmt(input); + assert!( + out.contains("- Item one.\n Item two.\n"), + "a list item ends the table, got:\n{out}" + ); + assert!( + out.contains("# Head\nPara one.\nPara two.\n"), + "a heading ends the table, got:\n{out}" + ); +} + +#[test] +fn quoted_table_keeps_its_lazy_row() { + let input = "> | q | w |\n> |---|---|\n> | e | t |\n> lazy quoted. Row too.\n"; + assert_eq!(fmt(input), input); +} + +#[test] +fn outer_line_after_a_list_item_table_still_splits() { + let input = "- item\n\n | k | v |\n |---|---|\n | p | r |\nOuter line. Second one.\n"; + let out = fmt(input); + assert!( + out.contains(" | p | r |\nOuter line.\nSecond one.\n"), + "a line outside the list item is not a row, got:\n{out}" + ); +} diff --git a/tests/md_light_table_interrupt.rs b/tests/md_light_table_interrupt.rs index 2ae43b38..2ffda5f5 100644 --- a/tests/md_light_table_interrupt.rs +++ b/tests/md_light_table_interrupt.rs @@ -69,6 +69,8 @@ fn heavy_table_still_interrupts() { "| alpha. | bravo. |\n", "| --- | --- |\n", "After. Next.\n", + "\n", + "One more. Last one.\n", ); let regions = MarkdownParser.parse(input); assert!( @@ -79,8 +81,13 @@ fn heavy_table_still_interrupts() { "heavy table must still interrupt, got {regions:?}" ); let out = format_text(input, &md_cfg()).unwrap(); + // GFM ex. 202: the line under the delimiter row is a body row. assert!( - out.contains("After.\nNext."), - "following prose must still split, got:\n{out}" + out.contains("| --- | --- |\nAfter. Next.\n"), + "the pipe-less line under the table is a row, got:\n{out}" + ); + assert!( + out.contains("One more.\nLast one."), + "prose after the blank line must still split, got:\n{out}" ); }