From 0a2cc271b533a19b14c8b05981df4983dfdc8446 Mon Sep 17 00:00:00 2001 From: Danny Tuppeny Date: Tue, 28 Jul 2026 15:19:47 +0100 Subject: [PATCH 1/2] Fix the patterns for code blocks and the handling of newlines when matching patterns TextMate grammars are matched by-line and therefore can never match `\n`. However our Dart implementation allowed this, which resulted in some bugs when code blocks are not closed. This updates the grammar to instead use $ to match the end of the line, and updates the parser to only ever match on the current line and not consumer the newline. With this change, the result matches the output of the TypeScript implementation used for testing in Dart-Code. The source PR for this version of the grammar is at https://github.com/dart-lang/dart-syntax-highlight/pull/91 (I'm filing these together because one repo owns the syntax and one owns the parser). See Dart-Code/Dart-Code#6130 See Dart-Code/Dart-Code#6131 --- packages/devtools_app/assets/dart_syntax.json | 6 +-- .../lib/src/screens/debugger/span_parser.dart | 54 +++++++++++++++---- .../open_code_block.golden | 3 +- 3 files changed, 48 insertions(+), 15 deletions(-) diff --git a/packages/devtools_app/assets/dart_syntax.json b/packages/devtools_app/assets/dart_syntax.json index b73e9bd36d1..f05d62d8ed6 100644 --- a/packages/devtools_app/assets/dart_syntax.json +++ b/packages/devtools_app/assets/dart_syntax.json @@ -1,6 +1,6 @@ { "name": "Dart", - "version": "1.5.0", + "version": "1.6.0", "fileTypes": [ "dart" ], @@ -77,12 +77,12 @@ "repository": { "dartdoc-codeblock-triple": { "begin": "^\\s*///\\s*(?!\\s*```)", - "end": "\n", + "end": "$", "contentName": "variable.other.source.dart" }, "dartdoc-codeblock-block": { "begin": "^\\s*\\*\\s*(?!(\\s*```|\/))", - "end": "\n", + "end": "$", "contentName": "variable.other.source.dart" }, "dartdoc": { diff --git a/packages/devtools_app/lib/src/screens/debugger/span_parser.dart b/packages/devtools_app/lib/src/screens/debugger/span_parser.dart index 74cbe0137ad..5358a35cb0f 100644 --- a/packages/devtools_app/lib/src/screens/debugger/span_parser.dart +++ b/packages/devtools_app/lib/src/screens/debugger/span_parser.dart @@ -127,13 +127,13 @@ class ScopeSpan { ); while (!splitScanner.isDone) { - if (splitScanner.matches(cond)) { + if (splitScanner.matchesOnCurrentLine(cond)) { // Update the end position for this span as it's been fully processed. current._endLocation = splitScanner.location; splitSpans.add(current); // Move the scanner position past the matched condition. - splitScanner.scan(cond); + splitScanner.scanOnCurrentLine(cond); // Create a new span based on the current position. current = ScopeSpan( @@ -273,7 +273,7 @@ class _SimpleMatcher extends GrammarMatcher { @override bool scan(Grammar grammar, LineScanner scanner, ScopeStack scopeStack) { final location = scanner.location; - if (scanner.scan(match)) { + if (scanner.scanOnCurrentLine(match)) { scopeStack.push(name, location); _applyCapture(grammar, scanner, scopeStack, captures, location); scopeStack.pop(name, scanner.location); @@ -307,7 +307,7 @@ class _MultilineMatcher extends GrammarMatcher { : RegExp(json['while'] as String, multiLine: true), patterns = (json['patterns'] as List?) ?.cast>() - .map((e) => GrammarMatcher.fromJson(e)) + .map(GrammarMatcher.fromJson) .toList(), super._(); @@ -361,7 +361,7 @@ class _MultilineMatcher extends GrammarMatcher { void _scanBegin(Grammar grammar, LineScanner scanner, ScopeStack scopeStack) { final location = scanner.location; - if (!scanner.scan(begin)) { + if (!scanner.scanOnCurrentLine(begin)) { // This shouldn't happen since we've already checked that `begin` matches // the beginning of the string. throw StateError('Expected ${begin.pattern} to match.'); @@ -403,7 +403,9 @@ class _MultilineMatcher extends GrammarMatcher { LineScanner scanner, ScopeStack scopeStack, ) { - while (!scanner.isDone && end != null && !scanner.matches(end!)) { + while (!scanner.isDone && + end != null && + !scanner.matchesOnCurrentLine(end!)) { bool foundMatch = false; for (final pattern in patterns ?? []) { if (pattern.scan(grammar, scanner, scopeStack)) { @@ -420,7 +422,7 @@ class _MultilineMatcher extends GrammarMatcher { void _scanEnd(Grammar grammar, LineScanner scanner, ScopeStack scopeStack) { final location = scanner.location; - if (end != null && !scanner.scan(end!)) { + if (end != null && !scanner.scanOnCurrentLine(end!)) { return; } _processCaptureHelper(grammar, scanner, scopeStack, endCaptures, location); @@ -446,7 +448,7 @@ class _MultilineMatcher extends GrammarMatcher { @override bool scan(Grammar grammar, LineScanner scanner, ScopeStack scopeStack) { - if (!scanner.matches(begin)) { + if (!scanner.matchesOnCurrentLine(begin)) { return false; } @@ -461,7 +463,9 @@ class _MultilineMatcher extends GrammarMatcher { // Find the range of the string that is matched by the while condition. final start = scanner.position; _skipLine(scanner); - while (!scanner.isDone && whileCond != null && scanner.scan(whileCond!)) { + while (!scanner.isDone && + whileCond != null && + scanner.scanOnCurrentLine(whileCond!)) { _skipLine(scanner); } final end = scanner.position; @@ -482,7 +486,7 @@ class _MultilineMatcher extends GrammarMatcher { // Process each line until the `while` condition fails. while (!contentScanner.isDone && whileCond != null && - contentScanner.scan(whileCond!)) { + contentScanner.scanOnCurrentLine(whileCond!)) { _scanToEndOfLine(grammar, contentScanner, scopeStack); } @@ -520,7 +524,7 @@ class _PatternMatcher extends GrammarMatcher { _PatternMatcher(super.json) : patterns = (json['patterns'] as List?) ?.cast>() - .map((e) => GrammarMatcher.fromJson(e)) + .map(GrammarMatcher.fromJson) .toList(), super._(); @@ -758,4 +762,32 @@ class ScopeStackLocation { extension LineScannerExtension on LineScanner { ScopeStackLocation get location => ScopeStackLocation(position: position, line: line, column: column); + + /// Returns whether [pattern] matches at the scanner's current position + /// without consuming a line break. + /// + /// Use this for all TextMate grammar regular expressions. The TextMate + /// [Language Rules documentation](https://macromates.com/manual/en/language_grammars#language_rules) + /// specifies that regular expressions are matched against one document line + /// at a time, while [LineScanner] searches the remaining source text. Without + /// this check, a pattern containing whitespace such as `\s*` can consume text + /// from the next line and produce scopes that TextMate would not. + bool matchesOnCurrentLine(RegExp pattern) { + if (!matches(pattern)) return false; + + final match = lastMatch![0]!; + return !match.contains('\n') && !match.contains('\r'); + } + + /// Scans [pattern] only when it matches without consuming a line break. + /// + /// Use this instead of [LineScanner.scan] when consuming a TextMate grammar + /// regular expression. See [matchesOnCurrentLine] for the rule requiring + /// line-local matching. Use the scanner directly only for parser operations + /// that intentionally move across lines, such as skipping a physical line. + bool scanOnCurrentLine(RegExp pattern) { + if (!matchesOnCurrentLine(pattern)) return false; + + return scan(pattern); + } } diff --git a/packages/devtools_app/test/test_infra/goldens/syntax_highlighting/open_code_block.golden b/packages/devtools_app/test/test_infra/goldens/syntax_highlighting/open_code_block.golden index 83962b341ec..0c2faa00eb8 100644 --- a/packages/devtools_app/test/test_infra/goldens/syntax_highlighting/open_code_block.golden +++ b/packages/devtools_app/test/test_infra/goldens/syntax_highlighting/open_code_block.golden @@ -5,7 +5,8 @@ >/// #^^^ comment.block.documentation.dart >/// This should not cause parsing to fail. -#^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ comment.block.documentation.dart variable.other.source.dart +#^^^^ comment.block.documentation.dart +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ comment.block.documentation.dart variable.other.source.dart > >void main() { #^^^^ storage.type.primitive.dart From b2dd71d4ae19d4203b3776c10a9ce73e7d5dac8f Mon Sep 17 00:00:00 2001 From: Danny Tuppeny Date: Tue, 28 Jul 2026 17:35:05 +0100 Subject: [PATCH 2/2] Update NEXT_RELEASE_NOTES.md --- packages/devtools_app/release_notes/NEXT_RELEASE_NOTES.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/devtools_app/release_notes/NEXT_RELEASE_NOTES.md b/packages/devtools_app/release_notes/NEXT_RELEASE_NOTES.md index aa9373543f5..2d797c8109e 100644 --- a/packages/devtools_app/release_notes/NEXT_RELEASE_NOTES.md +++ b/packages/devtools_app/release_notes/NEXT_RELEASE_NOTES.md @@ -41,6 +41,9 @@ TODO: Remove this section if there are not any updates. [#9885](https://github.com/flutter/devtools/pull/9885) * Update to latest version of the Dart syntax highlighting grammar [#9920](https://github.com/flutter/devtools/pull/9920). +* Fix a bug in the TextMate grammar parser that could result in code after + comments being classified as comments. + [#9921](https://github.com/flutter/devtools/pull/9921). ## Network profiler updates