From d40c0c86543d14e00fc00892fd1554d4c0e2d3f6 Mon Sep 17 00:00:00 2001 From: Vladyslav Kuksiuk Date: Mon, 6 Jul 2026 17:51:08 +0200 Subject: [PATCH 1/4] Reduce code duplication. --- embedding/commentfilter/csharp_filter.go | 12 +- embedding/commentfilter/filter.go | 17 +++ embedding/commentfilter/javascript_filter.go | 12 +- embedding/commentfilter/kotlin_filter.go | 12 +- .../commentfilter/marker_comment_filter.go | 121 ++++++++---------- .../commentfilter/visual_basic_filter.go | 13 +- 6 files changed, 83 insertions(+), 104 deletions(-) diff --git a/embedding/commentfilter/csharp_filter.go b/embedding/commentfilter/csharp_filter.go index 0ec705c..7a56d24 100644 --- a/embedding/commentfilter/csharp_filter.go +++ b/embedding/commentfilter/csharp_filter.go @@ -86,17 +86,11 @@ type csharpLineFilter struct { // // Returns filtered source lines. func (CSharpCommentFilter) Filter(lines []string, mode Mode) []string { - var filtered []string state := csharpState{} - for _, line := range lines { - filteredLine, hadComment := filterCSharpLine(line, mode, &state) - if hadComment && strings.TrimSpace(filteredLine) == "" { - continue - } - filtered = append(filtered, filteredLine) - } - return filtered + return filterLines(lines, func(line string) (string, bool) { + return filterCSharpLine(line, mode, &state) + }) } // filterCSharpLine removes or preserves recognized C# comments from one line. diff --git a/embedding/commentfilter/filter.go b/embedding/commentfilter/filter.go index b29eca4..e9f72f6 100644 --- a/embedding/commentfilter/filter.go +++ b/embedding/commentfilter/filter.go @@ -50,6 +50,23 @@ type CommentFilter interface { Filter(lines []string, mode Mode) []string } +// filterLines applies a line filter and drops lines made empty by comment removal. +func filterLines( + lines []string, + filterLine func(line string) (filteredLine string, hadComment bool), +) []string { + var filtered []string + for _, line := range lines { + filteredLine, hadComment := filterLine(line) + if hadComment && strings.TrimSpace(filteredLine) == "" { + continue + } + filtered = append(filtered, filteredLine) + } + + return filtered +} + // Filter returns source lines with comments stripped according to the requested mode. // // Parameters: diff --git a/embedding/commentfilter/javascript_filter.go b/embedding/commentfilter/javascript_filter.go index 57332d2..0c0ac04 100644 --- a/embedding/commentfilter/javascript_filter.go +++ b/embedding/commentfilter/javascript_filter.go @@ -93,17 +93,11 @@ type interpolationCodeResult struct { // // Returns filtered source lines. func (JavaScriptCommentFilter) Filter(lines []string, mode Mode) []string { - var filtered []string state := javascriptState{} - for _, line := range lines { - filteredLine, hadComment := filterJavaScriptLine(line, mode, &state) - if hadComment && strings.TrimSpace(filteredLine) == "" { - continue - } - filtered = append(filtered, filteredLine) - } - return filtered + return filterLines(lines, func(line string) (string, bool) { + return filterJavaScriptLine(line, mode, &state) + }) } // filterJavaScriptLine removes or preserves recognized JavaScript comments from one line. diff --git a/embedding/commentfilter/kotlin_filter.go b/embedding/commentfilter/kotlin_filter.go index 4990306..afab1d6 100644 --- a/embedding/commentfilter/kotlin_filter.go +++ b/embedding/commentfilter/kotlin_filter.go @@ -69,17 +69,11 @@ type kotlinLineFilter struct { // // Returns filtered source lines. func (KotlinCommentFilter) Filter(lines []string, mode Mode) []string { - var filtered []string state := kotlinState{} - for _, line := range lines { - filteredLine, hadComment := filterKotlinLine(line, mode, &state) - if hadComment && strings.TrimSpace(filteredLine) == "" { - continue - } - filtered = append(filtered, filteredLine) - } - return filtered + return filterLines(lines, func(line string) (string, bool) { + return filterKotlinLine(line, mode, &state) + }) } // filterKotlinLine removes or preserves recognized Kotlin comments from one line. diff --git a/embedding/commentfilter/marker_comment_filter.go b/embedding/commentfilter/marker_comment_filter.go index d245331..b99b79f 100644 --- a/embedding/commentfilter/marker_comment_filter.go +++ b/embedding/commentfilter/marker_comment_filter.go @@ -47,6 +47,21 @@ type TextBlockMarker struct { Escapes bool } +// activeSegment describes a multi-line construct currently being scanned. +type activeSegment struct { + // end closes the active construct. + end string + + // keep reports whether the active construct is copied to output. + keep bool + + // escapes reports whether backslashes escape bytes while searching for end. + escapes bool + + // comment reports whether the active construct is a source comment. + comment bool +} + // CommentMarker describes lexical comment markers and string delimiters for a language family. type CommentMarker struct { // Inline contains line-comment markers. @@ -71,22 +86,13 @@ type MarkerCommentFilter struct { Syntax CommentMarker } -// blockState tracks active multi-line lexical constructs across source lines. -type blockState struct { - // active reports whether scanning is inside a block comment. +// markerState tracks active multi-line lexical constructs across source lines. +type markerState struct { + // active reports whether scanning is inside a multi-line construct. active bool - // block contains the active block comment markers. - block BlockMarker - - // keep reports whether the active comment should be retained. - keep bool - - // textBlockActive reports whether scanning is inside a text block. - textBlockActive bool - - // textBlock contains the active text block marker. - textBlock TextBlockMarker + // segment contains the active construct configuration. + segment activeSegment } // markerLineFilter tracks lexical comment filtering state for one source line. @@ -101,7 +107,7 @@ type markerLineFilter struct { mode Mode // state tracks multi-line lexical constructs across lines. - state *blockState + state *markerState // result accumulates the filtered source line. result strings.Builder @@ -121,24 +127,18 @@ type markerLineFilter struct { // // Returns filtered source lines. func (f MarkerCommentFilter) Filter(lines []string, mode Mode) []string { - var filtered []string - state := blockState{} - for _, line := range lines { - filteredLine, hadComment := f.filterLine(line, mode, &state) - if hadComment && strings.TrimSpace(filteredLine) == "" { - continue - } - filtered = append(filtered, filteredLine) - } + state := markerState{} - return filtered + return filterLines(lines, func(line string) (string, bool) { + return f.filterLine(line, mode, &state) + }) } // filterLine removes or preserves recognized comments from a single source line. func (f MarkerCommentFilter) filterLine( line string, mode Mode, - state *blockState, + state *markerState, ) (string, bool) { filter := markerLineFilter{ filter: f, @@ -153,10 +153,7 @@ func (f MarkerCommentFilter) filterLine( // filterLine walks the current line until it reaches its end or a line comment. func (f *markerLineFilter) filterLine() (string, bool) { for f.position < len(f.line) { - if f.consumeActiveBlock() { - continue - } - if f.consumeActiveTextBlock() { + if f.consumeActiveSegment() { continue } if f.consumeTextBlockStart() { @@ -178,61 +175,44 @@ func (f *markerLineFilter) filterLine() (string, bool) { return f.result.String(), f.hadComment } -// consumeActiveBlock consumes text while the scanner is inside a block comment. -func (f *markerLineFilter) consumeActiveBlock() bool { +// consumeActiveSegment consumes text while the scanner is inside a multi-line construct. +func (f *markerLineFilter) consumeActiveSegment() bool { if !f.state.active { return false } - f.hadComment = true - end := strings.Index(f.line[f.position:], f.state.block.End) - if end < 0 { - if f.state.keep { + segment := f.state.segment + if segment.comment { + f.hadComment = true + } + endPosition, found := segmentEnd(f.line, f.position, segment) + if !found { + if segment.keep { f.result.WriteString(f.line[f.position:]) } f.position = len(f.line) return true } - endPosition := f.position + end + len(f.state.block.End) - if f.state.keep { + if segment.keep { f.result.WriteString(f.line[f.position:endPosition]) } f.position = endPosition f.state.active = false + f.state.segment = activeSegment{} return true } -// consumeActiveTextBlock copies text block content until the closing delimiter. -func (f *markerLineFilter) consumeActiveTextBlock() bool { - if !f.state.textBlockActive { - return false - } - endPosition, found := textBlockEnd(f.line, f.position, f.state.textBlock) - if !found { - f.result.WriteString(f.line[f.position:]) - f.position = len(f.line) - - return true - } - f.result.WriteString(f.line[f.position:endPosition]) - f.position = endPosition - f.state.textBlockActive = false - f.state.textBlock = TextBlockMarker{} - - return true -} - -// textBlockEnd returns the end offset of a text block close delimiter. -func textBlockEnd(line string, position int, marker TextBlockMarker) (int, bool) { +// segmentEnd returns the end offset of an active segment close delimiter. +func segmentEnd(line string, position int, segment activeSegment) (int, bool) { for cursor := position; cursor < len(line); { - if marker.Escapes && line[cursor] == '\\' { + if segment.escapes && line[cursor] == '\\' { cursor += 2 continue } - if strings.HasPrefix(line[cursor:], marker.Delimiter) { - return cursor + len(marker.Delimiter), true + if strings.HasPrefix(line[cursor:], segment.end) { + return cursor + len(segment.end), true } cursor++ } @@ -248,8 +228,12 @@ func (f *markerLineFilter) consumeTextBlockStart() bool { } f.result.WriteString(marker.Delimiter) f.position += len(marker.Delimiter) - f.state.textBlockActive = true - f.state.textBlock = marker + f.state.active = true + f.state.segment = activeSegment{ + end: marker.Delimiter, + keep: true, + escapes: marker.Escapes, + } return true } @@ -327,8 +311,11 @@ func (f *markerLineFilter) consumeInlineComment(keep bool) { func (f *markerLineFilter) startBlockComment(block BlockMarker, keep bool) { f.hadComment = true f.state.active = true - f.state.block = block - f.state.keep = keep + f.state.segment = activeSegment{ + end: block.End, + keep: keep, + comment: true, + } } // consumeCodeByte copies one source byte that does not belong to a recognized comment. diff --git a/embedding/commentfilter/visual_basic_filter.go b/embedding/commentfilter/visual_basic_filter.go index 5478692..251add0 100644 --- a/embedding/commentfilter/visual_basic_filter.go +++ b/embedding/commentfilter/visual_basic_filter.go @@ -43,16 +43,9 @@ type VisualBasicCommentFilter struct{} // // Returns filtered source lines. func (VisualBasicCommentFilter) Filter(lines []string, mode Mode) []string { - var filtered []string - for _, line := range lines { - filteredLine, hadComment := filterVisualBasicLine(line, mode) - if hadComment && strings.TrimSpace(filteredLine) == "" { - continue - } - filtered = append(filtered, filteredLine) - } - - return filtered + return filterLines(lines, func(line string) (string, bool) { + return filterVisualBasicLine(line, mode) + }) } // filterVisualBasicLine removes or preserves one Visual Basic comment. From e3130399e8847516b68828a09dfbcef2e72faa97 Mon Sep 17 00:00:00 2001 From: Vladyslav Kuksiuk Date: Mon, 6 Jul 2026 17:58:05 +0200 Subject: [PATCH 2/4] Extract types. --- embedding/commentfilter/comment_result.go | 28 +++++ embedding/commentfilter/config.go | 1 - embedding/commentfilter/javascript_filter.go | 9 -- embedding/commentfilter/kotlin_filter.go | 20 ++-- .../commentfilter/marker_comment_filter.go | 102 ++++-------------- embedding/commentfilter/marker_syntax.go | 70 ++++++++++++ 6 files changed, 127 insertions(+), 103 deletions(-) create mode 100644 embedding/commentfilter/comment_result.go create mode 100644 embedding/commentfilter/marker_syntax.go diff --git a/embedding/commentfilter/comment_result.go b/embedding/commentfilter/comment_result.go new file mode 100644 index 0000000..6506284 --- /dev/null +++ b/embedding/commentfilter/comment_result.go @@ -0,0 +1,28 @@ +// Copyright 2026, TeamDev. All rights reserved. +// +// Redistribution and use in source and/or binary forms, with or without +// modification, must retain the above copyright notice and the following +// disclaimer. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package commentfilter + +// commentConsumeResult describes a consumed source comment. +type commentConsumeResult struct { + // consumed reports whether a recognized comment marker was consumed. + consumed bool + + // stopLine reports whether the consumed comment reaches the end of the source line. + stopLine bool +} diff --git a/embedding/commentfilter/config.go b/embedding/commentfilter/config.go index 906f4c5..bab8e92 100644 --- a/embedding/commentfilter/config.go +++ b/embedding/commentfilter/config.go @@ -27,7 +27,6 @@ const ( pythonSingleQuoteBlock = "'''" goRawStringDelimiter = "`" goQuoteChars = "\"'" - jsQuoteChars = "\"'`" ) // filtersByExtension is a mapping of the file extension to its comment filter. diff --git a/embedding/commentfilter/javascript_filter.go b/embedding/commentfilter/javascript_filter.go index 0c0ac04..4038f7c 100644 --- a/embedding/commentfilter/javascript_filter.go +++ b/embedding/commentfilter/javascript_filter.go @@ -67,15 +67,6 @@ type javascriptLineFilter struct { hadComment bool } -// commentConsumeResult describes a consumed JavaScript comment. -type commentConsumeResult struct { - // consumed reports whether a recognized comment marker was consumed. - consumed bool - - // stopLine reports whether the consumed comment reaches the end of the source line. - stopLine bool -} - // interpolationCodeResult describes the effect of one consumed interpolation byte. type interpolationCodeResult struct { // depth is the brace depth after consuming the byte at the scanner position. diff --git a/embedding/commentfilter/kotlin_filter.go b/embedding/commentfilter/kotlin_filter.go index afab1d6..ada8d01 100644 --- a/embedding/commentfilter/kotlin_filter.go +++ b/embedding/commentfilter/kotlin_filter.go @@ -102,8 +102,8 @@ func (f *kotlinLineFilter) filterLine() (string, bool) { if f.consumeString() { continue } - if consumed, stop := f.consumeComment(); consumed { - if stop { + if comment := f.consumeComment(); comment.consumed { + if comment.stopLine { break } @@ -258,8 +258,8 @@ func (f *kotlinLineFilter) consumeInterpolationDepth(depth *int) { if f.consumeString() { continue } - if consumed, stop := f.consumeComment(); consumed { - if stop { + if comment := f.consumeComment(); comment.consumed { + if comment.stopLine { return } @@ -295,17 +295,17 @@ func (f *kotlinLineFilter) consumeInterpolationCode(depth int) (int, bool) { } } -// consumeComment consumes a Kotlin comment and reports whether it ended the line. -func (f *kotlinLineFilter) consumeComment() (bool, bool) { +// consumeComment consumes a Kotlin comment when one starts at the scanner position. +func (f *kotlinLineFilter) consumeComment() commentConsumeResult { if strings.HasPrefix(f.line[f.position:], cStyleDocCommentStart) { f.startBlockComment(f.mode == RetainDocumentation) - return true, false + return commentConsumeResult{consumed: true} } if strings.HasPrefix(f.line[f.position:], cStyleBlockCommentStart) { f.startBlockComment(f.mode == RetainBlock || f.mode == RetainRegular) - return true, false + return commentConsumeResult{consumed: true} } if strings.HasPrefix(f.line[f.position:], "//") { f.hadComment = true @@ -314,10 +314,10 @@ func (f *kotlinLineFilter) consumeComment() (bool, bool) { } f.position = len(f.line) - return true, true + return commentConsumeResult{consumed: true, stopLine: true} } - return false, false + return commentConsumeResult{} } // startBlockComment starts a Kotlin block comment with nesting depth one. diff --git a/embedding/commentfilter/marker_comment_filter.go b/embedding/commentfilter/marker_comment_filter.go index b99b79f..921f2ce 100644 --- a/embedding/commentfilter/marker_comment_filter.go +++ b/embedding/commentfilter/marker_comment_filter.go @@ -20,33 +20,6 @@ package commentfilter import "strings" -// BlockMarker describes a block comment marker pair. -type BlockMarker struct { - // Start is the block comment opening marker. - Start string - - // End is the block comment closing marker. - End string -} - -// DocumentationMarker describes API documentation comment markers. -type DocumentationMarker struct { - // Inline contains documentation line-comment markers. - Inline []string - - // Block contains documentation block-comment marker pairs. - Block []BlockMarker -} - -// TextBlockMarker describes a multi-line text literal delimiter. -type TextBlockMarker struct { - // Delimiter opens and closes the text literal. - Delimiter string - - // Escapes reports whether backslashes escape delimiter bytes. - Escapes bool -} - // activeSegment describes a multi-line construct currently being scanned. type activeSegment struct { // end closes the active construct. @@ -62,37 +35,10 @@ type activeSegment struct { comment bool } -// CommentMarker describes lexical comment markers and string delimiters for a language family. -type CommentMarker struct { - // Inline contains line-comment markers. - Inline []string - - // Block contains block-comment marker pairs. - Block []BlockMarker - - // Documentation contains API documentation comment markers. - Documentation DocumentationMarker - - // TextBlocks contains markers that open and close multi-line text literals. - TextBlocks []TextBlockMarker - - // QuoteChars contains characters that open and close quoted strings. - QuoteChars string -} - -// MarkerCommentFilter removes comments using lexical markers declared in CommentMarker. -type MarkerCommentFilter struct { - // Syntax contains the comment markers and string delimiters to recognize. - Syntax CommentMarker -} - // markerState tracks active multi-line lexical constructs across source lines. type markerState struct { - // active reports whether scanning is inside a multi-line construct. - active bool - - // segment contains the active construct configuration. - segment activeSegment + // segment contains the active construct configuration, if one is open. + segment *activeSegment } // markerLineFilter tracks lexical comment filtering state for one source line. @@ -162,8 +108,8 @@ func (f *markerLineFilter) filterLine() (string, bool) { if f.consumeQuotedSegment() { continue } - if consumed, stop := f.consumeComment(); consumed { - if stop { + if comment := f.consumeComment(); comment.consumed { + if comment.stopLine { break } @@ -177,14 +123,14 @@ func (f *markerLineFilter) filterLine() (string, bool) { // consumeActiveSegment consumes text while the scanner is inside a multi-line construct. func (f *markerLineFilter) consumeActiveSegment() bool { - if !f.state.active { + segment := f.state.segment + if segment == nil { return false } - segment := f.state.segment if segment.comment { f.hadComment = true } - endPosition, found := segmentEnd(f.line, f.position, segment) + endPosition, found := segmentEnd(f.line, f.position, *segment) if !found { if segment.keep { f.result.WriteString(f.line[f.position:]) @@ -197,8 +143,7 @@ func (f *markerLineFilter) consumeActiveSegment() bool { f.result.WriteString(f.line[f.position:endPosition]) } f.position = endPosition - f.state.active = false - f.state.segment = activeSegment{} + f.state.segment = nil return true } @@ -228,8 +173,7 @@ func (f *markerLineFilter) consumeTextBlockStart() bool { } f.result.WriteString(marker.Delimiter) f.position += len(marker.Delimiter) - f.state.active = true - f.state.segment = activeSegment{ + f.state.segment = &activeSegment{ end: marker.Delimiter, keep: true, escapes: marker.Escapes, @@ -272,30 +216,30 @@ func quotedSegmentEnd(line string, position int, quoteChars string) int { return len(line) } -// consumeComment consumes a comment and reports whether it consumed input and ended the line. -func (f *markerLineFilter) consumeComment() (bool, bool) { +// consumeComment consumes a comment when one starts at the scanner position. +func (f *markerLineFilter) consumeComment() commentConsumeResult { if prefixAt(f.line, f.position, f.filter.Syntax.Documentation.Inline) { f.consumeInlineComment(f.mode == RetainDocumentation) - return true, true + return commentConsumeResult{consumed: true, stopLine: true} } if block, found := blockAt(f.line, f.position, f.filter.Syntax.Documentation.Block); found { f.startBlockComment(block, f.mode == RetainDocumentation) - return true, false + return commentConsumeResult{consumed: true} } if prefixAt(f.line, f.position, f.filter.Syntax.Inline) { f.consumeInlineComment(f.mode == RetainInline || f.mode == RetainRegular) - return true, true + return commentConsumeResult{consumed: true, stopLine: true} } if block, found := blockAt(f.line, f.position, f.filter.Syntax.Block); found { f.startBlockComment(block, f.mode == RetainBlock || f.mode == RetainRegular) - return true, false + return commentConsumeResult{consumed: true} } - return false, false + return commentConsumeResult{} } // consumeInlineComment consumes the rest of the line as a line comment. @@ -310,8 +254,7 @@ func (f *markerLineFilter) consumeInlineComment(keep bool) { // startBlockComment records the active block comment markers and whether to keep them. func (f *markerLineFilter) startBlockComment(block BlockMarker, keep bool) { f.hadComment = true - f.state.active = true - f.state.segment = activeSegment{ + f.state.segment = &activeSegment{ end: block.End, keep: keep, comment: true, @@ -326,20 +269,13 @@ func (f *markerLineFilter) consumeCodeByte() { // prefixAt reports whether one of the given prefixes starts at the position. func prefixAt(line string, position int, prefixes []string) bool { - _, found := prefixFrom(line, position, prefixes) - - return found -} - -// prefixFrom returns the prefix starting at position when one exists. -func prefixFrom(line string, position int, prefixes []string) (string, bool) { for _, prefix := range prefixes { if strings.HasPrefix(line[position:], prefix) { - return prefix, true + return true } } - return "", false + return false } // textBlockAt reports whether one of the given text block markers starts at the position. diff --git a/embedding/commentfilter/marker_syntax.go b/embedding/commentfilter/marker_syntax.go new file mode 100644 index 0000000..153b42e --- /dev/null +++ b/embedding/commentfilter/marker_syntax.go @@ -0,0 +1,70 @@ +// Copyright 2026, TeamDev. All rights reserved. +// +// Redistribution and use in source and/or binary forms, with or without +// modification, must retain the above copyright notice and the following +// disclaimer. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package commentfilter + +// BlockMarker describes a block comment marker pair. +type BlockMarker struct { + // Start is the block comment opening marker. + Start string + + // End is the block comment closing marker. + End string +} + +// DocumentationMarker describes API documentation comment markers. +type DocumentationMarker struct { + // Inline contains documentation line-comment markers. + Inline []string + + // Block contains documentation block-comment marker pairs. + Block []BlockMarker +} + +// TextBlockMarker describes a multi-line text literal delimiter. +type TextBlockMarker struct { + // Delimiter opens and closes the text literal. + Delimiter string + + // Escapes reports whether backslashes escape delimiter bytes. + Escapes bool +} + +// CommentMarker describes lexical comment markers and string delimiters for a language family. +type CommentMarker struct { + // Inline contains line-comment markers. + Inline []string + + // Block contains block-comment marker pairs. + Block []BlockMarker + + // Documentation contains API documentation comment markers. + Documentation DocumentationMarker + + // TextBlocks contains markers that open and close multi-line text literals. + TextBlocks []TextBlockMarker + + // QuoteChars contains characters that open and close quoted strings. + QuoteChars string +} + +// MarkerCommentFilter removes comments using lexical markers declared in CommentMarker. +type MarkerCommentFilter struct { + // Syntax contains the comment markers and string delimiters to recognize. + Syntax CommentMarker +} From 9539830aa8f1cd0845365fc2a9a05b375ef5bf66 Mon Sep 17 00:00:00 2001 From: Vladyslav Kuksiuk Date: Mon, 6 Jul 2026 18:34:16 +0200 Subject: [PATCH 3/4] Refactor comments filtering. --- embedding/commentfilter/csharp_filter.go | 198 ++---------- embedding/commentfilter/filter.go | 14 +- embedding/commentfilter/javascript_filter.go | 272 +++------------- embedding/commentfilter/kotlin_filter.go | 171 +++------- embedding/commentfilter/line_filter.go | 298 ++++++++++++++++++ .../commentfilter/marker_comment_filter.go | 99 +----- 6 files changed, 428 insertions(+), 624 deletions(-) create mode 100644 embedding/commentfilter/line_filter.go diff --git a/embedding/commentfilter/csharp_filter.go b/embedding/commentfilter/csharp_filter.go index 7a56d24..d8c1abc 100644 --- a/embedding/commentfilter/csharp_filter.go +++ b/embedding/commentfilter/csharp_filter.go @@ -28,18 +28,24 @@ const ( csharpEscapedQuote = `""` csharpEscapedOpenBrace = `{{` csharpEscapedCloseBrace = `}}` + csharpDocLineComment = `///` ) +// csharpInterpolation describes the C# `{...}` string interpolation form. +// Holes open with a bare `{` inside interpolated string text. +var csharpInterpolation = interpolationForm{ + start: "{", + openBrace: '{', + closeBrace: '}', +} + // CSharpCommentFilter filters C# comments while preserving string literal text. type CSharpCommentFilter struct{} // csharpState tracks C# lexical state that can span source lines. type csharpState struct { - // blockActive reports whether scanning is inside a block comment. - blockActive bool - - // blockKeep reports whether the active block comment should be retained. - blockKeep bool + // block tracks a block comment across source lines. + block blockCommentState // stringActive reports whether scanning is inside a string. stringActive bool @@ -59,23 +65,10 @@ type csharpState struct { // csharpLineFilter filters one C# source line. type csharpLineFilter struct { - // line is the source line being filtered. - line string - - // mode selects which comments to retain. - mode Mode + lineFilter // state tracks C# constructs across lines. state *csharpState - - // result accumulates the filtered source line. - result strings.Builder - - // position is the current byte index in line. - position int - - // hadComment reports whether the line contained a recognized comment. - hadComment bool } // Filter removes or preserves C# comments according to mode. @@ -89,25 +82,19 @@ func (CSharpCommentFilter) Filter(lines []string, mode Mode) []string { state := csharpState{} return filterLines(lines, func(line string) (string, bool) { - return filterCSharpLine(line, mode, &state) - }) -} - -// filterCSharpLine removes or preserves recognized C# comments from one line. -func filterCSharpLine(line string, mode Mode, state *csharpState) (string, bool) { - filter := csharpLineFilter{ - line: line, - mode: mode, - state: state, - } + filter := csharpLineFilter{ + lineFilter: lineFilter{line: line, mode: mode}, + state: &state, + } - return filter.filterLine() + return filter.filterLine() + }) } // filterLine walks the current line until it reaches its end or a line comment. func (f *csharpLineFilter) filterLine() (string, bool) { for f.position < len(f.line) { - if f.consumeActiveBlock() { + if f.consumeActiveBlock(&f.state.block) { continue } if f.consumeStringInterpolation() { @@ -116,7 +103,7 @@ func (f *csharpLineFilter) filterLine() (string, bool) { if f.consumeStringText() { continue } - if f.consumeCharacterLiteral() { + if f.consumeQuotedSegment("'") { continue } if f.consumeStringStart() { @@ -137,31 +124,6 @@ func (f *csharpLineFilter) filterLine() (string, bool) { return f.result.String(), f.hadComment } -// consumeActiveBlock consumes text while the scanner is inside a block comment. -func (f *csharpLineFilter) consumeActiveBlock() bool { - if !f.state.blockActive { - return false - } - f.hadComment = true - end := strings.Index(f.line[f.position:], cStyleBlockCommentEnd) - if end < 0 { - if f.state.blockKeep { - f.result.WriteString(f.line[f.position:]) - } - f.position = len(f.line) - - return true - } - endPosition := f.position + end + len(cStyleBlockCommentEnd) - if f.state.blockKeep { - f.result.WriteString(f.line[f.position:endPosition]) - } - f.position = endPosition - f.state.blockActive = false - - return true -} - // consumeStringInterpolation filters code inside an active interpolation expression. func (f *csharpLineFilter) consumeStringInterpolation() bool { if !f.state.stringActive || f.state.interpolationDepth == 0 { @@ -175,7 +137,7 @@ func (f *csharpLineFilter) consumeStringInterpolation() bool { continue } - if f.consumeInterpolationCodeByte() { + if f.consumeInterpolationCodeByte(csharpInterpolation, &f.state.interpolationDepth) { return true } } @@ -185,7 +147,7 @@ func (f *csharpLineFilter) consumeStringInterpolation() bool { // consumeInterpolationSegment consumes multi-byte interpolation content. func (f *csharpLineFilter) consumeInterpolationSegment() bool { - if f.consumeActiveBlock() { + if f.consumeActiveBlock(&f.state.block) { return true } if f.consumeInterpolationFormat() { @@ -201,26 +163,6 @@ func (f *csharpLineFilter) consumeInterpolationSegment() bool { return false } -// consumeInterpolationCodeByte copies expression code and reports whether interpolation closed. -func (f *csharpLineFilter) consumeInterpolationCodeByte() bool { - switch f.line[f.position] { - case '{': - f.state.interpolationDepth++ - f.consumeCodeByte() - - return false - case '}': - f.state.interpolationDepth-- - f.consumeCodeByte() - - return f.state.interpolationDepth == 0 - default: - f.consumeCodeByte() - - return false - } -} - // consumeInterpolationFormat copies C# format text after a top-level interpolation colon. // This lightweight scanner does not track parentheses inside interpolation expressions, // so a ternary colon in parentheses is treated as format text. @@ -253,13 +195,11 @@ func (f *csharpLineFilter) consumeInterpolationString() bool { if !found { return false } - f.result.WriteString(token) - f.position += len(token) + f.consumeMarker(token) for f.position < len(f.line) { switch { - case verbatim && strings.HasPrefix(f.line[f.position:], csharpEscapedQuote): - f.result.WriteString(csharpEscapedQuote) - f.position += len(csharpEscapedQuote) + case verbatim && f.hasPrefix(csharpEscapedQuote): + f.consumeMarker(csharpEscapedQuote) case !verbatim && f.line[f.position] == '\\': f.writeEscapedByte() case f.line[f.position] == '"': @@ -292,17 +232,15 @@ func (f *csharpLineFilter) consumeStringText() bool { // consumeStringTextSegment consumes special syntax inside active string text. func (f *csharpLineFilter) consumeStringTextSegment() bool { switch { - case f.state.stringVerbatim && strings.HasPrefix(f.line[f.position:], csharpEscapedQuote): - f.result.WriteString(csharpEscapedQuote) - f.position += len(csharpEscapedQuote) + case f.state.stringVerbatim && f.hasPrefix(csharpEscapedQuote): + f.consumeMarker(csharpEscapedQuote) case !f.state.stringVerbatim && f.line[f.position] == '\\': f.writeEscapedByte() case f.line[f.position] == '"': f.consumeCodeByte() f.closeString() case f.startsEscapedInterpolationBrace(): - f.result.WriteString(f.line[f.position : f.position+2]) - f.position += 2 + f.consumeMarker(f.line[f.position : f.position+2]) case f.state.stringInterpolated && f.line[f.position] == '{': f.consumeCodeByte() f.state.interpolationDepth = 1 @@ -313,18 +251,6 @@ func (f *csharpLineFilter) consumeStringTextSegment() bool { return true } -// consumeCharacterLiteral copies a C# character literal. -func (f *csharpLineFilter) consumeCharacterLiteral() bool { - quoteEnd := quotedSegmentEnd(f.line, f.position, "'") - if quoteEnd <= f.position { - return false - } - f.result.WriteString(f.line[f.position:quoteEnd]) - f.position = quoteEnd - - return true -} - // consumeStringStart starts a C# string literal at the current position. func (f *csharpLineFilter) consumeStringStart() bool { token, verbatim, found := csharpStringTokenAt(f.line, f.position) @@ -332,7 +258,10 @@ func (f *csharpLineFilter) consumeStringStart() bool { return false } interpolated := strings.HasPrefix(token, "$") || strings.HasPrefix(token, "@$") - f.startString(token, verbatim, interpolated) + f.consumeMarker(token) + f.state.stringActive = true + f.state.stringVerbatim = verbatim + f.state.stringInterpolated = interpolated return true } @@ -355,23 +284,13 @@ func csharpStringTokenAt(line string, position int) (string, bool, bool) { } } -// startString records the active string literal and copies its opening token. -func (f *csharpLineFilter) startString(token string, verbatim bool, interpolated bool) { - f.result.WriteString(token) - f.position += len(token) - f.state.stringActive = true - f.state.stringVerbatim = verbatim - f.state.stringInterpolated = interpolated -} - // startsEscapedInterpolationBrace reports whether the position starts {{ or }} in string text. func (f *csharpLineFilter) startsEscapedInterpolationBrace() bool { if !f.state.stringInterpolated || f.position+1 >= len(f.line) { return false } - return strings.HasPrefix(f.line[f.position:], csharpEscapedOpenBrace) || - strings.HasPrefix(f.line[f.position:], csharpEscapedCloseBrace) + return f.hasPrefix(csharpEscapedOpenBrace) || f.hasPrefix(csharpEscapedCloseBrace) } // closeSingleLineStringAtLineEnd clears invalid single-line strings at end of line. @@ -392,57 +311,10 @@ func (f *csharpLineFilter) closeString() { // consumeComment consumes a C# comment when one starts at the scanner position. func (f *csharpLineFilter) consumeComment() commentConsumeResult { - if strings.HasPrefix(f.line[f.position:], cStyleDocCommentStart) { - f.startBlockComment(f.mode == RetainDocumentation) - - return commentConsumeResult{consumed: true} - } - if strings.HasPrefix(f.line[f.position:], cStyleBlockCommentStart) { - f.startBlockComment(f.mode == RetainBlock || f.mode == RetainRegular) - - return commentConsumeResult{consumed: true} - } - if strings.HasPrefix(f.line[f.position:], "///") { - f.hadComment = true - if f.mode == RetainDocumentation { - f.result.WriteString(f.line[f.position:]) - } - f.position = len(f.line) - - return commentConsumeResult{consumed: true, stopLine: true} - } - if strings.HasPrefix(f.line[f.position:], "//") { - f.hadComment = true - if f.mode == RetainInline || f.mode == RetainRegular { - f.result.WriteString(f.line[f.position:]) - } - f.position = len(f.line) - - return commentConsumeResult{consumed: true, stopLine: true} - } - - return commentConsumeResult{} + return f.consumeCStyleComment(csharpDocLineComment, f.startBlockComment) } // startBlockComment records the active block comment and whether to keep it. func (f *csharpLineFilter) startBlockComment(keep bool) { - f.hadComment = true - f.state.blockActive = true - f.state.blockKeep = keep -} - -// writeEscapedByte copies an escaped byte pair from a regular string. -func (f *csharpLineFilter) writeEscapedByte() { - f.result.WriteByte(f.line[f.position]) - f.position++ - if f.position < len(f.line) { - f.result.WriteByte(f.line[f.position]) - f.position++ - } -} - -// consumeCodeByte copies one source byte. -func (f *csharpLineFilter) consumeCodeByte() { - f.result.WriteByte(f.line[f.position]) - f.position++ + f.lineFilter.startBlockComment(&f.state.block, keep) } diff --git a/embedding/commentfilter/filter.go b/embedding/commentfilter/filter.go index e9f72f6..166addf 100644 --- a/embedding/commentfilter/filter.go +++ b/embedding/commentfilter/filter.go @@ -23,6 +23,7 @@ import ( "fmt" "log/slog" "path/filepath" + "slices" "strings" ) @@ -178,7 +179,7 @@ func warnUnsupportedCommentsMode( embeddingLine int, supportedModes []Mode, ) bool { - if containsMode(supportedModes, mode) { + if slices.Contains(supportedModes, mode) { return false } var wrappedModes []string @@ -199,14 +200,3 @@ func warnUnsupportedCommentsMode( return true } - -// containsMode reports whether the list includes the given mode. -func containsMode(modes []Mode, mode Mode) bool { - for _, supportedMode := range modes { - if supportedMode == mode { - return true - } - } - - return false -} diff --git a/embedding/commentfilter/javascript_filter.go b/embedding/commentfilter/javascript_filter.go index 4038f7c..a68beeb 100644 --- a/embedding/commentfilter/javascript_filter.go +++ b/embedding/commentfilter/javascript_filter.go @@ -20,18 +20,29 @@ package commentfilter import "strings" -const jsTemplateInterpolationStart = "${" +const jsTemplateDelimiter = "`" + +// jsInterpolation describes the JavaScript `${...}` template interpolation form. +var jsInterpolation = interpolationForm{ + start: "${", + openBrace: '{', + closeBrace: '}', +} + +// jsTemplateLiteral describes the JavaScript template literal form. +var jsTemplateLiteral = interpolatedLiteral{ + delimiter: jsTemplateDelimiter, + escapes: true, + interpolation: jsInterpolation, +} // JavaScriptCommentFilter filters JavaScript and TypeScript comments while preserving literal text. type JavaScriptCommentFilter struct{} // javascriptState tracks JavaScript lexical state that can span source lines. type javascriptState struct { - // blockActive reports whether scanning is inside a block comment. - blockActive bool - - // blockKeep reports whether the active block comment should be retained. - blockKeep bool + // block tracks a block comment across source lines. + block blockCommentState // template reports whether scanning is inside template literal text. template bool @@ -48,32 +59,10 @@ type javascriptState struct { // javascriptLineFilter filters one JavaScript or TypeScript source line. type javascriptLineFilter struct { - // line is the source line being filtered. - line string - - // mode selects which comments to retain. - mode Mode + lineFilter // state tracks JavaScript constructs across lines. state *javascriptState - - // result accumulates the filtered source line. - result strings.Builder - - // position is the current byte index in line. - position int - - // hadComment reports whether the line contained a recognized comment. - hadComment bool -} - -// interpolationCodeResult describes the effect of one consumed interpolation byte. -type interpolationCodeResult struct { - // depth is the brace depth after consuming the byte at the scanner position. - depth int - - // closed reports whether the consumed byte closed the current interpolation expression. - closed bool } // Filter removes or preserves JavaScript and TypeScript comments according to mode. @@ -87,25 +76,19 @@ func (JavaScriptCommentFilter) Filter(lines []string, mode Mode) []string { state := javascriptState{} return filterLines(lines, func(line string) (string, bool) { - return filterJavaScriptLine(line, mode, &state) - }) -} - -// filterJavaScriptLine removes or preserves recognized JavaScript comments from one line. -func filterJavaScriptLine(line string, mode Mode, state *javascriptState) (string, bool) { - filter := javascriptLineFilter{ - line: line, - mode: mode, - state: state, - } + filter := javascriptLineFilter{ + lineFilter: lineFilter{line: line, mode: mode}, + state: &state, + } - return filter.filterLine() + return filter.filterLine() + }) } // filterLine walks the current line until it reaches its end or a line comment. func (f *javascriptLineFilter) filterLine() (string, bool) { for f.position < len(f.line) { - if f.consumeActiveBlock() { + if f.consumeActiveBlock(&f.state.block) { continue } if f.consumeTemplateInterpolation() { @@ -133,31 +116,6 @@ func (f *javascriptLineFilter) filterLine() (string, bool) { return f.result.String(), f.hadComment } -// consumeActiveBlock consumes text while the scanner is inside a block comment. -func (f *javascriptLineFilter) consumeActiveBlock() bool { - if !f.state.blockActive { - return false - } - f.hadComment = true - end := strings.Index(f.line[f.position:], cStyleBlockCommentEnd) - if end < 0 { - if f.state.blockKeep { - f.result.WriteString(f.line[f.position:]) - } - f.position = len(f.line) - - return true - } - endPosition := f.position + end + len(cStyleBlockCommentEnd) - if f.state.blockKeep { - f.result.WriteString(f.line[f.position:endPosition]) - } - f.position = endPosition - f.state.blockActive = false - - return true -} - // consumeTemplateInterpolation resumes JavaScript expression scanning inside `${...}`. func (f *javascriptLineFilter) consumeTemplateInterpolation() bool { if f.state.templateInterpolationDepth == 0 { @@ -176,54 +134,17 @@ func (f *javascriptLineFilter) consumeTemplateInterpolation() bool { // consumeTemplateText copies template text and filters comments inside `${...}` code. func (f *javascriptLineFilter) consumeTemplateText() bool { - if !f.state.template && f.line[f.position] != '`' { - return false - } - if !f.state.template { - f.state.template = true - f.consumeCodeByte() - } - for f.position < len(f.line) { - switch { - case f.line[f.position] == '\\': - f.writeEscapedByte() - case f.line[f.position] == '`': - f.consumeCodeByte() - f.state.template = false - - return true - case strings.HasPrefix(f.line[f.position:], jsTemplateInterpolationStart): - f.result.WriteString(jsTemplateInterpolationStart) - f.position += len(jsTemplateInterpolationStart) - f.state.template = false - f.state.templateInterpolationDepth = 1 - f.consumeTemplateInterpolation() - if f.state.templateInterpolationDepth > 0 { - return true - } - default: - f.consumeCodeByte() - } - } - - return true + return f.consumeInterpolatedText( + jsTemplateLiteral, + &f.state.template, + &f.state.templateInterpolationDepth, + f.consumeTemplateInterpolation, + ) } // consumeString copies a quoted string without scanning comment markers inside it. func (f *javascriptLineFilter) consumeString() bool { - if f.position >= len(f.line) { - return false - } - switch f.line[f.position] { - case '"', '\'': - quoteEnd := quotedSegmentEnd(f.line, f.position, "\"'") - f.result.WriteString(f.line[f.position:quoteEnd]) - f.position = quoteEnd - - return true - default: - return false - } + return f.consumeQuotedSegment("\"'") } // consumeRegexLiteral copies a regular-expression literal without treating its content as comments. @@ -266,8 +187,7 @@ func (f *javascriptLineFilter) regexStartsHere() bool { if f.line[f.position] != '/' { return false } - if strings.HasPrefix(f.line[f.position:], "//") || - strings.HasPrefix(f.line[f.position:], cStyleBlockCommentStart) { + if f.hasPrefix("//") || f.hasPrefix(cStyleBlockCommentStart) { return false } previous := previousSignificantToken(f.line[:f.position]) @@ -304,7 +224,7 @@ func (f *javascriptLineFilter) consumeRegexFlags() { // depth - current brace depth of the interpolation expression; updated in place. func (f *javascriptLineFilter) consumeInterpolationDepth(depth *int) { for f.position < len(f.line) { - if f.consumeActiveBlock() { + if f.consumeActiveBlock(&f.state.block) { continue } if f.consumeNestedTemplateLiteral() { @@ -323,11 +243,7 @@ func (f *javascriptLineFilter) consumeInterpolationDepth(depth *int) { continue } - code := f.consumeInterpolationCode(*depth) - *depth = code.depth - if code.closed { - *depth = 0 - + if f.consumeInterpolationCodeByte(jsInterpolation, depth) { return } } @@ -335,33 +251,12 @@ func (f *javascriptLineFilter) consumeInterpolationDepth(depth *int) { // consumeNestedTemplateLiteral copies a template literal found inside interpolation code. func (f *javascriptLineFilter) consumeNestedTemplateLiteral() bool { - if !f.startOrResumeNestedTemplateLiteral() { - return false - } - for f.position < len(f.line) { - switch { - case f.line[f.position] == '\\': - f.writeEscapedByte() - case f.line[f.position] == '`': - f.consumeCodeByte() - f.state.nestedTemplate = false - - return true - case strings.HasPrefix(f.line[f.position:], jsTemplateInterpolationStart): - f.result.WriteString(jsTemplateInterpolationStart) - f.position += len(jsTemplateInterpolationStart) - f.state.nestedTemplate = false - f.state.nestedTemplateInterpolationDepth = 1 - f.consumeNestedTemplateInterpolation() - if f.state.nestedTemplateInterpolationDepth > 0 { - return true - } - default: - f.consumeCodeByte() - } - } - - return true + return f.consumeInterpolatedText( + jsTemplateLiteral, + &f.state.nestedTemplate, + &f.state.nestedTemplateInterpolationDepth, + f.consumeNestedTemplateInterpolation, + ) } // consumeNestedTemplateInterpolation resumes code inside nested template `${...}`. @@ -377,93 +272,14 @@ func (f *javascriptLineFilter) consumeNestedTemplateInterpolation() bool { return true } -// startOrResumeNestedTemplateLiteral enters or resumes nested template scanning. -func (f *javascriptLineFilter) startOrResumeNestedTemplateLiteral() bool { - if f.state.nestedTemplate { - return true - } - if f.position >= len(f.line) || f.line[f.position] != '`' { - return false - } - f.state.nestedTemplate = true - f.consumeCodeByte() - - return true -} - -// consumeInterpolationCode copies expression code and updates interpolation brace depth. -// -// Parameters: -// depth - current brace depth before consuming the byte at the scanner position. -// -// Returns interpolation code result. -func (f *javascriptLineFilter) consumeInterpolationCode(depth int) interpolationCodeResult { - switch f.line[f.position] { - case '{': - depth++ - f.consumeCodeByte() - - return interpolationCodeResult{depth: depth} - case '}': - depth-- - f.consumeCodeByte() - - return interpolationCodeResult{depth: depth, closed: depth == 0} - default: - f.consumeCodeByte() - - return interpolationCodeResult{depth: depth} - } -} - // consumeComment consumes a JavaScript comment when one starts at the scanner position. -// -// Returns comment consume result. func (f *javascriptLineFilter) consumeComment() commentConsumeResult { - if strings.HasPrefix(f.line[f.position:], cStyleDocCommentStart) { - f.startBlockComment(f.mode == RetainDocumentation) - - return commentConsumeResult{consumed: true} - } - if strings.HasPrefix(f.line[f.position:], cStyleBlockCommentStart) { - f.startBlockComment(f.mode == RetainBlock || f.mode == RetainRegular) - - return commentConsumeResult{consumed: true} - } - if strings.HasPrefix(f.line[f.position:], "//") { - f.hadComment = true - if f.mode == RetainInline || f.mode == RetainRegular { - f.result.WriteString(f.line[f.position:]) - } - f.position = len(f.line) - - return commentConsumeResult{consumed: true, stopLine: true} - } - - return commentConsumeResult{} + return f.consumeCStyleComment("", f.startBlockComment) } -// startBlockComment records the active block comment markers and whether to keep them. +// startBlockComment records the active block comment and whether to keep it. func (f *javascriptLineFilter) startBlockComment(keep bool) { - f.hadComment = true - f.state.blockActive = true - f.state.blockKeep = keep -} - -// writeEscapedByte copies an escaped byte pair from a literal. -func (f *javascriptLineFilter) writeEscapedByte() { - f.result.WriteByte(f.line[f.position]) - f.position++ - if f.position < len(f.line) { - f.result.WriteByte(f.line[f.position]) - f.position++ - } -} - -// consumeCodeByte copies one source byte. -func (f *javascriptLineFilter) consumeCodeByte() { - f.result.WriteByte(f.line[f.position]) - f.position++ + f.lineFilter.startBlockComment(&f.state.block, keep) } // previousSignificantToken returns the last non-space token in text. diff --git a/embedding/commentfilter/kotlin_filter.go b/embedding/commentfilter/kotlin_filter.go index ada8d01..12ed456 100644 --- a/embedding/commentfilter/kotlin_filter.go +++ b/embedding/commentfilter/kotlin_filter.go @@ -18,10 +18,22 @@ package commentfilter -import "strings" - const kotlinRawStringDelimiter = "\"\"\"" +// kotlinInterpolation describes the Kotlin `${...}` string interpolation form. +var kotlinInterpolation = interpolationForm{ + start: "${", + openBrace: '{', + closeBrace: '}', +} + +// kotlinRawStringLiteral describes the Kotlin raw triple-quoted string form. +// Raw strings have no backslash escapes. +var kotlinRawStringLiteral = interpolatedLiteral{ + delimiter: kotlinRawStringDelimiter, + interpolation: kotlinInterpolation, +} + // KotlinCommentFilter filters Kotlin comments while preserving Kotlin string forms. type KotlinCommentFilter struct{} @@ -42,23 +54,10 @@ type kotlinState struct { // kotlinLineFilter filters one Kotlin source line. type kotlinLineFilter struct { - // line is the source line being filtered. - line string - - // mode selects which comments to retain. - mode Mode + lineFilter // state tracks Kotlin constructs across lines. state *kotlinState - - // result accumulates the filtered source line. - result strings.Builder - - // position is the current byte index in line. - position int - - // hadComment reports whether the line contained a recognized comment. - hadComment bool } // Filter removes or preserves Kotlin comments according to mode. @@ -72,19 +71,13 @@ func (KotlinCommentFilter) Filter(lines []string, mode Mode) []string { state := kotlinState{} return filterLines(lines, func(line string) (string, bool) { - return filterKotlinLine(line, mode, &state) - }) -} - -// filterKotlinLine removes or preserves recognized Kotlin comments from one line. -func filterKotlinLine(line string, mode Mode, state *kotlinState) (string, bool) { - filter := kotlinLineFilter{ - line: line, - mode: mode, - state: state, - } + filter := kotlinLineFilter{ + lineFilter: lineFilter{line: line, mode: mode}, + state: &state, + } - return filter.filterLine() + return filter.filterLine() + }) } // filterLine walks the current line until it reaches its end or a line comment. @@ -123,11 +116,11 @@ func (f *kotlinLineFilter) consumeActiveBlock() bool { f.hadComment = true for f.position < len(f.line) { switch { - case strings.HasPrefix(f.line[f.position:], cStyleBlockCommentStart): + case f.hasPrefix(cStyleBlockCommentStart): f.writeBlockText(cStyleBlockCommentStart) f.state.blockDepth++ f.position += len(cStyleBlockCommentStart) - case strings.HasPrefix(f.line[f.position:], cStyleBlockCommentEnd): + case f.hasPrefix(cStyleBlockCommentEnd): f.writeBlockText(cStyleBlockCommentEnd) f.state.blockDepth-- f.position += len(cStyleBlockCommentEnd) @@ -149,37 +142,12 @@ func (f *kotlinLineFilter) consumeActiveBlock() bool { // // It treats the first three quotes in a run of four or more quotes as the raw-string delimiter. func (f *kotlinLineFilter) consumeRawString() bool { - if !f.state.rawString && !strings.HasPrefix(f.line[f.position:], kotlinRawStringDelimiter) { - return false - } - if !f.state.rawString { - f.state.rawString = true - f.result.WriteString(kotlinRawStringDelimiter) - f.position += len(kotlinRawStringDelimiter) - } - for f.position < len(f.line) { - switch { - case strings.HasPrefix(f.line[f.position:], kotlinRawStringDelimiter): - f.result.WriteString(kotlinRawStringDelimiter) - f.position += len(kotlinRawStringDelimiter) - f.state.rawString = false - - return true - case strings.HasPrefix(f.line[f.position:], "${"): - f.result.WriteString("${") - f.position += len("${") - f.state.rawString = false - f.state.rawInterpolationDepth = 1 - f.consumeRawInterpolation() - if f.state.rawInterpolationDepth > 0 { - return true - } - default: - f.consumeCodeByte() - } - } - - return true + return f.consumeInterpolatedText( + kotlinRawStringLiteral, + &f.state.rawString, + &f.state.rawInterpolationDepth, + f.consumeRawInterpolation, + ) } // consumeRawInterpolation resumes Kotlin expression scanning inside a raw-string interpolation. @@ -206,11 +174,7 @@ func (f *kotlinLineFilter) consumeString() bool { return true case '\'': - quoteEnd := quotedSegmentEnd(f.line, f.position, "'") - f.result.WriteString(f.line[f.position:quoteEnd]) - f.position = quoteEnd - - return true + return f.consumeQuotedSegment("'") default: return false } @@ -218,20 +182,17 @@ func (f *kotlinLineFilter) consumeString() bool { // consumeQuotedString copies a Kotlin quoted string and filters comments inside `${...}`. func (f *kotlinLineFilter) consumeQuotedString() { - f.result.WriteByte(f.line[f.position]) - f.position++ + f.consumeCodeByte() for f.position < len(f.line) { switch { case f.line[f.position] == '\\': f.writeEscapedByte() case f.line[f.position] == '"': - f.result.WriteByte(f.line[f.position]) - f.position++ + f.consumeCodeByte() return - case strings.HasPrefix(f.line[f.position:], "${"): - f.result.WriteString("${") - f.position += len("${") + case f.hasPrefix(kotlinInterpolation.start): + f.consumeMarker(kotlinInterpolation.start) f.consumeInterpolation() default: f.consumeCodeByte() @@ -265,59 +226,15 @@ func (f *kotlinLineFilter) consumeInterpolationDepth(depth *int) { continue } - var done bool - *depth, done = f.consumeInterpolationCode(*depth) - if done { - *depth = 0 - + if f.consumeInterpolationCodeByte(kotlinInterpolation, depth) { return } } } -// consumeInterpolationCode copies expression code and updates interpolation brace depth. -func (f *kotlinLineFilter) consumeInterpolationCode(depth int) (int, bool) { - switch f.line[f.position] { - case '{': - depth++ - f.consumeCodeByte() - - return depth, false - case '}': - depth-- - f.consumeCodeByte() - - return depth, depth == 0 - default: - f.consumeCodeByte() - - return depth, false - } -} - // consumeComment consumes a Kotlin comment when one starts at the scanner position. func (f *kotlinLineFilter) consumeComment() commentConsumeResult { - if strings.HasPrefix(f.line[f.position:], cStyleDocCommentStart) { - f.startBlockComment(f.mode == RetainDocumentation) - - return commentConsumeResult{consumed: true} - } - if strings.HasPrefix(f.line[f.position:], cStyleBlockCommentStart) { - f.startBlockComment(f.mode == RetainBlock || f.mode == RetainRegular) - - return commentConsumeResult{consumed: true} - } - if strings.HasPrefix(f.line[f.position:], "//") { - f.hadComment = true - if f.mode == RetainInline || f.mode == RetainRegular { - f.result.WriteString(f.line[f.position:]) - } - f.position = len(f.line) - - return commentConsumeResult{consumed: true, stopLine: true} - } - - return commentConsumeResult{} + return f.consumeCStyleComment("", f.startBlockComment) } // startBlockComment starts a Kotlin block comment with nesting depth one. @@ -337,19 +254,3 @@ func (f *kotlinLineFilter) writeBlockText(text string) { f.result.WriteString(text) } } - -// writeEscapedByte copies an escaped byte pair from a quoted string. -func (f *kotlinLineFilter) writeEscapedByte() { - f.result.WriteByte(f.line[f.position]) - f.position++ - if f.position < len(f.line) { - f.result.WriteByte(f.line[f.position]) - f.position++ - } -} - -// consumeCodeByte copies one source byte. -func (f *kotlinLineFilter) consumeCodeByte() { - f.result.WriteByte(f.line[f.position]) - f.position++ -} diff --git a/embedding/commentfilter/line_filter.go b/embedding/commentfilter/line_filter.go new file mode 100644 index 0000000..c01af5b --- /dev/null +++ b/embedding/commentfilter/line_filter.go @@ -0,0 +1,298 @@ +// Copyright 2026, TeamDev. All rights reserved. +// +// Redistribution and use in source and/or binary forms, with or without +// modification, must retain the above copyright notice and the following +// disclaimer. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package commentfilter + +import "strings" + +// interpolationForm describes how a language opens and closes interpolation expressions. +// +// start and openBrace play different roles: start is the marker that opens an +// interpolation in literal text, while openBrace is the plain brace that nests +// inside expression code. In `${items.map(i => { return i })}` the interpolation +// opens with `${`, but the lambda `{` — which no longer follows a `$` — must +// still deepen the nesting so that its matching closeBrace does not end the +// interpolation early. +type interpolationForm struct { + // start opens an interpolation expression in literal text, e.g. `${`. + start string + + // openBrace nests the expression code one level deeper. + openBrace byte + + // closeBrace closes one nesting level. + // + // The expression ends at the closeBrace that returns the nesting depth to zero. + closeBrace byte +} + +// interpolatedLiteral describes a string literal form that embeds interpolation expressions. +type interpolatedLiteral struct { + // delimiter opens and closes the literal text. + delimiter string + + // escapes reports whether backslashes escape literal bytes. + escapes bool + + // interpolation opens and closes interpolation expressions inside the literal. + interpolation interpolationForm +} + +// lineFilter carries the scanning state shared by the per-language line filters. +type lineFilter struct { + // line is the source line being filtered. + line string + + // mode selects which comments to retain. + mode Mode + + // result accumulates the filtered source line. + result strings.Builder + + // position is the current byte index in line. + position int + + // hadComment reports whether the line contained a recognized comment. + hadComment bool +} + +// blockCommentState tracks a non-nested block comment across source lines. +type blockCommentState struct { + // active reports whether scanning is inside a block comment. + active bool + + // keep reports whether the active block comment should be retained. + keep bool +} + +// hasPrefix reports whether text starts at the scanner position. +func (f *lineFilter) hasPrefix(text string) bool { + return strings.HasPrefix(f.line[f.position:], text) +} + +// consumeCodeByte copies one source byte. +func (f *lineFilter) consumeCodeByte() { + f.result.WriteByte(f.line[f.position]) + f.position++ +} + +// consumeMarker copies marker text and advances past it. +func (f *lineFilter) consumeMarker(marker string) { + f.result.WriteString(marker) + f.position += len(marker) +} + +// writeEscapedByte copies an escaped byte pair from a literal. +func (f *lineFilter) writeEscapedByte() { + f.result.WriteByte(f.line[f.position]) + f.position++ + if f.position < len(f.line) { + f.result.WriteByte(f.line[f.position]) + f.position++ + } +} + +// consumeQuotedSegment copies a quoted literal when one starts at the scanner position. +func (f *lineFilter) consumeQuotedSegment(quoteChars string) bool { + quoteEnd := quotedSegmentEnd(f.line, f.position, quoteChars) + if quoteEnd <= f.position { + return false + } + f.result.WriteString(f.line[f.position:quoteEnd]) + f.position = quoteEnd + + return true +} + +// quotedSegmentEnd returns the end offset of a quoted string starting at position. +func quotedSegmentEnd(line string, position int, quoteChars string) int { + if position >= len(line) || !strings.ContainsRune(quoteChars, rune(line[position])) { + return position + } + quote := line[position] + cursor := position + 1 + for cursor < len(line) { + if line[cursor] == '\\' { + cursor += 2 + + continue + } + if line[cursor] == quote { + return cursor + 1 + } + cursor++ + } + + return len(line) +} + +// consumeLineComment consumes the rest of the line as a line comment. +func (f *lineFilter) consumeLineComment(keep bool) { + f.hadComment = true + if keep { + f.result.WriteString(f.line[f.position:]) + } + f.position = len(f.line) +} + +// consumeCStyleComment consumes a C-style comment when one starts at the scanner position. +// +// Parameters: +// docLineMarker - optional documentation line-comment marker such as `///`; empty when absent. +// startBlock - language hook that records an opened block comment. +// +// Returns comment consume result. +func (f *lineFilter) consumeCStyleComment( + docLineMarker string, + startBlock func(keep bool), +) commentConsumeResult { + if f.hasPrefix(cStyleDocCommentStart) { + startBlock(f.mode == RetainDocumentation) + + return commentConsumeResult{consumed: true} + } + if f.hasPrefix(cStyleBlockCommentStart) { + startBlock(f.mode == RetainBlock || f.mode == RetainRegular) + + return commentConsumeResult{consumed: true} + } + if docLineMarker != "" && f.hasPrefix(docLineMarker) { + f.consumeLineComment(f.mode == RetainDocumentation) + + return commentConsumeResult{consumed: true, stopLine: true} + } + if f.hasPrefix("//") { + f.consumeLineComment(f.mode == RetainInline || f.mode == RetainRegular) + + return commentConsumeResult{consumed: true, stopLine: true} + } + + return commentConsumeResult{} +} + +// startBlockComment records an opened non-nested block comment and whether to keep it. +func (f *lineFilter) startBlockComment(state *blockCommentState, keep bool) { + f.hadComment = true + state.active = true + state.keep = keep +} + +// consumeActiveBlock consumes text while the scanner is inside a non-nested block comment. +func (f *lineFilter) consumeActiveBlock(state *blockCommentState) bool { + if !state.active { + return false + } + f.hadComment = true + end := strings.Index(f.line[f.position:], cStyleBlockCommentEnd) + if end < 0 { + if state.keep { + f.result.WriteString(f.line[f.position:]) + } + f.position = len(f.line) + + return true + } + endPosition := f.position + end + len(cStyleBlockCommentEnd) + if state.keep { + f.result.WriteString(f.line[f.position:endPosition]) + } + f.position = endPosition + state.active = false + + return true +} + +// consumeInterpolationCodeByte copies one expression byte and updates interpolation brace depth. +// +// Returns true when the consumed byte closed the interpolation expression. +func (f *lineFilter) consumeInterpolationCodeByte(form interpolationForm, depth *int) bool { + char := f.line[f.position] + f.consumeCodeByte() + switch char { + case form.openBrace: + *depth++ + case form.closeBrace: + *depth-- + + return *depth == 0 + } + + return false +} + +// consumeInterpolatedText copies interpolated literal text and enters interpolation code. +// +// Parameters: +// literal - describes the literal delimiter, escape rule, and interpolation opener. +// active - tracks whether the scanner is inside the literal text across lines. +// depth - tracks the interpolation brace depth across lines. +// resumeInterpolation - language hook that scans interpolation code until depth closes. +// +// Returns true when literal text was consumed. +func (f *lineFilter) consumeInterpolatedText( + literal interpolatedLiteral, + active *bool, + depth *int, + resumeInterpolation func() bool, +) bool { + if !*active && !f.hasPrefix(literal.delimiter) { + return false + } + if !*active { + *active = true + f.consumeMarker(literal.delimiter) + } + for f.position < len(f.line) { + if f.consumeInterpolatedTextSegment(literal, active, depth, resumeInterpolation) { + return true + } + } + + return true +} + +// consumeInterpolatedTextSegment consumes one piece of interpolated literal text. +// +// Returns true when the literal closed or its interpolation stayed open at the line end. +func (f *lineFilter) consumeInterpolatedTextSegment( + literal interpolatedLiteral, + active *bool, + depth *int, + resumeInterpolation func() bool, +) bool { + switch { + case literal.escapes && f.line[f.position] == '\\': + f.writeEscapedByte() + case f.hasPrefix(literal.delimiter): + f.consumeMarker(literal.delimiter) + *active = false + + return true + case f.hasPrefix(literal.interpolation.start): + f.consumeMarker(literal.interpolation.start) + *active = false + *depth = 1 + resumeInterpolation() + + return *depth > 0 + default: + f.consumeCodeByte() + } + + return false +} diff --git a/embedding/commentfilter/marker_comment_filter.go b/embedding/commentfilter/marker_comment_filter.go index 921f2ce..22338c8 100644 --- a/embedding/commentfilter/marker_comment_filter.go +++ b/embedding/commentfilter/marker_comment_filter.go @@ -43,26 +43,13 @@ type markerState struct { // markerLineFilter tracks lexical comment filtering state for one source line. type markerLineFilter struct { + lineFilter + // filter contains the language syntax configuration. filter MarkerCommentFilter - // line is the source line being filtered. - line string - - // mode selects which comments to retain. - mode Mode - // state tracks multi-line lexical constructs across lines. state *markerState - - // result accumulates the filtered source line. - result strings.Builder - - // position is the current byte index in line. - position int - - // hadComment reports whether the line contained a recognized comment. - hadComment bool } // Filter removes or preserves recognized comments across all lines. @@ -76,24 +63,14 @@ func (f MarkerCommentFilter) Filter(lines []string, mode Mode) []string { state := markerState{} return filterLines(lines, func(line string) (string, bool) { - return f.filterLine(line, mode, &state) - }) -} - -// filterLine removes or preserves recognized comments from a single source line. -func (f MarkerCommentFilter) filterLine( - line string, - mode Mode, - state *markerState, -) (string, bool) { - filter := markerLineFilter{ - filter: f, - line: line, - mode: mode, - state: state, - } + filter := markerLineFilter{ + lineFilter: lineFilter{line: line, mode: mode}, + filter: f, + state: &state, + } - return filter.filterLine() + return filter.filterLine() + }) } // filterLine walks the current line until it reaches its end or a line comment. @@ -105,7 +82,7 @@ func (f *markerLineFilter) filterLine() (string, bool) { if f.consumeTextBlockStart() { continue } - if f.consumeQuotedSegment() { + if f.consumeQuotedSegment(f.filter.Syntax.QuoteChars) { continue } if comment := f.consumeComment(); comment.consumed { @@ -171,8 +148,7 @@ func (f *markerLineFilter) consumeTextBlockStart() bool { if !found { return false } - f.result.WriteString(marker.Delimiter) - f.position += len(marker.Delimiter) + f.consumeMarker(marker.Delimiter) f.state.segment = &activeSegment{ end: marker.Delimiter, keep: true, @@ -182,44 +158,10 @@ func (f *markerLineFilter) consumeTextBlockStart() bool { return true } -// consumeQuotedSegment copies a quoted segment without scanning comment markers inside it. -func (f *markerLineFilter) consumeQuotedSegment() bool { - quoteEnd := quotedSegmentEnd(f.line, f.position, f.filter.Syntax.QuoteChars) - if quoteEnd <= f.position { - return false - } - f.result.WriteString(f.line[f.position:quoteEnd]) - f.position = quoteEnd - - return true -} - -// quotedSegmentEnd returns the end offset of a quoted string starting at position. -func quotedSegmentEnd(line string, position int, quoteChars string) int { - if position >= len(line) || !strings.ContainsRune(quoteChars, rune(line[position])) { - return position - } - quote := line[position] - cursor := position + 1 - for cursor < len(line) { - if line[cursor] == '\\' { - cursor += 2 - - continue - } - if line[cursor] == quote { - return cursor + 1 - } - cursor++ - } - - return len(line) -} - // consumeComment consumes a comment when one starts at the scanner position. func (f *markerLineFilter) consumeComment() commentConsumeResult { if prefixAt(f.line, f.position, f.filter.Syntax.Documentation.Inline) { - f.consumeInlineComment(f.mode == RetainDocumentation) + f.consumeLineComment(f.mode == RetainDocumentation) return commentConsumeResult{consumed: true, stopLine: true} } @@ -229,7 +171,7 @@ func (f *markerLineFilter) consumeComment() commentConsumeResult { return commentConsumeResult{consumed: true} } if prefixAt(f.line, f.position, f.filter.Syntax.Inline) { - f.consumeInlineComment(f.mode == RetainInline || f.mode == RetainRegular) + f.consumeLineComment(f.mode == RetainInline || f.mode == RetainRegular) return commentConsumeResult{consumed: true, stopLine: true} } @@ -242,15 +184,6 @@ func (f *markerLineFilter) consumeComment() commentConsumeResult { return commentConsumeResult{} } -// consumeInlineComment consumes the rest of the line as a line comment. -func (f *markerLineFilter) consumeInlineComment(keep bool) { - f.hadComment = true - if keep { - f.result.WriteString(f.line[f.position:]) - } - f.position = len(f.line) -} - // startBlockComment records the active block comment markers and whether to keep them. func (f *markerLineFilter) startBlockComment(block BlockMarker, keep bool) { f.hadComment = true @@ -261,12 +194,6 @@ func (f *markerLineFilter) startBlockComment(block BlockMarker, keep bool) { } } -// consumeCodeByte copies one source byte that does not belong to a recognized comment. -func (f *markerLineFilter) consumeCodeByte() { - f.result.WriteByte(f.line[f.position]) - f.position++ -} - // prefixAt reports whether one of the given prefixes starts at the position. func prefixAt(line string, position int, prefixes []string) bool { for _, prefix := range prefixes { From 1e3c267bf77b5da8d638f44deb23308116f99d5e Mon Sep 17 00:00:00 2001 From: Vladyslav Kuksiuk Date: Mon, 6 Jul 2026 18:47:25 +0200 Subject: [PATCH 4/4] Move `commentConsumeResult`. --- embedding/commentfilter/comment_result.go | 28 ----------------------- embedding/commentfilter/line_filter.go | 9 ++++++++ 2 files changed, 9 insertions(+), 28 deletions(-) delete mode 100644 embedding/commentfilter/comment_result.go diff --git a/embedding/commentfilter/comment_result.go b/embedding/commentfilter/comment_result.go deleted file mode 100644 index 6506284..0000000 --- a/embedding/commentfilter/comment_result.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2026, TeamDev. All rights reserved. -// -// Redistribution and use in source and/or binary forms, with or without -// modification, must retain the above copyright notice and the following -// disclaimer. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -package commentfilter - -// commentConsumeResult describes a consumed source comment. -type commentConsumeResult struct { - // consumed reports whether a recognized comment marker was consumed. - consumed bool - - // stopLine reports whether the consumed comment reaches the end of the source line. - stopLine bool -} diff --git a/embedding/commentfilter/line_filter.go b/embedding/commentfilter/line_filter.go index c01af5b..c49037d 100644 --- a/embedding/commentfilter/line_filter.go +++ b/embedding/commentfilter/line_filter.go @@ -71,6 +71,15 @@ type lineFilter struct { hadComment bool } +// commentConsumeResult describes a consumed source comment. +type commentConsumeResult struct { + // consumed reports whether a recognized comment marker was consumed. + consumed bool + + // stopLine reports whether the consumed comment reaches the end of the source line. + stopLine bool +} + // blockCommentState tracks a non-nested block comment across source lines. type blockCommentState struct { // active reports whether scanning is inside a block comment.