From 3239fd6e5fc0108894d46dcae112a3cf7fd7a7a6 Mon Sep 17 00:00:00 2001 From: Vladyslav Kuksiuk Date: Mon, 6 Jul 2026 15:02:40 +0200 Subject: [PATCH 1/2] Improve go comments filtering. --- embedding/commentfilter/config.go | 5 ++++- embedding/commentfilter/go_filter_test.go | 12 ++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/embedding/commentfilter/config.go b/embedding/commentfilter/config.go index e28cfa0..90e24a2 100644 --- a/embedding/commentfilter/config.go +++ b/embedding/commentfilter/config.go @@ -23,6 +23,8 @@ const ( cStyleBlockCommentEnd = "*/" cStyleDocCommentStart = "/**" javaTextBlockDelimiter = "\"\"\"" + goRawStringDelimiter = "`" + goQuoteChars = "\"'" jsQuoteChars = "\"'`" ) @@ -128,7 +130,8 @@ var goSyntax = CommentMarker{ Block: []BlockMarker{ {Start: cStyleBlockCommentStart, End: cStyleBlockCommentEnd}, }, - QuoteChars: jsQuoteChars, + TextBlocks: []string{goRawStringDelimiter}, + QuoteChars: goQuoteChars, } var hashLineSyntax = CommentMarker{ diff --git a/embedding/commentfilter/go_filter_test.go b/embedding/commentfilter/go_filter_test.go index f2db454..6019b2d 100644 --- a/embedding/commentfilter/go_filter_test.go +++ b/embedding/commentfilter/go_filter_test.go @@ -33,7 +33,11 @@ var _ = Describe("Go", func() { "/* block comment */", "const slash = '/'", "const url = \"http://example.org\"", - "const raw = `/* not a comment */`", + "const raw = `Keep // and /* markers */ in raw strings`", + "const multi = `", + "Keep // and /* markers */ across lines", + "`", + "value := 1 // remove this real comment", "func create() {} // inline comment", } @@ -42,7 +46,11 @@ var _ = Describe("Go", func() { "", "const slash = '/'", "const url = \"http://example.org\"", - "const raw = `/* not a comment */`", + "const raw = `Keep // and /* markers */ in raw strings`", + "const multi = `", + "Keep // and /* markers */ across lines", + "`", + "value := 1 ", "func create() {} ", } From 206b207efec122d1c882f0792ad8fe9a079db542 Mon Sep 17 00:00:00 2001 From: Vladyslav Kuksiuk Date: Mon, 6 Jul 2026 15:15:18 +0200 Subject: [PATCH 2/2] Improve escape handling. --- embedding/commentfilter/config.go | 8 +++- embedding/commentfilter/go_filter_test.go | 2 + .../commentfilter/marker_comment_filter.go | 48 +++++++++++++------ 3 files changed, 42 insertions(+), 16 deletions(-) diff --git a/embedding/commentfilter/config.go b/embedding/commentfilter/config.go index 90e24a2..56a662c 100644 --- a/embedding/commentfilter/config.go +++ b/embedding/commentfilter/config.go @@ -101,7 +101,9 @@ var javaSyntax = CommentMarker{ Documentation: DocumentationMarker{ Block: []BlockMarker{{Start: cStyleDocCommentStart, End: cStyleBlockCommentEnd}}, }, - TextBlocks: []string{javaTextBlockDelimiter}, + TextBlocks: []TextBlockMarker{ + {Delimiter: javaTextBlockDelimiter, Escapes: true}, + }, QuoteChars: "\"'", } @@ -130,7 +132,9 @@ var goSyntax = CommentMarker{ Block: []BlockMarker{ {Start: cStyleBlockCommentStart, End: cStyleBlockCommentEnd}, }, - TextBlocks: []string{goRawStringDelimiter}, + TextBlocks: []TextBlockMarker{ + {Delimiter: goRawStringDelimiter}, + }, QuoteChars: goQuoteChars, } diff --git a/embedding/commentfilter/go_filter_test.go b/embedding/commentfilter/go_filter_test.go index 6019b2d..4f26464 100644 --- a/embedding/commentfilter/go_filter_test.go +++ b/embedding/commentfilter/go_filter_test.go @@ -34,6 +34,7 @@ var _ = Describe("Go", func() { "const slash = '/'", "const url = \"http://example.org\"", "const raw = `Keep // and /* markers */ in raw strings`", + "const path = `C:\\Users\\`", "const multi = `", "Keep // and /* markers */ across lines", "`", @@ -47,6 +48,7 @@ var _ = Describe("Go", func() { "const slash = '/'", "const url = \"http://example.org\"", "const raw = `Keep // and /* markers */ in raw strings`", + "const path = `C:\\Users\\`", "const multi = `", "Keep // and /* markers */ across lines", "`", diff --git a/embedding/commentfilter/marker_comment_filter.go b/embedding/commentfilter/marker_comment_filter.go index d764aa5..d245331 100644 --- a/embedding/commentfilter/marker_comment_filter.go +++ b/embedding/commentfilter/marker_comment_filter.go @@ -38,6 +38,15 @@ type DocumentationMarker struct { Block []BlockMarker } +// TextBlockMarker describes a multi-line text literal delimiter. +type TextBlockMarker struct { + // Delimiter opens and closes the text literal. + Delimiter string + + // Escapes reports whether backslashes escape delimiter bytes. + Escapes bool +} + // CommentMarker describes lexical comment markers and string delimiters for a language family. type CommentMarker struct { // Inline contains line-comment markers. @@ -49,8 +58,8 @@ type CommentMarker struct { // Documentation contains API documentation comment markers. Documentation DocumentationMarker - // TextBlocks contains delimiters that open and close multi-line text literals. - TextBlocks []string + // TextBlocks contains markers that open and close multi-line text literals. + TextBlocks []TextBlockMarker // QuoteChars contains characters that open and close quoted strings. QuoteChars string @@ -76,8 +85,8 @@ type blockState struct { // textBlockActive reports whether scanning is inside a text block. textBlockActive bool - // textBlockDelimiter contains the marker that closes the active text block. - textBlockDelimiter string + // textBlock contains the active text block marker. + textBlock TextBlockMarker } // markerLineFilter tracks lexical comment filtering state for one source line. @@ -199,7 +208,7 @@ func (f *markerLineFilter) consumeActiveTextBlock() bool { if !f.state.textBlockActive { return false } - endPosition, found := textBlockEnd(f.line, f.position, f.state.textBlockDelimiter) + endPosition, found := textBlockEnd(f.line, f.position, f.state.textBlock) if !found { f.result.WriteString(f.line[f.position:]) f.position = len(f.line) @@ -209,21 +218,21 @@ func (f *markerLineFilter) consumeActiveTextBlock() bool { f.result.WriteString(f.line[f.position:endPosition]) f.position = endPosition f.state.textBlockActive = false - f.state.textBlockDelimiter = "" + f.state.textBlock = TextBlockMarker{} return true } // textBlockEnd returns the end offset of a text block close delimiter. -func textBlockEnd(line string, position int, delimiter string) (int, bool) { +func textBlockEnd(line string, position int, marker TextBlockMarker) (int, bool) { for cursor := position; cursor < len(line); { - if line[cursor] == '\\' { + if marker.Escapes && line[cursor] == '\\' { cursor += 2 continue } - if strings.HasPrefix(line[cursor:], delimiter) { - return cursor + len(delimiter), true + if strings.HasPrefix(line[cursor:], marker.Delimiter) { + return cursor + len(marker.Delimiter), true } cursor++ } @@ -233,14 +242,14 @@ func textBlockEnd(line string, position int, delimiter string) (int, bool) { // consumeTextBlockStart starts a configured text block literal. func (f *markerLineFilter) consumeTextBlockStart() bool { - delimiter, found := prefixFrom(f.line, f.position, f.filter.Syntax.TextBlocks) + marker, found := textBlockAt(f.line, f.position, f.filter.Syntax.TextBlocks) if !found { return false } - f.result.WriteString(delimiter) - f.position += len(delimiter) + f.result.WriteString(marker.Delimiter) + f.position += len(marker.Delimiter) f.state.textBlockActive = true - f.state.textBlockDelimiter = delimiter + f.state.textBlock = marker return true } @@ -346,6 +355,17 @@ func prefixFrom(line string, position int, prefixes []string) (string, bool) { return "", false } +// textBlockAt reports whether one of the given text block markers starts at the position. +func textBlockAt(line string, position int, markers []TextBlockMarker) (TextBlockMarker, bool) { + for _, marker := range markers { + if strings.HasPrefix(line[position:], marker.Delimiter) { + return marker, true + } + } + + return TextBlockMarker{}, false +} + // blockAt reports whether one of the given block markers starts at the position. func blockAt(line string, position int, blocks []BlockMarker) (BlockMarker, bool) { for _, block := range blocks {