From b279b8cb5f91190bdf82dd249769f6a8c3bce959 Mon Sep 17 00:00:00 2001 From: Vladyslav Kuksiuk Date: Fri, 3 Jul 2026 12:55:30 +0200 Subject: [PATCH 1/4] Improve text block comments stripping. --- embedding/commentfilter/config.go | 2 + embedding/commentfilter/filter_test.go | 21 ++++++ .../commentfilter/marker_comment_filter.go | 65 +++++++++++++++++-- 3 files changed, 84 insertions(+), 4 deletions(-) diff --git a/embedding/commentfilter/config.go b/embedding/commentfilter/config.go index f900b55..e4df86f 100644 --- a/embedding/commentfilter/config.go +++ b/embedding/commentfilter/config.go @@ -22,6 +22,7 @@ const ( cStyleBlockCommentStart = "/*" cStyleBlockCommentEnd = "*/" cStyleDocCommentStart = "/**" + javaTextBlockDelimiter = "\"\"\"" jsQuoteChars = "\"'`" ) @@ -98,6 +99,7 @@ var javaSyntax = CommentMarker{ Documentation: DocumentationMarker{ Block: []BlockMarker{{Start: cStyleDocCommentStart, End: cStyleBlockCommentEnd}}, }, + TextBlocks: []string{javaTextBlockDelimiter}, QuoteChars: "\"'", } diff --git a/embedding/commentfilter/filter_test.go b/embedding/commentfilter/filter_test.go index e98e3b8..85f72b0 100644 --- a/embedding/commentfilter/filter_test.go +++ b/embedding/commentfilter/filter_test.go @@ -117,6 +117,27 @@ var _ = Describe("Comment filter", func() { assertFiltered("Api.java", RetainRegular, lines, expected) }) + + It("should strip comments without treating text block content as comments", func() { + lines := []string{ + "// header comment", + "String help = \"\"\"", + " Keep this // text.", + " Keep this /* text */ too.", + " \"\"\";", + "String value = \"not a // comment\"; // inline comment", + } + + expected := []string{ + "String help = \"\"\"", + " Keep this // text.", + " Keep this /* text */ too.", + " \"\"\";", + "String value = \"not a // comment\"; ", + } + + assertFiltered("Api.java", RetainNone, lines, expected) + }) }) Describe("Kotlin", func() { diff --git a/embedding/commentfilter/marker_comment_filter.go b/embedding/commentfilter/marker_comment_filter.go index 8f49222..2855596 100644 --- a/embedding/commentfilter/marker_comment_filter.go +++ b/embedding/commentfilter/marker_comment_filter.go @@ -49,6 +49,9 @@ type CommentMarker struct { // Documentation contains API documentation comment markers. Documentation DocumentationMarker + // TextBlocks contains delimiters that open and close multi-line text literals. + TextBlocks []string + // QuoteChars contains characters that open and close quoted strings. QuoteChars string } @@ -59,7 +62,7 @@ type MarkerCommentFilter struct { Syntax CommentMarker } -// blockState tracks an active block comment across source lines. +// blockState tracks active multi-line lexical constructs across source lines. type blockState struct { // active reports whether scanning is inside a block comment. active bool @@ -69,6 +72,12 @@ type blockState struct { // keep reports whether the active comment should be retained. keep bool + + // textBlockActive reports whether scanning is inside a text block. + textBlockActive bool + + // textBlockDelimiter contains the marker that closes the active text block. + textBlockDelimiter string } // markerLineFilter tracks lexical comment filtering state for one source line. @@ -82,7 +91,7 @@ type markerLineFilter struct { // mode selects which comments to retain. mode Mode - // state tracks block comments across lines. + // state tracks multi-line lexical constructs across lines. state *blockState // result accumulates the filtered source line. @@ -138,6 +147,12 @@ func (f *markerLineFilter) filterLine() (string, bool) { if f.consumeActiveBlock() { continue } + if f.consumeActiveTextBlock() { + continue + } + if f.consumeTextBlockStart() { + continue + } if f.consumeQuotedSegment() { continue } @@ -179,6 +194,41 @@ func (f *markerLineFilter) consumeActiveBlock() bool { return true } +// consumeActiveTextBlock copies text block content until the closing delimiter. +func (f *markerLineFilter) consumeActiveTextBlock() bool { + if !f.state.textBlockActive { + return false + } + end := strings.Index(f.line[f.position:], f.state.textBlockDelimiter) + if end < 0 { + f.result.WriteString(f.line[f.position:]) + f.position = len(f.line) + + return true + } + endPosition := f.position + end + len(f.state.textBlockDelimiter) + f.result.WriteString(f.line[f.position:endPosition]) + f.position = endPosition + f.state.textBlockActive = false + f.state.textBlockDelimiter = "" + + return true +} + +// consumeTextBlockStart starts a configured text block literal. +func (f *markerLineFilter) consumeTextBlockStart() bool { + delimiter, found := prefixFrom(f.line, f.position, f.filter.Syntax.TextBlocks) + if !found { + return false + } + f.result.WriteString(delimiter) + f.position += len(delimiter) + f.state.textBlockActive = true + f.state.textBlockDelimiter = delimiter + + return true +} + // consumeQuotedSegment copies a quoted segment without scanning comment markers inside it. func (f *markerLineFilter) consumeQuotedSegment() bool { quoteEnd := quotedSegmentEnd(f.line, f.position, f.filter.Syntax.QuoteChars) @@ -264,13 +314,20 @@ func (f *markerLineFilter) consumeCodeByte() { // prefixAt reports whether one of the given prefixes starts at the position. func prefixAt(line string, position int, prefixes []string) bool { + _, found := prefixFrom(line, position, prefixes) + + return found +} + +// prefixFrom returns the prefix starting at position when one exists. +func prefixFrom(line string, position int, prefixes []string) (string, bool) { for _, prefix := range prefixes { if strings.HasPrefix(line[position:], prefix) { - return true + return prefix, true } } - return false + return "", false } // blockAt reports whether one of the given block markers starts at the position. From c1e982c0c7e1ca2a10306d0c8b98a05c8138aea6 Mon Sep 17 00:00:00 2001 From: Vladyslav Kuksiuk Date: Fri, 3 Jul 2026 13:36:44 +0200 Subject: [PATCH 2/4] Improve names. --- embedding/commentfilter/{kotlin.go => kotlin_filter.go} | 0 .../commentfilter/{visual_basic.go => visual_basic_filter.go} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename embedding/commentfilter/{kotlin.go => kotlin_filter.go} (100%) rename embedding/commentfilter/{visual_basic.go => visual_basic_filter.go} (100%) diff --git a/embedding/commentfilter/kotlin.go b/embedding/commentfilter/kotlin_filter.go similarity index 100% rename from embedding/commentfilter/kotlin.go rename to embedding/commentfilter/kotlin_filter.go diff --git a/embedding/commentfilter/visual_basic.go b/embedding/commentfilter/visual_basic_filter.go similarity index 100% rename from embedding/commentfilter/visual_basic.go rename to embedding/commentfilter/visual_basic_filter.go From d8a14fc35c4edd8b5ba4034c088e196ea134c2ba Mon Sep 17 00:00:00 2001 From: Vladyslav Kuksiuk Date: Fri, 3 Jul 2026 13:44:46 +0200 Subject: [PATCH 3/4] Improve escape handling. --- embedding/commentfilter/filter_test.go | 20 +++++++++++++++++ .../commentfilter/marker_comment_filter.go | 22 ++++++++++++++++--- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/embedding/commentfilter/filter_test.go b/embedding/commentfilter/filter_test.go index 25ac115..1c0906e 100644 --- a/embedding/commentfilter/filter_test.go +++ b/embedding/commentfilter/filter_test.go @@ -138,6 +138,26 @@ var _ = Describe("Comment filter", func() { assertFiltered("Api.java", RetainNone, lines, expected) }) + + It("should not close text blocks on escaped triple quotes", func() { + lines := []string{ + "String help = \"\"\"", + ` Quote: \"""`, + " Keep this // text.", + " \"\"\";", + "String value = \"kept\"; // real comment", + } + + expected := []string{ + "String help = \"\"\"", + ` Quote: \"""`, + " Keep this // text.", + " \"\"\";", + "String value = \"kept\"; ", + } + + assertFiltered("Api.java", RetainNone, lines, expected) + }) }) Describe("Kotlin", func() { diff --git a/embedding/commentfilter/marker_comment_filter.go b/embedding/commentfilter/marker_comment_filter.go index 2855596..d764aa5 100644 --- a/embedding/commentfilter/marker_comment_filter.go +++ b/embedding/commentfilter/marker_comment_filter.go @@ -199,14 +199,13 @@ func (f *markerLineFilter) consumeActiveTextBlock() bool { if !f.state.textBlockActive { return false } - end := strings.Index(f.line[f.position:], f.state.textBlockDelimiter) - if end < 0 { + endPosition, found := textBlockEnd(f.line, f.position, f.state.textBlockDelimiter) + if !found { f.result.WriteString(f.line[f.position:]) f.position = len(f.line) return true } - endPosition := f.position + end + len(f.state.textBlockDelimiter) f.result.WriteString(f.line[f.position:endPosition]) f.position = endPosition f.state.textBlockActive = false @@ -215,6 +214,23 @@ func (f *markerLineFilter) consumeActiveTextBlock() bool { return true } +// textBlockEnd returns the end offset of a text block close delimiter. +func textBlockEnd(line string, position int, delimiter string) (int, bool) { + for cursor := position; cursor < len(line); { + if line[cursor] == '\\' { + cursor += 2 + + continue + } + if strings.HasPrefix(line[cursor:], delimiter) { + return cursor + len(delimiter), true + } + cursor++ + } + + return len(line), false +} + // consumeTextBlockStart starts a configured text block literal. func (f *markerLineFilter) consumeTextBlockStart() bool { delimiter, found := prefixFrom(f.line, f.position, f.filter.Syntax.TextBlocks) From 3a7896adbd4cec915aad21801859f32ea4580789 Mon Sep 17 00:00:00 2001 From: Vladyslav Kuksiuk Date: Fri, 3 Jul 2026 14:24:34 +0200 Subject: [PATCH 4/4] Improve tests. --- embedding/commentfilter/filter_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/embedding/commentfilter/filter_test.go b/embedding/commentfilter/filter_test.go index 1c0906e..8c23506 100644 --- a/embedding/commentfilter/filter_test.go +++ b/embedding/commentfilter/filter_test.go @@ -143,6 +143,7 @@ var _ = Describe("Comment filter", func() { lines := []string{ "String help = \"\"\"", ` Quote: \"""`, + ` Escaped quote: \"`, " Keep this // text.", " \"\"\";", "String value = \"kept\"; // real comment", @@ -151,6 +152,7 @@ var _ = Describe("Comment filter", func() { expected := []string{ "String help = \"\"\"", ` Quote: \"""`, + ` Escaped quote: \"`, " Keep this // text.", " \"\"\";", "String value = \"kept\"; ",