From 76e9cf1db23b47fd66bf061d4afdd113e427a9da Mon Sep 17 00:00:00 2001 From: Vladyslav Kuksiuk Date: Mon, 6 Jul 2026 16:20:53 +0200 Subject: [PATCH 1/2] Improve python comment filtering. --- embedding/commentfilter/config.go | 17 +++++++++--- embedding/commentfilter/python_filter_test.go | 27 +++++++++++++++++++ 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/embedding/commentfilter/config.go b/embedding/commentfilter/config.go index 0b023d6..906f4c5 100644 --- a/embedding/commentfilter/config.go +++ b/embedding/commentfilter/config.go @@ -23,6 +23,8 @@ const ( cStyleBlockCommentEnd = "*/" cStyleDocCommentStart = "/**" javaTextBlockDelimiter = "\"\"\"" + pythonDoubleQuoteBlock = "\"\"\"" + pythonSingleQuoteBlock = "'''" goRawStringDelimiter = "`" goQuoteChars = "\"'" jsQuoteChars = "\"'`" @@ -62,9 +64,9 @@ var filtersByExtension = map[string]filterEntry{ ".proto": filterConfig(MarkerCommentFilter{Syntax: cStyleSyntax}, regularModes), // Python - ".py": filterConfig(MarkerCommentFilter{Syntax: hashLineSyntax}, noneMode), - ".pyi": filterConfig(MarkerCommentFilter{Syntax: hashLineSyntax}, noneMode), - ".pyw": filterConfig(MarkerCommentFilter{Syntax: hashLineSyntax}, noneMode), + ".py": filterConfig(MarkerCommentFilter{Syntax: pythonSyntax}, noneMode), + ".pyi": filterConfig(MarkerCommentFilter{Syntax: pythonSyntax}, noneMode), + ".pyw": filterConfig(MarkerCommentFilter{Syntax: pythonSyntax}, noneMode), // YAML ".yml": filterConfig(MarkerCommentFilter{Syntax: hashLineSyntax}, noneMode), @@ -131,6 +133,15 @@ var hashLineSyntax = CommentMarker{ QuoteChars: "\"'", } +var pythonSyntax = CommentMarker{ + Inline: []string{"#"}, + TextBlocks: []TextBlockMarker{ + {Delimiter: pythonDoubleQuoteBlock, Escapes: true}, + {Delimiter: pythonSingleQuoteBlock, Escapes: true}, + }, + QuoteChars: "\"'", +} + var xmlSyntax = CommentMarker{ Block: []BlockMarker{ {Start: ""}, diff --git a/embedding/commentfilter/python_filter_test.go b/embedding/commentfilter/python_filter_test.go index a945342..69ef4f5 100644 --- a/embedding/commentfilter/python_filter_test.go +++ b/embedding/commentfilter/python_filter_test.go @@ -39,4 +39,31 @@ var _ = Describe("Python", func() { assertFiltered("module.py", RetainNone, lines, expected) }) + + It("should strip comments without treating triple-quoted string content as comments", func() { + lines := []string{ + "# module comment", + "def message():", + " \"\"\"", + " Keep this # docstring text.", + " \"\"\"", + " value = '''", + " Keep this # multiline text.", + " ''' # inline comment", + " return value # real comment", + } + + expected := []string{ + "def message():", + " \"\"\"", + " Keep this # docstring text.", + " \"\"\"", + " value = '''", + " Keep this # multiline text.", + " ''' ", + " return value ", + } + + assertFiltered("module.py", RetainNone, lines, expected) + }) }) From d213346468e13db6aebdfb54a42b8a3df84a42b1 Mon Sep 17 00:00:00 2001 From: Vladyslav Kuksiuk Date: Mon, 6 Jul 2026 16:35:13 +0200 Subject: [PATCH 2/2] Improve tests. --- embedding/commentfilter/python_filter_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/embedding/commentfilter/python_filter_test.go b/embedding/commentfilter/python_filter_test.go index 69ef4f5..ad39fc9 100644 --- a/embedding/commentfilter/python_filter_test.go +++ b/embedding/commentfilter/python_filter_test.go @@ -47,6 +47,7 @@ var _ = Describe("Python", func() { " \"\"\"", " Keep this # docstring text.", " \"\"\"", + ` escaped = """a\"""" # inline comment`, " value = '''", " Keep this # multiline text.", " ''' # inline comment", @@ -58,6 +59,7 @@ var _ = Describe("Python", func() { " \"\"\"", " Keep this # docstring text.", " \"\"\"", + ` escaped = """a\"""" `, " value = '''", " Keep this # multiline text.", " ''' ",