diff --git a/embedding/commentfilter/config.go b/embedding/commentfilter/config.go index 0b023d6..906f4c5 100644 --- a/embedding/commentfilter/config.go +++ b/embedding/commentfilter/config.go @@ -23,6 +23,8 @@ const ( cStyleBlockCommentEnd = "*/" cStyleDocCommentStart = "/**" javaTextBlockDelimiter = "\"\"\"" + pythonDoubleQuoteBlock = "\"\"\"" + pythonSingleQuoteBlock = "'''" goRawStringDelimiter = "`" goQuoteChars = "\"'" jsQuoteChars = "\"'`" @@ -62,9 +64,9 @@ var filtersByExtension = map[string]filterEntry{ ".proto": filterConfig(MarkerCommentFilter{Syntax: cStyleSyntax}, regularModes), // Python - ".py": filterConfig(MarkerCommentFilter{Syntax: hashLineSyntax}, noneMode), - ".pyi": filterConfig(MarkerCommentFilter{Syntax: hashLineSyntax}, noneMode), - ".pyw": filterConfig(MarkerCommentFilter{Syntax: hashLineSyntax}, noneMode), + ".py": filterConfig(MarkerCommentFilter{Syntax: pythonSyntax}, noneMode), + ".pyi": filterConfig(MarkerCommentFilter{Syntax: pythonSyntax}, noneMode), + ".pyw": filterConfig(MarkerCommentFilter{Syntax: pythonSyntax}, noneMode), // YAML ".yml": filterConfig(MarkerCommentFilter{Syntax: hashLineSyntax}, noneMode), @@ -131,6 +133,15 @@ var hashLineSyntax = CommentMarker{ QuoteChars: "\"'", } +var pythonSyntax = CommentMarker{ + Inline: []string{"#"}, + TextBlocks: []TextBlockMarker{ + {Delimiter: pythonDoubleQuoteBlock, Escapes: true}, + {Delimiter: pythonSingleQuoteBlock, Escapes: true}, + }, + QuoteChars: "\"'", +} + var xmlSyntax = CommentMarker{ Block: []BlockMarker{ {Start: ""}, diff --git a/embedding/commentfilter/python_filter_test.go b/embedding/commentfilter/python_filter_test.go index a945342..ad39fc9 100644 --- a/embedding/commentfilter/python_filter_test.go +++ b/embedding/commentfilter/python_filter_test.go @@ -39,4 +39,33 @@ var _ = Describe("Python", func() { assertFiltered("module.py", RetainNone, lines, expected) }) + + It("should strip comments without treating triple-quoted string content as comments", func() { + lines := []string{ + "# module comment", + "def message():", + " \"\"\"", + " Keep this # docstring text.", + " \"\"\"", + ` escaped = """a\"""" # inline comment`, + " value = '''", + " Keep this # multiline text.", + " ''' # inline comment", + " return value # real comment", + } + + expected := []string{ + "def message():", + " \"\"\"", + " Keep this # docstring text.", + " \"\"\"", + ` escaped = """a\"""" `, + " value = '''", + " Keep this # multiline text.", + " ''' ", + " return value ", + } + + assertFiltered("module.py", RetainNone, lines, expected) + }) })