diff --git a/packages/markitdown/src/markitdown/converters/_csv_converter.py b/packages/markitdown/src/markitdown/converters/_csv_converter.py index 7e9631e1b..b11417e03 100644 --- a/packages/markitdown/src/markitdown/converters/_csv_converter.py +++ b/packages/markitdown/src/markitdown/converters/_csv_converter.py @@ -45,7 +45,15 @@ def convert( if stream_info.charset: content = file_stream.read().decode(stream_info.charset) else: - content = str(from_bytes(file_stream.read()).best()) + data = file_stream.read() + best_guess = from_bytes(data).best() + if best_guess is not None: + content = str(best_guess) + else: + # charset_normalizer could not classify the bytes; decode as + # UTF-8 with replacement rather than stringifying None, which + # would silently turn the document into the literal "None". + content = data.decode("utf-8", errors="replace") # Parse CSV content reader = csv.reader(io.StringIO(content)) diff --git a/packages/markitdown/src/markitdown/converters/_plain_text_converter.py b/packages/markitdown/src/markitdown/converters/_plain_text_converter.py index 6f1306fe8..6cccc2eef 100644 --- a/packages/markitdown/src/markitdown/converters/_plain_text_converter.py +++ b/packages/markitdown/src/markitdown/converters/_plain_text_converter.py @@ -66,6 +66,14 @@ def convert( if stream_info.charset: text_content = file_stream.read().decode(stream_info.charset) else: - text_content = str(from_bytes(file_stream.read()).best()) + data = file_stream.read() + best_guess = from_bytes(data).best() + if best_guess is not None: + text_content = str(best_guess) + else: + # charset_normalizer could not classify the bytes; decode as + # UTF-8 with replacement rather than stringifying None, which + # would silently turn the document into the literal "None". + text_content = data.decode("utf-8", errors="replace") return DocumentConverterResult(markdown=text_content) diff --git a/packages/markitdown/tests/test_module_misc.py b/packages/markitdown/tests/test_module_misc.py index f966eee1e..af8175514 100644 --- a/packages/markitdown/tests/test_module_misc.py +++ b/packages/markitdown/tests/test_module_misc.py @@ -288,6 +288,23 @@ def test_input_as_strings() -> None: assert "# Test" in result.text_content +def test_undetectable_charset_does_not_become_none() -> None: + """When charset_normalizer cannot classify the bytes (best() returns None), + the converters must not stringify None and present the literal "None" as + the converted document content.""" + markitdown = MarkItDown() + + # A corrupt/truncated UTF-16-looking byte sequence that charset_normalizer + # cannot classify: from_bytes(...).best() returns None for these bytes. + undetectable = b"\xff\xfe\xff\xff\x00" + + result = markitdown.convert_stream(io.BytesIO(undetectable), file_extension=".txt") + assert result.markdown != "None" + + result = markitdown.convert_stream(io.BytesIO(undetectable), file_extension=".csv") + assert "None" not in result.markdown + + def test_pptx_chart_multi_series_conversion() -> None: """Charts with multiple series and many categories must convert correctly.