From cda759dfea1303306c6ad7ecd0be34c16aba7d8c Mon Sep 17 00:00:00 2001 From: Sohum Trivedi Date: Fri, 17 Jul 2026 10:37:41 -0700 Subject: [PATCH 1/2] Fix literal "None" output when charset detection fails in plain text and CSV converters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When no charset hint is available, PlainTextConverter and CsvConverter decode via str(from_bytes(data).best()). charset_normalizer's best() returns None for byte sequences it cannot classify (e.g. a corrupt or truncated UTF-16 file), so str(None) silently replaced the entire document with the 4-character string "None" — a .txt converted to the markdown 'None' and a .csv to the bogus table '| None |', with no error raised. Guard the best() result: when it is None, fall back to decoding as UTF-8 with errors="replace", which preserves whatever is decodable and never fabricates content. Add a regression test covering both converters. --- .../src/markitdown/converters/_csv_converter.py | 10 +++++++++- .../converters/_plain_text_converter.py | 10 +++++++++- packages/markitdown/tests/test_module_misc.py | 17 +++++++++++++++++ 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/packages/markitdown/src/markitdown/converters/_csv_converter.py b/packages/markitdown/src/markitdown/converters/_csv_converter.py index 7e9631e1b..b11417e03 100644 --- a/packages/markitdown/src/markitdown/converters/_csv_converter.py +++ b/packages/markitdown/src/markitdown/converters/_csv_converter.py @@ -45,7 +45,15 @@ def convert( if stream_info.charset: content = file_stream.read().decode(stream_info.charset) else: - content = str(from_bytes(file_stream.read()).best()) + data = file_stream.read() + best_guess = from_bytes(data).best() + if best_guess is not None: + content = str(best_guess) + else: + # charset_normalizer could not classify the bytes; decode as + # UTF-8 with replacement rather than stringifying None, which + # would silently turn the document into the literal "None". + content = data.decode("utf-8", errors="replace") # Parse CSV content reader = csv.reader(io.StringIO(content)) diff --git a/packages/markitdown/src/markitdown/converters/_plain_text_converter.py b/packages/markitdown/src/markitdown/converters/_plain_text_converter.py index 6f1306fe8..6cccc2eef 100644 --- a/packages/markitdown/src/markitdown/converters/_plain_text_converter.py +++ b/packages/markitdown/src/markitdown/converters/_plain_text_converter.py @@ -66,6 +66,14 @@ def convert( if stream_info.charset: text_content = file_stream.read().decode(stream_info.charset) else: - text_content = str(from_bytes(file_stream.read()).best()) + data = file_stream.read() + best_guess = from_bytes(data).best() + if best_guess is not None: + text_content = str(best_guess) + else: + # charset_normalizer could not classify the bytes; decode as + # UTF-8 with replacement rather than stringifying None, which + # would silently turn the document into the literal "None". + text_content = data.decode("utf-8", errors="replace") return DocumentConverterResult(markdown=text_content) diff --git a/packages/markitdown/tests/test_module_misc.py b/packages/markitdown/tests/test_module_misc.py index 4d62e4919..41fd2c0c0 100644 --- a/packages/markitdown/tests/test_module_misc.py +++ b/packages/markitdown/tests/test_module_misc.py @@ -288,6 +288,23 @@ def test_input_as_strings() -> None: assert "# Test" in result.text_content +def test_undetectable_charset_does_not_become_none() -> None: + """When charset_normalizer cannot classify the bytes (best() returns None), + the converters must not stringify None and present the literal "None" as + the converted document content.""" + markitdown = MarkItDown() + + # A corrupt/truncated UTF-16-looking byte sequence that charset_normalizer + # cannot classify: from_bytes(...).best() returns None for these bytes. + undetectable = b"\xff\xfe\xff\xff\x00" + + result = markitdown.convert_stream(io.BytesIO(undetectable), file_extension=".txt") + assert result.markdown != "None" + + result = markitdown.convert_stream(io.BytesIO(undetectable), file_extension=".csv") + assert "None" not in result.markdown + + def test_deeply_nested_html_fallback() -> None: """Large, deeply nested HTML should fall back to plain-text extraction instead of silently returning unconverted HTML (issue #1636). From ca0a3f5bca1f3a49ef6d6b1882e7e1ecafcdbba3 Mon Sep 17 00:00:00 2001 From: Sohum Trivedi Date: Tue, 21 Jul 2026 18:46:40 -0700 Subject: [PATCH 2/2] Apply black formatting to test_module_misc.py --- packages/markitdown/tests/test_module_misc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/markitdown/tests/test_module_misc.py b/packages/markitdown/tests/test_module_misc.py index 31cf27731..af8175514 100644 --- a/packages/markitdown/tests/test_module_misc.py +++ b/packages/markitdown/tests/test_module_misc.py @@ -303,8 +303,8 @@ def test_undetectable_charset_does_not_become_none() -> None: result = markitdown.convert_stream(io.BytesIO(undetectable), file_extension=".csv") assert "None" not in result.markdown - - + + def test_pptx_chart_multi_series_conversion() -> None: """Charts with multiple series and many categories must convert correctly.