Fix literal "None" output when charset detection fails in PlainTextConverter and CsvConverter

packages/markitdown/src/markitdown/converters/_csv_converter.py

@@ -45,7 +45,15 @@ def convert(
         if stream_info.charset:
             content = file_stream.read().decode(stream_info.charset)
         else:
-            content = str(from_bytes(file_stream.read()).best())
+            data = file_stream.read()
+            best_guess = from_bytes(data).best()
+            if best_guess is not None:
+                content = str(best_guess)
+            else:
+                # charset_normalizer could not classify the bytes; decode as
+                # UTF-8 with replacement rather than stringifying None, which
+                # would silently turn the document into the literal "None".
+                content = data.decode("utf-8", errors="replace")
 
         # Parse CSV content
         reader = csv.reader(io.StringIO(content))

packages/markitdown/src/markitdown/converters/_plain_text_converter.py

@@ -66,6 +66,14 @@ def convert(
         if stream_info.charset:
             text_content = file_stream.read().decode(stream_info.charset)
         else:
-            text_content = str(from_bytes(file_stream.read()).best())
+            data = file_stream.read()
+            best_guess = from_bytes(data).best()
+            if best_guess is not None:
+                text_content = str(best_guess)
+            else:
+                # charset_normalizer could not classify the bytes; decode as
+                # UTF-8 with replacement rather than stringifying None, which
+                # would silently turn the document into the literal "None".
+                text_content = data.decode("utf-8", errors="replace")
 
         return DocumentConverterResult(markdown=text_content)

packages/markitdown/tests/test_module_misc.py

@@ -288,6 +288,23 @@ def test_input_as_strings() -> None:
     assert "# Test" in result.text_content
 
 
+def test_undetectable_charset_does_not_become_none() -> None:
+    """When charset_normalizer cannot classify the bytes (best() returns None),
+    the converters must not stringify None and present the literal "None" as
+    the converted document content."""
+    markitdown = MarkItDown()
+
+    # A corrupt/truncated UTF-16-looking byte sequence that charset_normalizer
+    # cannot classify: from_bytes(...).best() returns None for these bytes.
+    undetectable = b"\xff\xfe\xff\xff\x00"
+
+    result = markitdown.convert_stream(io.BytesIO(undetectable), file_extension=".txt")
+    assert result.markdown != "None"
+
+    result = markitdown.convert_stream(io.BytesIO(undetectable), file_extension=".csv")
+    assert "None" not in result.markdown
+
+
 def test_pptx_chart_multi_series_conversion() -> None:
     """Charts with multiple series and many categories must convert correctly.