From 615f7e304ac8b285b5154b7b950b3c7220806da2 Mon Sep 17 00:00:00 2001 From: zhengyu chen Date: Sun, 15 Nov 2026 23:04:37 +0000 Subject: [PATCH 1/2] Default insert-files text decoding to UTF-8 --- sqlite_utils/cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index c23090283..55ba25650 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -2899,6 +2899,7 @@ def extract( @click.option("--text", is_flag=True, help="Store file content as TEXT, not BLOB") @click.option( "--encoding", + default="utf-8", help="Character encoding for input, defaults to utf-8", ) @click.option("-s", "--silent", is_flag=True, help="Don't show a progress bar") From 9bd36a53d8b087bce569cfb746ca5b3842362430 Mon Sep 17 00:00:00 2001 From: zhengyu chen Date: Sun, 15 Nov 2026 23:05:08 +0000 Subject: [PATCH 2/2] Test insert-files UTF-8 default under the C locale --- tests/test_insert_files.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/test_insert_files.py b/tests/test_insert_files.py index 93c4daf1b..ca40faa0e 100644 --- a/tests/test_insert_files.py +++ b/tests/test_insert_files.py @@ -1,5 +1,6 @@ import os import pathlib +import subprocess import sys import pytest @@ -151,6 +152,38 @@ def test_insert_files_stdin(use_text, encoding, input, expected): assert {"path": "stdin-name", key: expected}.items() <= row.items() +@pytest.mark.skipif( + sys.platform.startswith("win"), reason="Requires the POSIX C locale" +) +def test_insert_files_uses_utf8_by_default_in_ascii_locale(tmp_path): + source = tmp_path / "input.txt" + source.write_text("São Paulo", encoding="utf-8") + db_path = tmp_path / "files.db" + result = subprocess.run( + [ + sys.executable, + "-m", + "sqlite_utils", + "insert-files", + str(db_path), + "files", + str(source), + "--text", + "--silent", + ], + env={ + **os.environ, + "LC_ALL": "C", + "PYTHONCOERCECLOCALE": "0", + "PYTHONUTF8": "0", + }, + capture_output=True, + ) + assert result.returncode == 0, result.stderr + with Database(db_path) as db: + assert next(db.table("files").rows)["content_text"] == "São Paulo" + + @pytest.mark.skipif( sys.platform.startswith("win"), reason="Windows has a different way of handling default encodings",