From 274b1ef1a3b21e52707b2be0154ea564d9a0517d Mon Sep 17 00:00:00 2001 From: Nash Aschenbath Date: Tue, 3 Nov 2026 21:11:57 +0800 Subject: [PATCH 1/3] =?UTF-8?q?=F0=9F=A7=AA=20Read=20the=20CommonMark=20sp?= =?UTF-8?q?ec=20fixtures=20as=20UTF-8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `test_spec.py` and `get_cmark_spec.py` call `Path.read_text()` without an encoding, so on Windows with a non-UTF-8 locale (e.g. GBK on Chinese Windows) collecting the spec tests fails with UnicodeDecodeError and the whole module errors out. The fixtures are UTF-8, so read them as such. Co-Authored-By: Claude Opus 5.5 --- tests/test_cmark_spec/get_cmark_spec.py | 4 ++-- tests/test_cmark_spec/test_spec.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_cmark_spec/get_cmark_spec.py b/tests/test_cmark_spec/get_cmark_spec.py index 2883835e..0d7ba9c9 100644 --- a/tests/test_cmark_spec/get_cmark_spec.py +++ b/tests/test_cmark_spec/get_cmark_spec.py @@ -72,7 +72,7 @@ def _json_to_fixture(data: list[dict[str, Any]]) -> str: print(f"Downloading CommonMark spec from {url}") response = requests.get(url) response.raise_for_status() - if not output_path.exists() or output_path.read_text() != response.text: + if not output_path.exists() or output_path.read_text(encoding="utf-8") != response.text: changed = True with output_path.open("w") as f: f.write(response.text) @@ -85,7 +85,7 @@ def _json_to_fixture(data: list[dict[str, Any]]) -> str: response.raise_for_status() data = response.json() text = _json_to_fixture(data) - if not test_fixture.exists() or test_fixture.read_text() != text: + if not test_fixture.exists() or test_fixture.read_text(encoding="utf-8") != text: changed = True with test_fixture.open("w") as f: f.write(text) diff --git a/tests/test_cmark_spec/test_spec.py b/tests/test_cmark_spec/test_spec.py index e5199477..83a4d548 100644 --- a/tests/test_cmark_spec/test_spec.py +++ b/tests/test_cmark_spec/test_spec.py @@ -15,10 +15,10 @@ def test_file(file_regression): md = MarkdownIt("commonmark") - file_regression.check(md.render(SPEC_INPUT.read_text()), extension=".html") + file_regression.check(md.render(SPEC_INPUT.read_text(encoding="utf-8")), extension=".html") -@pytest.mark.parametrize("entry", json.loads(TESTS_INPUT.read_text())) +@pytest.mark.parametrize("entry", json.loads(TESTS_INPUT.read_text(encoding="utf-8"))) def test_spec(entry): md = MarkdownIt("commonmark") output = md.render(entry["markdown"]) From c2a0009b3beac7ef75916ae530db8a1711063753 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 3 Nov 2026 13:13:45 +0000 Subject: [PATCH 2/3] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_cmark_spec/get_cmark_spec.py | 5 ++++- tests/test_cmark_spec/test_spec.py | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/test_cmark_spec/get_cmark_spec.py b/tests/test_cmark_spec/get_cmark_spec.py index 0d7ba9c9..832e1385 100644 --- a/tests/test_cmark_spec/get_cmark_spec.py +++ b/tests/test_cmark_spec/get_cmark_spec.py @@ -72,7 +72,10 @@ def _json_to_fixture(data: list[dict[str, Any]]) -> str: print(f"Downloading CommonMark spec from {url}") response = requests.get(url) response.raise_for_status() - if not output_path.exists() or output_path.read_text(encoding="utf-8") != response.text: + if ( + not output_path.exists() + or output_path.read_text(encoding="utf-8") != response.text + ): changed = True with output_path.open("w") as f: f.write(response.text) diff --git a/tests/test_cmark_spec/test_spec.py b/tests/test_cmark_spec/test_spec.py index 83a4d548..b4becc25 100644 --- a/tests/test_cmark_spec/test_spec.py +++ b/tests/test_cmark_spec/test_spec.py @@ -15,7 +15,9 @@ def test_file(file_regression): md = MarkdownIt("commonmark") - file_regression.check(md.render(SPEC_INPUT.read_text(encoding="utf-8")), extension=".html") + file_regression.check( + md.render(SPEC_INPUT.read_text(encoding="utf-8")), extension=".html" + ) @pytest.mark.parametrize("entry", json.loads(TESTS_INPUT.read_text(encoding="utf-8"))) From 66858de4f3f8edfdabd99b0659615ccaca19cc7d Mon Sep 17 00:00:00 2001 From: Nash Aschenbath Date: Wed, 4 Nov 2026 15:33:15 +0800 Subject: [PATCH 3/3] =?UTF-8?q?=F0=9F=A7=AA=20Also=20write=20the=20spec=20?= =?UTF-8?q?fixtures=20as=20UTF-8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit get_cmark_spec.py now reads the fixtures as UTF-8, but still wrote them with the locale encoding, so refreshing them on a non-UTF-8 Windows locale would write bytes the tests can no longer read. Co-Authored-By: Claude Opus 5.5 --- tests/test_cmark_spec/get_cmark_spec.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_cmark_spec/get_cmark_spec.py b/tests/test_cmark_spec/get_cmark_spec.py index 832e1385..2ced0285 100644 --- a/tests/test_cmark_spec/get_cmark_spec.py +++ b/tests/test_cmark_spec/get_cmark_spec.py @@ -77,7 +77,7 @@ def _json_to_fixture(data: list[dict[str, Any]]) -> str: or output_path.read_text(encoding="utf-8") != response.text ): changed = True - with output_path.open("w") as f: + with output_path.open("w", encoding="utf-8") as f: f.write(response.text) print(f"Updated to {output_path}") else: @@ -90,7 +90,7 @@ def _json_to_fixture(data: list[dict[str, Any]]) -> str: text = _json_to_fixture(data) if not test_fixture.exists() or test_fixture.read_text(encoding="utf-8") != text: changed = True - with test_fixture.open("w") as f: + with test_fixture.open("w", encoding="utf-8") as f: f.write(text) print(f"Also updated to {test_fixture}") else: