From f6a0cf844edb828350f07a0815c4c2cb5524f481 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Thu, 20 Aug 2026 18:15:48 -0700 Subject: [PATCH 1/8] test: update toml-test submodule and mark 3 known gaps as xfail MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps tests/toml-test to d168c2a (main), pulling in 23 new upstream commits including duplicate-repeated-table and no-close-table array coverage. Three of the new cases expose genuine tomlkit bugs and are marked xfail(strict=True) with the toml-test commit/issue that added them, so a fix will surface as an unexpected pass: - valid/utf8-bom-{01,02}: leading UTF-8 BOM not stripped before parsing (toml-test 542746b, BurntSushi/toml-test#199) - invalid/control/linetab-number-{01,02,03}: trailing \x0b after a number is not rejected (toml-test 4f76d84, BurntSushi/toml-test#195) - invalid/{float,integer}/arabic-zero-*: Arabic-Indic digit zero (٠) is accepted as a digit in several number positions (toml-test d736b6f, BurntSushi/toml-test#196) --- tests/test_toml_tests.py | 103 ++++++++++++++++++++++++--------------- tests/toml-test | 2 +- 2 files changed, 64 insertions(+), 41 deletions(-) diff --git a/tests/test_toml_tests.py b/tests/test_toml_tests.py index 667353ee..0d0d7f65 100644 --- a/tests/test_toml_tests.py +++ b/tests/test_toml_tests.py @@ -16,6 +16,58 @@ TESTS_ROOT = os.path.join(os.path.dirname(__file__), "toml-test", "tests") FILES_LIST = os.path.join(TESTS_ROOT, "files-toml-1.1.0") +# Cases added upstream (toml-test) that tomlkit does not yet handle correctly. +# Each reason cites the toml-test commit that introduced the case and its +# upstream issue, so these can be found again once the underlying bug is fixed. +KNOWN_FAILURES = { + "valid/utf8-bom-01": ( + "leading UTF-8 BOM is not stripped before parsing " + "(toml-test 542746b, BurntSushi/toml-test#199)" + ), + "valid/utf8-bom-02": ( + "leading UTF-8 BOM is not stripped before parsing " + "(toml-test 542746b, BurntSushi/toml-test#199)" + ), + "invalid/control/linetab-number-01": ( + "trailing \\x0b (vertical tab) after an integer is not rejected " + "(toml-test 4f76d84, BurntSushi/toml-test#195)" + ), + "invalid/control/linetab-number-02": ( + "trailing \\x0b (vertical tab) after a float is not rejected " + "(toml-test 4f76d84, BurntSushi/toml-test#195)" + ), + "invalid/control/linetab-number-03": ( + "trailing \\x0b (vertical tab) after a hex integer is not rejected " + "(toml-test 4f76d84, BurntSushi/toml-test#195)" + ), + "invalid/float/arabic-zero-01": ( + "Arabic-Indic digit zero (٠) is accepted as a fraction digit " + "(toml-test d736b6f, BurntSushi/toml-test#196)" + ), + "invalid/float/arabic-zero-03": ( + "Arabic-Indic digit zero (٠) is accepted in an exponent " + "(toml-test d736b6f, BurntSushi/toml-test#196)" + ), + "invalid/float/arabic-zero-04": ( + "Arabic-Indic digit zero (٠) is accepted as a signed float value " + "(toml-test d736b6f, BurntSushi/toml-test#196)" + ), + "invalid/integer/arabic-zero-01": ( + "Arabic-Indic digit zero (٠) is accepted as a trailing integer digit " + "(toml-test d736b6f, BurntSushi/toml-test#196)" + ), + "invalid/integer/arabic-zero-02": ( + "Arabic-Indic digit zero (٠) is accepted after an underscore digit " + "separator (toml-test d736b6f, BurntSushi/toml-test#196)" + ), +} + + +def _param(case_id: str, value: Any) -> Any: + reason = KNOWN_FAILURES.get(case_id) + marks = [pytest.mark.xfail(reason=reason, strict=True)] if reason else [] + return pytest.param(value, id=case_id, marks=marks) + def to_bool(s: str) -> bool: assert s in ["true", "false"] @@ -56,20 +108,10 @@ def _load_case_list() -> list[str]: return [line.strip() for line in f if line.strip()] -def _build_cases() -> tuple[ - list[dict[str, str]], - list[str], - list[dict[str, str]], - list[str], - list[str], - list[str], -]: +def _build_cases() -> tuple[list[Any], list[Any], list[Any]]: valid_cases = [] - valid_ids = [] invalid_decode_cases = [] - invalid_decode_ids = [] invalid_encode_cases = [] - invalid_encode_ids = [] for relpath in _load_case_list(): full_path = os.path.join(TESTS_ROOT, relpath) @@ -79,8 +121,7 @@ def _build_cases() -> tuple[ case_id = relpath.rsplit(".", 1)[0] if relpath.startswith("invalid/encoding/"): - invalid_encode_cases.append(full_path) - invalid_encode_ids.append(case_id) + invalid_encode_cases.append(_param(case_id, full_path)) elif relpath.startswith("valid/"): with open(full_path, encoding="utf-8", newline="") as f: toml_content = f.read() @@ -89,36 +130,22 @@ def _build_cases() -> tuple[ with open(json_path, encoding="utf-8") as f: json_content = f.read() - valid_cases.append({"toml": toml_content, "json": json_content}) - valid_ids.append(case_id) + valid_cases.append( + _param(case_id, {"toml": toml_content, "json": json_content}) + ) elif relpath.startswith("invalid/"): with open(full_path, encoding="utf-8", newline="") as f: toml_content = f.read() - invalid_decode_cases.append({"toml": toml_content}) - invalid_decode_ids.append(case_id) + invalid_decode_cases.append(_param(case_id, {"toml": toml_content})) - return ( - valid_cases, - valid_ids, - invalid_decode_cases, - invalid_decode_ids, - invalid_encode_cases, - invalid_encode_ids, - ) + return valid_cases, invalid_decode_cases, invalid_encode_cases -( - VALID_CASES, - VALID_IDS, - INVALID_DECODE_CASES, - INVALID_DECODE_IDS, - INVALID_ENCODE_CASES, - INVALID_ENCODE_IDS, -) = _build_cases() +VALID_CASES, INVALID_DECODE_CASES, INVALID_ENCODE_CASES = _build_cases() -@pytest.mark.parametrize("toml11_valid_case", VALID_CASES, ids=VALID_IDS) +@pytest.mark.parametrize("toml11_valid_case", VALID_CASES) def test_valid_decode(toml11_valid_case: dict[str, str]) -> None: json_val = untag(json.loads(toml11_valid_case["json"])) toml_val = parse(toml11_valid_case["toml"]) @@ -127,17 +154,13 @@ def test_valid_decode(toml11_valid_case: dict[str, str]) -> None: assert toml_val.as_string() == toml11_valid_case["toml"] -@pytest.mark.parametrize( - "toml11_invalid_decode_case", INVALID_DECODE_CASES, ids=INVALID_DECODE_IDS -) +@pytest.mark.parametrize("toml11_invalid_decode_case", INVALID_DECODE_CASES) def test_invalid_decode(toml11_invalid_decode_case: dict[str, str]) -> None: with pytest.raises(TOMLKitError): parse(toml11_invalid_decode_case["toml"]) -@pytest.mark.parametrize( - "toml11_invalid_encode_case", INVALID_ENCODE_CASES, ids=INVALID_ENCODE_IDS -) +@pytest.mark.parametrize("toml11_invalid_encode_case", INVALID_ENCODE_CASES) def test_invalid_encode(toml11_invalid_encode_case: str) -> None: with open(toml11_invalid_encode_case, encoding="utf-8") as f: with pytest.raises((TOMLKitError, UnicodeDecodeError)): diff --git a/tests/toml-test b/tests/toml-test index 08ed8697..d168c2a4 160000 --- a/tests/toml-test +++ b/tests/toml-test @@ -1 +1 @@ -Subproject commit 08ed8697864548b3cdb4b8decbf496bef47e1c82 +Subproject commit d168c2a4f539a5219c804055af1600b8cf9ca6d7 From 32226834da681fb71bab09b47ff5dad937d19b53 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Thu, 20 Aug 2026 18:24:04 -0700 Subject: [PATCH 2/8] fix: reject numbers with an embedded/trailing whitespace-like control char int()/float() silently strip leading/trailing whitespace-like characters (e.g. \x0b, vertical tab) when converting a string, so a number token like "1\x0b" parsed as valid instead of raising. _NUM_STOP does not include \x0b, so the character was absorbed into the raw token rather than ending it. Guard _parse_number to reject any raw token containing a str.isspace() character before attempting the int()/float() conversion. A syntactically valid number token never contains whitespace, so this is safe. Fixes invalid/control/linetab-number-{01,02,03} (toml-test 4f76d84, BurntSushi/toml-test#195). --- tests/test_toml_tests.py | 12 ------------ tomlkit/parser.py | 6 ++++++ 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/tests/test_toml_tests.py b/tests/test_toml_tests.py index 0d0d7f65..788dd674 100644 --- a/tests/test_toml_tests.py +++ b/tests/test_toml_tests.py @@ -28,18 +28,6 @@ "leading UTF-8 BOM is not stripped before parsing " "(toml-test 542746b, BurntSushi/toml-test#199)" ), - "invalid/control/linetab-number-01": ( - "trailing \\x0b (vertical tab) after an integer is not rejected " - "(toml-test 4f76d84, BurntSushi/toml-test#195)" - ), - "invalid/control/linetab-number-02": ( - "trailing \\x0b (vertical tab) after a float is not rejected " - "(toml-test 4f76d84, BurntSushi/toml-test#195)" - ), - "invalid/control/linetab-number-03": ( - "trailing \\x0b (vertical tab) after a hex integer is not rejected " - "(toml-test 4f76d84, BurntSushi/toml-test#195)" - ), "invalid/float/arabic-zero-01": ( "Arabic-Indic digit zero (٠) is accepted as a fraction digit " "(toml-test d736b6f, BurntSushi/toml-test#196)" diff --git a/tomlkit/parser.py b/tomlkit/parser.py index 8c7b1a64..bc620d0c 100644 --- a/tomlkit/parser.py +++ b/tomlkit/parser.py @@ -783,6 +783,12 @@ def _parse_number(self, raw: str, trivia: Trivia) -> Item | None: ): return None + # int()/float() silently strip leading/trailing whitespace-like chars + # (e.g. \x0b), which would otherwise let a stray control char right + # after a number token slip through unnoticed. + if any(c.isspace() for c in clean): + return None + try: return Integer(int(sign + clean, base), trivia, sign + raw) except ValueError: From 30e830b050e61d332371a0d0ac3c5bcbc0893132 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Thu, 20 Aug 2026 18:23:19 -0700 Subject: [PATCH 3/8] fix: reject non-ASCII digit characters in numeric literals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Python's int()/float() accept any Unicode decimal-digit character (category Nd), such as Arabic-Indic zero (٠, U+0660), as equivalent to ASCII digits. TOML numbers are ASCII-only, so tomlkit was accepting invalid literals like `1٠` or `0.1٠`. Fixes invalid/float/arabic-zero-{01,03,04} and invalid/integer/arabic-zero-{01,02} (toml-test d736b6f, BurntSushi/toml-test#196). --- tests/test_toml_tests.py | 20 -------------------- tomlkit/parser.py | 6 ++++++ 2 files changed, 6 insertions(+), 20 deletions(-) diff --git a/tests/test_toml_tests.py b/tests/test_toml_tests.py index 788dd674..5bbef528 100644 --- a/tests/test_toml_tests.py +++ b/tests/test_toml_tests.py @@ -28,26 +28,6 @@ "leading UTF-8 BOM is not stripped before parsing " "(toml-test 542746b, BurntSushi/toml-test#199)" ), - "invalid/float/arabic-zero-01": ( - "Arabic-Indic digit zero (٠) is accepted as a fraction digit " - "(toml-test d736b6f, BurntSushi/toml-test#196)" - ), - "invalid/float/arabic-zero-03": ( - "Arabic-Indic digit zero (٠) is accepted in an exponent " - "(toml-test d736b6f, BurntSushi/toml-test#196)" - ), - "invalid/float/arabic-zero-04": ( - "Arabic-Indic digit zero (٠) is accepted as a signed float value " - "(toml-test d736b6f, BurntSushi/toml-test#196)" - ), - "invalid/integer/arabic-zero-01": ( - "Arabic-Indic digit zero (٠) is accepted as a trailing integer digit " - "(toml-test d736b6f, BurntSushi/toml-test#196)" - ), - "invalid/integer/arabic-zero-02": ( - "Arabic-Indic digit zero (٠) is accepted after an underscore digit " - "separator (toml-test d736b6f, BurntSushi/toml-test#196)" - ), } diff --git a/tomlkit/parser.py b/tomlkit/parser.py index bc620d0c..e81c3680 100644 --- a/tomlkit/parser.py +++ b/tomlkit/parser.py @@ -742,6 +742,12 @@ def _parse_inline_table(self) -> InlineTable: return InlineTable(elems, Trivia()) def _parse_number(self, raw: str, trivia: Trivia) -> Item | None: + # Reject non-ASCII digit characters (e.g. Arabic-Indic zero, ٠): + # int()/float() accept any Unicode decimal digit, but TOML numbers + # are ASCII-only. + if not raw.isascii(): + return None + # Leading zeros are not allowed sign = "" if raw.startswith(("+", "-")): From 279a1cbe4183288af52f6c2ae02428bc4819c728 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Thu, 20 Aug 2026 18:40:21 -0700 Subject: [PATCH 4/8] fix: strip a leading Unicode BOM before parsing A single leading BOM (U+FEFF) is allowed by TOML and must be ignored, but a BOM anywhere else in the document is invalid. Parser.__init__ now strips one leading BOM before handing the string to Source, and parse() re-attaches it to the first item's indent trivia so that as_string() still reproduces the original text. Fixes toml-test 542746b, BurntSushi/toml-test#199. --- tests/test_toml_tests.py | 8 -------- tomlkit/parser.py | 17 ++++++++++++++++- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/tests/test_toml_tests.py b/tests/test_toml_tests.py index 5bbef528..663ba137 100644 --- a/tests/test_toml_tests.py +++ b/tests/test_toml_tests.py @@ -20,14 +20,6 @@ # Each reason cites the toml-test commit that introduced the case and its # upstream issue, so these can be found again once the underlying bug is fixed. KNOWN_FAILURES = { - "valid/utf8-bom-01": ( - "leading UTF-8 BOM is not stripped before parsing " - "(toml-test 542746b, BurntSushi/toml-test#199)" - ), - "valid/utf8-bom-02": ( - "leading UTF-8 BOM is not stripped before parsing " - "(toml-test 542746b, BurntSushi/toml-test#199)" - ), } diff --git a/tomlkit/parser.py b/tomlkit/parser.py index e81c3680..7c8f2752 100644 --- a/tomlkit/parser.py +++ b/tomlkit/parser.py @@ -56,6 +56,7 @@ CTRL_M = 0x0D # Carriage return CTRL_CHAR_LIMIT = 0x1F CHR_DEL = 0x7F +BOM = "\ufeff" # TOML character classes (formerly the `TOMLChar` constants), as frozensets for # O(1) membership tests; also the stop-sets for the Source.advance_while / @@ -102,7 +103,14 @@ class Parser: def __init__(self, string: str | bytes) -> None: # Input to parse - self._src = Source(decode(string)) + decoded = decode(string) + # A single leading BOM is allowed and ignored for parsing purposes, but + # it is re-attached to the first item's indent below so that dumping + # the parsed document reproduces the original text. + self._leading_bom = decoded[:1] == BOM + if self._leading_bom: + decoded = decoded[1:] + self._src = Source(decoded) self._aot_stack: list[Key] = [] self._nesting_depth = 0 @@ -210,6 +218,13 @@ def parse(self) -> TOMLDocument: body.parsing(False) + if self._leading_bom: + if body.body: + first_item = body.body[0][1] + first_item.trivia.indent = BOM + first_item.trivia.indent + else: + body.append(None, Whitespace(BOM)) + return body def _merge_ws(self, item: Item, container: Container) -> bool: From 47e35fe3c73193927cf575617d3f24fd3df0a50e Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Thu, 20 Aug 2026 18:48:59 -0700 Subject: [PATCH 5/8] test: drop the now-empty KNOWN_FAILURES xfail wrapper All three known toml-test gaps it tracked (BOM, vertical-tab, and Arabic-Indic digit handling) are fixed, so the dict is permanently empty. Use plain pytest.param(..., id=case_id) instead. --- tests/test_toml_tests.py | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/tests/test_toml_tests.py b/tests/test_toml_tests.py index 663ba137..a184f18f 100644 --- a/tests/test_toml_tests.py +++ b/tests/test_toml_tests.py @@ -16,18 +16,6 @@ TESTS_ROOT = os.path.join(os.path.dirname(__file__), "toml-test", "tests") FILES_LIST = os.path.join(TESTS_ROOT, "files-toml-1.1.0") -# Cases added upstream (toml-test) that tomlkit does not yet handle correctly. -# Each reason cites the toml-test commit that introduced the case and its -# upstream issue, so these can be found again once the underlying bug is fixed. -KNOWN_FAILURES = { -} - - -def _param(case_id: str, value: Any) -> Any: - reason = KNOWN_FAILURES.get(case_id) - marks = [pytest.mark.xfail(reason=reason, strict=True)] if reason else [] - return pytest.param(value, id=case_id, marks=marks) - def to_bool(s: str) -> bool: assert s in ["true", "false"] @@ -81,7 +69,7 @@ def _build_cases() -> tuple[list[Any], list[Any], list[Any]]: case_id = relpath.rsplit(".", 1)[0] if relpath.startswith("invalid/encoding/"): - invalid_encode_cases.append(_param(case_id, full_path)) + invalid_encode_cases.append(pytest.param(full_path, id=case_id)) elif relpath.startswith("valid/"): with open(full_path, encoding="utf-8", newline="") as f: toml_content = f.read() @@ -91,13 +79,17 @@ def _build_cases() -> tuple[list[Any], list[Any], list[Any]]: json_content = f.read() valid_cases.append( - _param(case_id, {"toml": toml_content, "json": json_content}) + pytest.param( + {"toml": toml_content, "json": json_content}, id=case_id + ) ) elif relpath.startswith("invalid/"): with open(full_path, encoding="utf-8", newline="") as f: toml_content = f.read() - invalid_decode_cases.append(_param(case_id, {"toml": toml_content})) + invalid_decode_cases.append( + pytest.param({"toml": toml_content}, id=case_id) + ) return valid_cases, invalid_decode_cases, invalid_encode_cases From 88b3be47ddc4405f110b2d30f49ac3d30fff7bc9 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Thu, 20 Aug 2026 19:11:37 -0700 Subject: [PATCH 6/8] fix: keep leading BOM at start when mutating a BOM-only document A document consisting only of a leading BOM has no body item to attach the BOM to, so it's stored as its own Whitespace entry. Without fixed=True, Container's insertion logic treats it as discardable filler and inserts new top-level items before it, moving the BOM off the front of the file. --- tests/test_parser.py | 11 +++++++++++ tomlkit/parser.py | 6 +++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/test_parser.py b/tests/test_parser.py index 44d5cd6c..0e6cc031 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -246,3 +246,14 @@ def test_parser_accepts_uppercase_exponent_after_leading_zero() -> None: value = Parser(f"a = {raw}").parse()["a"] assert isinstance(value, Float) assert value == float(raw) + + +def test_bom_only_document_keeps_bom_at_start_after_mutation() -> None: + # A document that is just a leading BOM has no body item to attach the + # BOM to, so it's stored as its own Whitespace entry. If that entry isn't + # marked fixed=True, Container's insertion logic treats it as discardable + # filler and puts newly added keys before it, moving the BOM off the + # front of the file. + doc = Parser("\ufeff").parse() + doc["a"] = 1 + assert doc.as_string() == "\ufeffa = 1\n" diff --git a/tomlkit/parser.py b/tomlkit/parser.py index 7c8f2752..bcea5aa1 100644 --- a/tomlkit/parser.py +++ b/tomlkit/parser.py @@ -223,7 +223,11 @@ def parse(self) -> TOMLDocument: first_item = body.body[0][1] first_item.trivia.indent = BOM + first_item.trivia.indent else: - body.append(None, Whitespace(BOM)) + # fixed=True: an ordinary (non-fixed) Whitespace is treated as + # discardable filler by Container's insertion logic, so a plain + # Whitespace(BOM) here would let new top-level items get + # inserted before it, moving the BOM away from the start. + body.append(None, Whitespace(BOM, fixed=True)) return body From bac0d9d35f6c6d652bebe7d50a4154ac8875a87f Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Thu, 20 Aug 2026 19:28:15 -0700 Subject: [PATCH 7/8] fix: replace literal Arabic-Indic zero in comment with code point MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ruff's RUF003 flagged the literal ٠ character in the comment as an ambiguous unicode character. Reference it by code point instead. --- tomlkit/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tomlkit/parser.py b/tomlkit/parser.py index bcea5aa1..2448b959 100644 --- a/tomlkit/parser.py +++ b/tomlkit/parser.py @@ -761,7 +761,7 @@ def _parse_inline_table(self) -> InlineTable: return InlineTable(elems, Trivia()) def _parse_number(self, raw: str, trivia: Trivia) -> Item | None: - # Reject non-ASCII digit characters (e.g. Arabic-Indic zero, ٠): + # Reject non-ASCII digit characters (e.g. Arabic-Indic zero, U+0660): # int()/float() accept any Unicode decimal digit, but TOML numbers # are ASCII-only. if not raw.isascii(): From c354558337924a162394e97ea7135d25519d95b6 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Thu, 20 Aug 2026 19:44:18 -0700 Subject: [PATCH 8/8] style: run ruff format on test_toml_tests.py The pre-commit ruff-format hook wanted a valid_cases pytest.param call collapsed onto one line. --- tests/test_toml_tests.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_toml_tests.py b/tests/test_toml_tests.py index a184f18f..1d6799d0 100644 --- a/tests/test_toml_tests.py +++ b/tests/test_toml_tests.py @@ -79,9 +79,7 @@ def _build_cases() -> tuple[list[Any], list[Any], list[Any]]: json_content = f.read() valid_cases.append( - pytest.param( - {"toml": toml_content, "json": json_content}, id=case_id - ) + pytest.param({"toml": toml_content, "json": json_content}, id=case_id) ) elif relpath.startswith("invalid/"): with open(full_path, encoding="utf-8", newline="") as f: