Fix three toml-test 1.1.0 compliance gaps

tests/test_parser.py

@@ -246,3 +246,14 @@ def test_parser_accepts_uppercase_exponent_after_leading_zero() -> None:
         value = Parser(f"a = {raw}").parse()["a"]
         assert isinstance(value, Float)
         assert value == float(raw)
+
+
+def test_bom_only_document_keeps_bom_at_start_after_mutation() -> None:
+    # A document that is just a leading BOM has no body item to attach the
+    # BOM to, so it's stored as its own Whitespace entry. If that entry isn't
+    # marked fixed=True, Container's insertion logic treats it as discardable
+    # filler and puts newly added keys before it, moving the BOM off the
+    # front of the file.
+    doc = Parser("\ufeff").parse()
+    doc["a"] = 1
+    assert doc.as_string() == "\ufeffa = 1\n"

tests/test_toml_tests.py

@@ -56,20 +56,10 @@ def _load_case_list() -> list[str]:
         return [line.strip() for line in f if line.strip()]
 
 
-def _build_cases() -> tuple[
-    list[dict[str, str]],
-    list[str],
-    list[dict[str, str]],
-    list[str],
-    list[str],
-    list[str],
-]:
+def _build_cases() -> tuple[list[Any], list[Any], list[Any]]:
     valid_cases = []
-    valid_ids = []
     invalid_decode_cases = []
-    invalid_decode_ids = []
     invalid_encode_cases = []
-    invalid_encode_ids = []
 
     for relpath in _load_case_list():
         full_path = os.path.join(TESTS_ROOT, relpath)
@@ -79,8 +69,7 @@ def _build_cases() -> tuple[
         case_id = relpath.rsplit(".", 1)[0]
 
         if relpath.startswith("invalid/encoding/"):
-            invalid_encode_cases.append(full_path)
-            invalid_encode_ids.append(case_id)
+            invalid_encode_cases.append(pytest.param(full_path, id=case_id))
         elif relpath.startswith("valid/"):
             with open(full_path, encoding="utf-8", newline="") as f:
                 toml_content = f.read()
@@ -89,36 +78,24 @@ def _build_cases() -> tuple[
             with open(json_path, encoding="utf-8") as f:
                 json_content = f.read()
 
-            valid_cases.append({"toml": toml_content, "json": json_content})
-            valid_ids.append(case_id)
+            valid_cases.append(
+                pytest.param({"toml": toml_content, "json": json_content}, id=case_id)
+            )
         elif relpath.startswith("invalid/"):
             with open(full_path, encoding="utf-8", newline="") as f:
                 toml_content = f.read()
 
-            invalid_decode_cases.append({"toml": toml_content})
-            invalid_decode_ids.append(case_id)
+            invalid_decode_cases.append(
+                pytest.param({"toml": toml_content}, id=case_id)
+            )
 
-    return (
-        valid_cases,
-        valid_ids,
-        invalid_decode_cases,
-        invalid_decode_ids,
-        invalid_encode_cases,
-        invalid_encode_ids,
-    )
+    return valid_cases, invalid_decode_cases, invalid_encode_cases
 
 
-(
-    VALID_CASES,
-    VALID_IDS,
-    INVALID_DECODE_CASES,
-    INVALID_DECODE_IDS,
-    INVALID_ENCODE_CASES,
-    INVALID_ENCODE_IDS,
-) = _build_cases()
+VALID_CASES, INVALID_DECODE_CASES, INVALID_ENCODE_CASES = _build_cases()
 
 
-@pytest.mark.parametrize("toml11_valid_case", VALID_CASES, ids=VALID_IDS)
+@pytest.mark.parametrize("toml11_valid_case", VALID_CASES)
 def test_valid_decode(toml11_valid_case: dict[str, str]) -> None:
     json_val = untag(json.loads(toml11_valid_case["json"]))
     toml_val = parse(toml11_valid_case["toml"])
@@ -127,17 +104,13 @@ def test_valid_decode(toml11_valid_case: dict[str, str]) -> None:
     assert toml_val.as_string() == toml11_valid_case["toml"]
 
 
-@pytest.mark.parametrize(
-    "toml11_invalid_decode_case", INVALID_DECODE_CASES, ids=INVALID_DECODE_IDS
-)
+@pytest.mark.parametrize("toml11_invalid_decode_case", INVALID_DECODE_CASES)
 def test_invalid_decode(toml11_invalid_decode_case: dict[str, str]) -> None:
     with pytest.raises(TOMLKitError):
         parse(toml11_invalid_decode_case["toml"])
 
 
-@pytest.mark.parametrize(
-    "toml11_invalid_encode_case", INVALID_ENCODE_CASES, ids=INVALID_ENCODE_IDS
-)
+@pytest.mark.parametrize("toml11_invalid_encode_case", INVALID_ENCODE_CASES)
 def test_invalid_encode(toml11_invalid_encode_case: str) -> None:
     with open(toml11_invalid_encode_case, encoding="utf-8") as f:
         with pytest.raises((TOMLKitError, UnicodeDecodeError)):

tomlkit/parser.py

@@ -56,6 +56,7 @@
 CTRL_M = 0x0D  # Carriage return
 CTRL_CHAR_LIMIT = 0x1F
 CHR_DEL = 0x7F
+BOM = "\ufeff"
 
 # TOML character classes (formerly the `TOMLChar` constants), as frozensets for
 # O(1) membership tests; also the stop-sets for the Source.advance_while /
@@ -102,7 +103,14 @@ class Parser:
 
     def __init__(self, string: str | bytes) -> None:
         # Input to parse
-        self._src = Source(decode(string))
+        decoded = decode(string)
+        # A single leading BOM is allowed and ignored for parsing purposes, but
+        # it is re-attached to the first item's indent below so that dumping
+        # the parsed document reproduces the original text.
+        self._leading_bom = decoded[:1] == BOM
+        if self._leading_bom:
+            decoded = decoded[1:]
+        self._src = Source(decoded)
 
         self._aot_stack: list[Key] = []
         self._nesting_depth = 0
@@ -210,6 +218,17 @@ def parse(self) -> TOMLDocument:
 
         body.parsing(False)
 
+        if self._leading_bom:
+            if body.body:
+                first_item = body.body[0][1]
+                first_item.trivia.indent = BOM + first_item.trivia.indent
+            else:
+                # fixed=True: an ordinary (non-fixed) Whitespace is treated as
+                # discardable filler by Container's insertion logic, so a plain
+                # Whitespace(BOM) here would let new top-level items get
+                # inserted before it, moving the BOM away from the start.
+                body.append(None, Whitespace(BOM, fixed=True))
+
         return body
 
     def _merge_ws(self, item: Item, container: Container) -> bool:
@@ -742,6 +761,12 @@ def _parse_inline_table(self) -> InlineTable:
         return InlineTable(elems, Trivia())
 
     def _parse_number(self, raw: str, trivia: Trivia) -> Item | None:
+        # Reject non-ASCII digit characters (e.g. Arabic-Indic zero, U+0660):
+        # int()/float() accept any Unicode decimal digit, but TOML numbers
+        # are ASCII-only.
+        if not raw.isascii():
+            return None
+
         # Leading zeros are not allowed
         sign = ""
         if raw.startswith(("+", "-")):
@@ -783,6 +808,12 @@ def _parse_number(self, raw: str, trivia: Trivia) -> Item | None:
         ):
             return None
 
+        # int()/float() silently strip leading/trailing whitespace-like chars
+        # (e.g. \x0b), which would otherwise let a stray control char right
+        # after a number token slip through unnoticed.
+        if any(c.isspace() for c in clean):
+            return None
+
         try:
             return Integer(int(sign + clean, base), trivia, sign + raw)
         except ValueError: