diff --git a/tests/test_parser.py b/tests/test_parser.py index 44d5cd6..0e6cc03 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -246,3 +246,14 @@ def test_parser_accepts_uppercase_exponent_after_leading_zero() -> None: value = Parser(f"a = {raw}").parse()["a"] assert isinstance(value, Float) assert value == float(raw) + + +def test_bom_only_document_keeps_bom_at_start_after_mutation() -> None: + # A document that is just a leading BOM has no body item to attach the + # BOM to, so it's stored as its own Whitespace entry. If that entry isn't + # marked fixed=True, Container's insertion logic treats it as discardable + # filler and puts newly added keys before it, moving the BOM off the + # front of the file. + doc = Parser("\ufeff").parse() + doc["a"] = 1 + assert doc.as_string() == "\ufeffa = 1\n" diff --git a/tests/test_toml_tests.py b/tests/test_toml_tests.py index 667353e..1d6799d 100644 --- a/tests/test_toml_tests.py +++ b/tests/test_toml_tests.py @@ -56,20 +56,10 @@ def _load_case_list() -> list[str]: return [line.strip() for line in f if line.strip()] -def _build_cases() -> tuple[ - list[dict[str, str]], - list[str], - list[dict[str, str]], - list[str], - list[str], - list[str], -]: +def _build_cases() -> tuple[list[Any], list[Any], list[Any]]: valid_cases = [] - valid_ids = [] invalid_decode_cases = [] - invalid_decode_ids = [] invalid_encode_cases = [] - invalid_encode_ids = [] for relpath in _load_case_list(): full_path = os.path.join(TESTS_ROOT, relpath) @@ -79,8 +69,7 @@ def _build_cases() -> tuple[ case_id = relpath.rsplit(".", 1)[0] if relpath.startswith("invalid/encoding/"): - invalid_encode_cases.append(full_path) - invalid_encode_ids.append(case_id) + invalid_encode_cases.append(pytest.param(full_path, id=case_id)) elif relpath.startswith("valid/"): with open(full_path, encoding="utf-8", newline="") as f: toml_content = f.read() @@ -89,36 +78,24 @@ def _build_cases() -> tuple[ with open(json_path, encoding="utf-8") as f: json_content = f.read() - valid_cases.append({"toml": toml_content, "json": json_content}) - valid_ids.append(case_id) + valid_cases.append( + pytest.param({"toml": toml_content, "json": json_content}, id=case_id) + ) elif relpath.startswith("invalid/"): with open(full_path, encoding="utf-8", newline="") as f: toml_content = f.read() - invalid_decode_cases.append({"toml": toml_content}) - invalid_decode_ids.append(case_id) + invalid_decode_cases.append( + pytest.param({"toml": toml_content}, id=case_id) + ) - return ( - valid_cases, - valid_ids, - invalid_decode_cases, - invalid_decode_ids, - invalid_encode_cases, - invalid_encode_ids, - ) + return valid_cases, invalid_decode_cases, invalid_encode_cases -( - VALID_CASES, - VALID_IDS, - INVALID_DECODE_CASES, - INVALID_DECODE_IDS, - INVALID_ENCODE_CASES, - INVALID_ENCODE_IDS, -) = _build_cases() +VALID_CASES, INVALID_DECODE_CASES, INVALID_ENCODE_CASES = _build_cases() -@pytest.mark.parametrize("toml11_valid_case", VALID_CASES, ids=VALID_IDS) +@pytest.mark.parametrize("toml11_valid_case", VALID_CASES) def test_valid_decode(toml11_valid_case: dict[str, str]) -> None: json_val = untag(json.loads(toml11_valid_case["json"])) toml_val = parse(toml11_valid_case["toml"]) @@ -127,17 +104,13 @@ def test_valid_decode(toml11_valid_case: dict[str, str]) -> None: assert toml_val.as_string() == toml11_valid_case["toml"] -@pytest.mark.parametrize( - "toml11_invalid_decode_case", INVALID_DECODE_CASES, ids=INVALID_DECODE_IDS -) +@pytest.mark.parametrize("toml11_invalid_decode_case", INVALID_DECODE_CASES) def test_invalid_decode(toml11_invalid_decode_case: dict[str, str]) -> None: with pytest.raises(TOMLKitError): parse(toml11_invalid_decode_case["toml"]) -@pytest.mark.parametrize( - "toml11_invalid_encode_case", INVALID_ENCODE_CASES, ids=INVALID_ENCODE_IDS -) +@pytest.mark.parametrize("toml11_invalid_encode_case", INVALID_ENCODE_CASES) def test_invalid_encode(toml11_invalid_encode_case: str) -> None: with open(toml11_invalid_encode_case, encoding="utf-8") as f: with pytest.raises((TOMLKitError, UnicodeDecodeError)): diff --git a/tests/toml-test b/tests/toml-test index 08ed869..d168c2a 160000 --- a/tests/toml-test +++ b/tests/toml-test @@ -1 +1 @@ -Subproject commit 08ed8697864548b3cdb4b8decbf496bef47e1c82 +Subproject commit d168c2a4f539a5219c804055af1600b8cf9ca6d7 diff --git a/tomlkit/parser.py b/tomlkit/parser.py index 8c7b1a6..2448b95 100644 --- a/tomlkit/parser.py +++ b/tomlkit/parser.py @@ -56,6 +56,7 @@ CTRL_M = 0x0D # Carriage return CTRL_CHAR_LIMIT = 0x1F CHR_DEL = 0x7F +BOM = "\ufeff" # TOML character classes (formerly the `TOMLChar` constants), as frozensets for # O(1) membership tests; also the stop-sets for the Source.advance_while / @@ -102,7 +103,14 @@ class Parser: def __init__(self, string: str | bytes) -> None: # Input to parse - self._src = Source(decode(string)) + decoded = decode(string) + # A single leading BOM is allowed and ignored for parsing purposes, but + # it is re-attached to the first item's indent below so that dumping + # the parsed document reproduces the original text. + self._leading_bom = decoded[:1] == BOM + if self._leading_bom: + decoded = decoded[1:] + self._src = Source(decoded) self._aot_stack: list[Key] = [] self._nesting_depth = 0 @@ -210,6 +218,17 @@ def parse(self) -> TOMLDocument: body.parsing(False) + if self._leading_bom: + if body.body: + first_item = body.body[0][1] + first_item.trivia.indent = BOM + first_item.trivia.indent + else: + # fixed=True: an ordinary (non-fixed) Whitespace is treated as + # discardable filler by Container's insertion logic, so a plain + # Whitespace(BOM) here would let new top-level items get + # inserted before it, moving the BOM away from the start. + body.append(None, Whitespace(BOM, fixed=True)) + return body def _merge_ws(self, item: Item, container: Container) -> bool: @@ -742,6 +761,12 @@ def _parse_inline_table(self) -> InlineTable: return InlineTable(elems, Trivia()) def _parse_number(self, raw: str, trivia: Trivia) -> Item | None: + # Reject non-ASCII digit characters (e.g. Arabic-Indic zero, U+0660): + # int()/float() accept any Unicode decimal digit, but TOML numbers + # are ASCII-only. + if not raw.isascii(): + return None + # Leading zeros are not allowed sign = "" if raw.startswith(("+", "-")): @@ -783,6 +808,12 @@ def _parse_number(self, raw: str, trivia: Trivia) -> Item | None: ): return None + # int()/float() silently strip leading/trailing whitespace-like chars + # (e.g. \x0b), which would otherwise let a stray control char right + # after a number token slip through unnoticed. + if any(c.isspace() for c in clean): + return None + try: return Integer(int(sign + clean, base), trivia, sign + raw) except ValueError: