From 0c1d17b9878ae2b390616fbd8b64f605f9fd9d67 Mon Sep 17 00:00:00 2001 From: 123 Date: Thu, 27 Aug 2026 20:47:04 +0800 Subject: [PATCH 1/3] Add optional native parser adapter boundary --- pyproject.toml | 3 + src/dotenv/_native.py | 68 +++++++++++++++ src/dotenv/parser.py | 14 +++ tests/test_native_adapter.py | 162 +++++++++++++++++++++++++++++++++++ 4 files changed, 247 insertions(+) create mode 100644 src/dotenv/_native.py create mode 100644 tests/test_native_adapter.py diff --git a/pyproject.toml b/pyproject.toml index 1753fd89..a8e98c52 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,9 @@ Source = "https://github.com/theskumar/python-dotenv" cli = [ "click>=5.0", ] +native = [ + "fast-dotenv-rs-backend>=0.1.1,<0.2; platform_python_implementation != 'PyPy'", +] [project.scripts] dotenv = "dotenv.__main__:cli" diff --git a/src/dotenv/_native.py b/src/dotenv/_native.py new file mode 100644 index 00000000..bfe8a30c --- /dev/null +++ b/src/dotenv/_native.py @@ -0,0 +1,68 @@ +"""Optional parser bridge for the backend-only distribution. + +``fast-dotenv-rs-backend`` is deliberately separate from the upstream +``dotenv`` package. Its only adapter contract is a ``parse_bindings`` +function returning lossless records in the order +``(key, value, original_string, original_line, error)``. + +The Python parser is the normal path. It is selected only when the backend +package is absent (or when running on PyPy). Once a backend is imported, an +exception or contract violation is raised instead of being hidden by a +fallback; this keeps native semantic mismatches observable. +""" + +from importlib import import_module +from platform import python_implementation +from typing import Iterator, Optional, Tuple + +BindingRecord = Tuple[Optional[str], Optional[str], str, int, bool] + +_BACKEND_MODULE = "fast_dotenv_rs_backend" +_BACKEND_DISTRIBUTION = "fast-dotenv-rs-backend" + + +class NativeBackendContractError(RuntimeError): + """The selected backend did not satisfy the parser adapter contract.""" + + +def _normalize_record(record: object) -> BindingRecord: + if not isinstance(record, tuple) or len(record) != 5: + raise NativeBackendContractError( + "native parse_bindings record must be a five-item tuple" + ) + + key, value, original, line, error = record + if key is not None and not isinstance(key, str): + raise NativeBackendContractError("native binding key must be str or None") + if value is not None and not isinstance(value, str): + raise NativeBackendContractError("native binding value must be str or None") + if not isinstance(original, str) or type(line) is not int or line < 1: + raise NativeBackendContractError( + "native binding original must be str and line must be a positive int" + ) + if type(error) is not bool: + raise NativeBackendContractError("native binding error must be bool") + + return key, value, original, line, error + + +def parse_bindings(text: str) -> Optional[Iterator[BindingRecord]]: + """Return validated native records, or ``None`` when no backend is present.""" + if python_implementation() == "PyPy": + return None + + try: + backend = import_module(_BACKEND_MODULE) + except ModuleNotFoundError as error: + if error.name == _BACKEND_MODULE: + return None + raise + + parser = getattr(backend, "parse_bindings", None) + if not callable(parser): + raise NativeBackendContractError( + f"{_BACKEND_DISTRIBUTION} does not expose parse_bindings" + ) + + records = [_normalize_record(record) for record in parser(text)] + return iter(records) diff --git a/src/dotenv/parser.py b/src/dotenv/parser.py index d9d47583..85230ca8 100644 --- a/src/dotenv/parser.py +++ b/src/dotenv/parser.py @@ -10,6 +10,8 @@ Sequence, ) +from ._native import parse_bindings as _parse_native_bindings + def make_regex(string: str, extra_flags: int = 0) -> Pattern[str]: return re.compile(string, re.UNICODE | extra_flags) @@ -187,5 +189,17 @@ def parse_binding(reader: Reader) -> Binding: def parse_stream(stream: IO[str]) -> Iterator[Binding]: reader = Reader(stream) + + native_bindings = _parse_native_bindings(reader.string) + if native_bindings is not None: + for key, value, original, line, error in native_bindings: + yield Binding( + key=key, + value=value, + original=Original(string=original, line=line), + error=error, + ) + return + while reader.has_next(): yield parse_binding(reader) diff --git a/tests/test_native_adapter.py b/tests/test_native_adapter.py new file mode 100644 index 00000000..4a2ba7d6 --- /dev/null +++ b/tests/test_native_adapter.py @@ -0,0 +1,162 @@ +import io +import sys +from pathlib import Path +from types import ModuleType + +import pytest + +import dotenv +import dotenv._native as native +import dotenv.parser as parser_module +from dotenv.parser import Binding, Original, parse_stream + + +def _python_bindings(text): + return list(parse_stream(io.StringIO(text))) + + +def test_missing_backend_falls_back_to_python_parser(monkeypatch): + def missing_backend(module_name): + raise ModuleNotFoundError(module_name, name=module_name) + + monkeypatch.setattr(native, "import_module", missing_backend) + + assert _python_bindings("a=b\n") == [ + Binding( + key="a", + value="b", + original=Original(string="a=b\n", line=1), + error=False, + ) + ] + + +def test_backend_exception_is_not_hidden_by_python_fallback(monkeypatch): + class BrokenBackend: + @staticmethod + def parse_bindings(text): + raise RuntimeError("backend semantic failure") + + monkeypatch.setattr(native, "import_module", lambda _: BrokenBackend) + + with pytest.raises(RuntimeError, match="backend semantic failure"): + _python_bindings("a=b\n") + + +@pytest.mark.parametrize( + "records", + [ + [("a", "b", "a=b", 1)], + [("a", "b", "a=b", 1, "false")], + [("a", "b", "a=b", 0, False)], + ], +) +def test_invalid_backend_records_are_hard_contract_errors(monkeypatch, records): + class InvalidBackend: + @staticmethod + def parse_bindings(text): + return records + + monkeypatch.setattr(native, "import_module", lambda _: InvalidBackend) + + with pytest.raises(native.NativeBackendContractError): + _python_bindings("a=b\n") + + +def test_backend_import_dependency_error_is_not_hidden(monkeypatch): + def broken_import(module_name): + raise ModuleNotFoundError("backend dependency missing", name="dependency") + + monkeypatch.setattr(native, "import_module", broken_import) + + with pytest.raises(ModuleNotFoundError, match="backend dependency missing"): + _python_bindings("a=b\n") + + +def test_pypy_never_attempts_native_import(monkeypatch): + monkeypatch.setattr(native, "python_implementation", lambda: "PyPy") + monkeypatch.setattr( + native, + "import_module", + lambda _: pytest.fail("PyPy must not import a native backend"), + ) + + assert _python_bindings("a=b\n") == [ + Binding( + key="a", + value="b", + original=Original(string="a=b\n", line=1), + error=False, + ) + ] + + +def test_valid_backend_records_are_adapted_at_parser_boundary(monkeypatch): + calls = [] + + class Backend: + @staticmethod + def parse_bindings(text): + calls.append(text) + return [("a", "b", "a=b\n", 1, False)] + + monkeypatch.setattr(native, "import_module", lambda _: Backend) + monkeypatch.setattr( + parser_module, + "parse_binding", + lambda _: pytest.fail("native path must not duplicate Python parsing"), + ) + + result = list(parse_stream(io.StringIO("\ufeffa=b\n"))) + + assert calls == ["a=b\n"] + assert result == [ + Binding( + key="a", + value="b", + original=Original(string="a=b\n", line=1), + error=False, + ) + ] + assert type(result[0]) is Binding + assert type(result[0].original) is Original + + +def test_legacy_dotenv_core_is_not_used(monkeypatch): + calls = [] + + def missing_backend(module_name): + calls.append(module_name) + raise ModuleNotFoundError(module_name, name=module_name) + + monkeypatch.setattr(native, "import_module", missing_backend) + assert _python_bindings("a=b\n")[0].value == "b" + assert calls == ["fast_dotenv_rs_backend"] + + +def test_extension_only_backend_can_coexist_with_upstream_package(monkeypatch): + backend = ModuleType(native._BACKEND_MODULE) + backend.parse_bindings = lambda text: [("a", "b", "a=b", 1, False)] + monkeypatch.setitem(sys.modules, native._BACKEND_MODULE, backend) + + result = list(parse_stream(io.StringIO("a=b"))) + + assert Path(dotenv.__file__).name == "__init__.py" + assert dotenv.__all__ == [ + "get_cli_string", + "load_dotenv", + "dotenv_values", + "get_key", + "set_key", + "unset_key", + "find_dotenv", + "load_ipython_extension", + ] + assert result == [ + Binding( + key="a", + value="b", + original=Original(string="a=b", line=1), + error=False, + ) + ] From f5856435e229485040e0e11530a191b0f51007df Mon Sep 17 00:00:00 2001 From: 123 Date: Thu, 27 Aug 2026 20:54:53 +0800 Subject: [PATCH 2/3] Validate backend-only parser contract markers --- src/dotenv/_native.py | 14 ++++++++++- tests/test_native_adapter.py | 46 +++++++++++++++++++++++++++++++++--- 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/src/dotenv/_native.py b/src/dotenv/_native.py index bfe8a30c..6c80c647 100644 --- a/src/dotenv/_native.py +++ b/src/dotenv/_native.py @@ -3,7 +3,8 @@ ``fast-dotenv-rs-backend`` is deliberately separate from the upstream ``dotenv`` package. Its only adapter contract is a ``parse_bindings`` function returning lossless records in the order -``(key, value, original_string, original_line, error)``. +``(key, value, original_string, original_line, error)`` and the versioned +contract markers ``BACKEND_CONTRACT`` and ``BACKEND_CONTRACT_VERSION``. The Python parser is the normal path. It is selected only when the backend package is absent (or when running on PyPy). Once a backend is imported, an @@ -19,6 +20,8 @@ _BACKEND_MODULE = "fast_dotenv_rs_backend" _BACKEND_DISTRIBUTION = "fast-dotenv-rs-backend" +_BACKEND_CONTRACT = "fast-dotenv-rs.backend.binding" +_BACKEND_CONTRACT_VERSION = 1 class NativeBackendContractError(RuntimeError): @@ -58,6 +61,15 @@ def parse_bindings(text: str) -> Optional[Iterator[BindingRecord]]: return None raise + if ( + getattr(backend, "BACKEND_CONTRACT", None) != _BACKEND_CONTRACT + or type(getattr(backend, "BACKEND_CONTRACT_VERSION", None)) is not int + or backend.BACKEND_CONTRACT_VERSION != _BACKEND_CONTRACT_VERSION + ): + raise NativeBackendContractError( + f"{_BACKEND_DISTRIBUTION} does not expose the supported parser contract" + ) + parser = getattr(backend, "parse_bindings", None) if not callable(parser): raise NativeBackendContractError( diff --git a/tests/test_native_adapter.py b/tests/test_native_adapter.py index 4a2ba7d6..cbe17c24 100644 --- a/tests/test_native_adapter.py +++ b/tests/test_native_adapter.py @@ -15,6 +15,12 @@ def _python_bindings(text): return list(parse_stream(io.StringIO(text))) +def _mark_backend(backend): + backend.BACKEND_CONTRACT = native._BACKEND_CONTRACT + backend.BACKEND_CONTRACT_VERSION = native._BACKEND_CONTRACT_VERSION + return backend + + def test_missing_backend_falls_back_to_python_parser(monkeypatch): def missing_backend(module_name): raise ModuleNotFoundError(module_name, name=module_name) @@ -37,7 +43,7 @@ class BrokenBackend: def parse_bindings(text): raise RuntimeError("backend semantic failure") - monkeypatch.setattr(native, "import_module", lambda _: BrokenBackend) + monkeypatch.setattr(native, "import_module", lambda _: _mark_backend(BrokenBackend)) with pytest.raises(RuntimeError, match="backend semantic failure"): _python_bindings("a=b\n") @@ -57,7 +63,9 @@ class InvalidBackend: def parse_bindings(text): return records - monkeypatch.setattr(native, "import_module", lambda _: InvalidBackend) + monkeypatch.setattr( + native, "import_module", lambda _: _mark_backend(InvalidBackend) + ) with pytest.raises(native.NativeBackendContractError): _python_bindings("a=b\n") @@ -91,6 +99,37 @@ def test_pypy_never_attempts_native_import(monkeypatch): ] +def test_backend_contract_markers_are_required(monkeypatch): + class UnversionedBackend: + @staticmethod + def parse_bindings(text): + return [("a", "b", "a=b\n", 1, False)] + + monkeypatch.setattr(native, "import_module", lambda _: UnversionedBackend) + + with pytest.raises( + native.NativeBackendContractError, match="supported parser contract" + ): + _python_bindings("a=b\n") + + +def test_backend_contract_marker_mismatch_is_not_hidden(monkeypatch): + class WrongContractBackend: + BACKEND_CONTRACT = "wrong.contract" + BACKEND_CONTRACT_VERSION = 99 + + @staticmethod + def parse_bindings(text): + return [("a", "b", "a=b\n", 1, False)] + + monkeypatch.setattr(native, "import_module", lambda _: WrongContractBackend) + + with pytest.raises( + native.NativeBackendContractError, match="supported parser contract" + ): + _python_bindings("a=b\n") + + def test_valid_backend_records_are_adapted_at_parser_boundary(monkeypatch): calls = [] @@ -100,7 +139,7 @@ def parse_bindings(text): calls.append(text) return [("a", "b", "a=b\n", 1, False)] - monkeypatch.setattr(native, "import_module", lambda _: Backend) + monkeypatch.setattr(native, "import_module", lambda _: _mark_backend(Backend)) monkeypatch.setattr( parser_module, "parse_binding", @@ -136,6 +175,7 @@ def missing_backend(module_name): def test_extension_only_backend_can_coexist_with_upstream_package(monkeypatch): backend = ModuleType(native._BACKEND_MODULE) + _mark_backend(backend) backend.parse_bindings = lambda text: [("a", "b", "a=b", 1, False)] monkeypatch.setitem(sys.modules, native._BACKEND_MODULE, backend) From 16059fcf43f93218174a5735f8018af059c5413c Mon Sep 17 00:00:00 2001 From: 123 Date: Thu, 27 Aug 2026 22:50:08 +0800 Subject: [PATCH 3/3] test: isolate native contract checks from PyPy --- tests/test_native_adapter.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/test_native_adapter.py b/tests/test_native_adapter.py index cbe17c24..1f4fb7b6 100644 --- a/tests/test_native_adapter.py +++ b/tests/test_native_adapter.py @@ -10,6 +10,11 @@ import dotenv.parser as parser_module from dotenv.parser import Binding, Original, parse_stream +native_only = pytest.mark.skipif( + sys.implementation.name == "pypy", + reason="native backend contract tests require CPython", +) + def _python_bindings(text): return list(parse_stream(io.StringIO(text))) @@ -37,6 +42,7 @@ def missing_backend(module_name): ] +@native_only def test_backend_exception_is_not_hidden_by_python_fallback(monkeypatch): class BrokenBackend: @staticmethod @@ -57,6 +63,7 @@ def parse_bindings(text): [("a", "b", "a=b", 0, False)], ], ) +@native_only def test_invalid_backend_records_are_hard_contract_errors(monkeypatch, records): class InvalidBackend: @staticmethod @@ -71,6 +78,7 @@ def parse_bindings(text): _python_bindings("a=b\n") +@native_only def test_backend_import_dependency_error_is_not_hidden(monkeypatch): def broken_import(module_name): raise ModuleNotFoundError("backend dependency missing", name="dependency") @@ -99,6 +107,7 @@ def test_pypy_never_attempts_native_import(monkeypatch): ] +@native_only def test_backend_contract_markers_are_required(monkeypatch): class UnversionedBackend: @staticmethod @@ -113,6 +122,7 @@ def parse_bindings(text): _python_bindings("a=b\n") +@native_only def test_backend_contract_marker_mismatch_is_not_hidden(monkeypatch): class WrongContractBackend: BACKEND_CONTRACT = "wrong.contract" @@ -130,6 +140,7 @@ def parse_bindings(text): _python_bindings("a=b\n") +@native_only def test_valid_backend_records_are_adapted_at_parser_boundary(monkeypatch): calls = [] @@ -161,6 +172,7 @@ def parse_bindings(text): assert type(result[0].original) is Original +@native_only def test_legacy_dotenv_core_is_not_used(monkeypatch): calls = [] @@ -173,10 +185,11 @@ def missing_backend(module_name): assert calls == ["fast_dotenv_rs_backend"] +@native_only def test_extension_only_backend_can_coexist_with_upstream_package(monkeypatch): backend = ModuleType(native._BACKEND_MODULE) _mark_backend(backend) - backend.parse_bindings = lambda text: [("a", "b", "a=b", 1, False)] + backend.__dict__["parse_bindings"] = lambda text: [("a", "b", "a=b", 1, False)] monkeypatch.setitem(sys.modules, native._BACKEND_MODULE, backend) result = list(parse_stream(io.StringIO("a=b")))