Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion docs/core_concepts/reproducibility.md
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,9 @@ compatibility is not guaranteed.

Supported built-in representations reproduce `transform` bit-for-bit in the same environment.
Third-party representations and pretrained language models are not supported by the JSON
serializer; unsupported state raises `PretabSerializationError`.
serializer; unsupported state raises `PretabSerializationError` when the spec is built. The
spec is serialized before `to_spec(path)` opens the file, so a failed save leaves an existing
spec at that path untouched.

## Fingerprint

Expand Down
37 changes: 36 additions & 1 deletion pretab/compose/serialize.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,18 @@
)


# Builtin types that are valid numpy dtype specifiers, encoded as that dtype.
_BUILTIN_DTYPES = (bool, int, float, complex)


# --- helpers -------------------------------------------------------------
def _map_elements(data, depth: int, func):
"""Apply ``func`` to the elements of a ``depth``-level nested list (an array's ``tolist()``)."""
if depth == 0:
return func(data)
return [_map_elements(item, depth - 1, func) for item in data]


def _qualname(obj) -> str:
cls = type(obj)
return f"{cls.__module__}:{cls.__qualname__}"
Expand Down Expand Up @@ -89,10 +100,24 @@ def _encode(obj):
if obj is UNSET:
return {"__unset__": True}
if isinstance(obj, np.ndarray):
return {"__ndarray__": {"dtype": obj.dtype.str, "shape": list(obj.shape), "data": obj.tolist()}}
data = obj.tolist()
if obj.dtype.kind == "O":
# tolist() keeps the elements of an object array as they are (e.g.
# numpy scalars such as np.bool_), so encode each one.
data = _map_elements(data, obj.ndim, _encode)
return {"__ndarray__": {"dtype": obj.dtype.str, "shape": list(obj.shape), "data": data}}
if isinstance(obj, np.dtype):
return {"__npdtype__": obj.str}
if isinstance(obj, type):
if obj in _BUILTIN_DTYPES:
# A dtype-like builtin (e.g. ``Preprocessor(dtype=float)``) is stored as
# the numpy dtype it denotes; ``builtins`` is never importable from a spec.
return {"__npdtype__": np.dtype(obj).str}
if not _module_allowed(obj.__module__):
raise PretabSerializationError(
f"Cannot serialize the type {obj.__module__}.{obj.__qualname__}: a spec can only reference "
f"types from {sorted(_ALLOWED_TOP_LEVEL)}."
)
return {"__type__": f"{obj.__module__}:{obj.__qualname__}"}
if isinstance(obj, slice):
return {"__slice__": [obj.start, obj.stop, obj.step]}
Expand Down Expand Up @@ -138,6 +163,16 @@ def _json_safe(obj):
# --- decoding ------------------------------------------------------------
def _decode_ndarray(payload: dict) -> np.ndarray:
dtype = np.dtype(payload["dtype"])
if dtype.kind == "O":
# Fill element by element so a decoded element that is itself a sequence
# stays a single element instead of adding a dimension.
shape = payload["shape"]
elements: list = []
_map_elements(payload["data"], len(shape), lambda item: elements.append(_decode(item)))
arr = np.empty(len(elements), dtype=object)
for index, element in enumerate(elements):
arr[index] = element
return arr.reshape(shape)
arr = np.array(payload["data"], dtype=dtype)
return arr.reshape(payload["shape"])

Expand Down
31 changes: 28 additions & 3 deletions pretab/preprocessor.py
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,14 @@ def _preset_numerical_method(task) -> str:
}


def _spec_json(spec: dict, **kwargs) -> str:
"""Serialize a spec to JSON text, raising a typed error for unsupported state."""
try:
return json.dumps(spec, **kwargs)
except (TypeError, ValueError) as exc:
raise PretabSerializationError(f"The fitted state cannot be written as JSON: {exc}") from exc


def _method_summary(features, *, is_numerical: bool, config: PreprocessorConfig) -> str:
"""Summarize the resolved method(s) actually used across ``features``.

Expand Down Expand Up @@ -275,7 +283,8 @@ class Preprocessor(TransformerMixin, BaseEstimator):
records the resolved choice and its memory footprint in ``output_report_``.
dtype : numpy dtype or None, default=None
Optional dtype to cast the transformed output to (e.g. ``numpy.float32`` to
halve memory). ``None`` keeps the native ``float64`` output.
halve memory). ``None`` keeps the native ``float64`` output. A builtin
specifier such as ``float`` is stored in a spec as the numpy dtype it denotes.
verbose : int, default=0
Verbosity level controlling ``fit``-time logging, applied through the shared
``"pretab"`` logger so a single setting on this entry point governs the whole
Expand Down Expand Up @@ -1015,12 +1024,23 @@ def to_spec(self, path=None) -> dict:
-------
dict
The spec dictionary (always returned, whether or not ``path`` is set).

Raises
------
PretabSerializationError
If the fitted state holds a value or type that a spec cannot store or
load (for example a type outside ``pretab``, ``sklearn``, ``numpy`` and
``scipy``). The error is raised before ``path`` is opened, so an
existing file at ``path`` is left untouched.
"""
check_is_fitted(self)
spec = preprocessor_to_spec(self)
# Serialize before opening the file, so a failure never truncates an
# existing spec at ``path``.
text = _spec_json(spec, indent=2)
if path is not None:
with open(path, "w", encoding="utf-8") as handle:
json.dump(spec, handle, indent=2)
handle.write(text)
return spec

@classmethod
Expand Down Expand Up @@ -1078,9 +1098,14 @@ def fingerprint_(self) -> str:
the fitted state (knot / center / bin locations, scaler statistics, encoder
categories). The digest is deterministic across processes and machines, so
the same fitted state and configuration produce the same fingerprint.

Raises
------
PretabSerializationError
If the fitted state cannot be written as a spec (see :meth:`to_spec`).
"""
check_is_fitted(self)
canonical = json.dumps(self._canonical_spec(), sort_keys=True, separators=(",", ":"), ensure_ascii=True)
canonical = _spec_json(self._canonical_spec(), sort_keys=True, separators=(",", ":"), ensure_ascii=True)
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()

def reproducibility_report(self) -> dict:
Expand Down
71 changes: 71 additions & 0 deletions tests/integration/test_serialization.py
Original file line number Diff line number Diff line change
Expand Up @@ -274,3 +274,74 @@ def test_from_spec_refuses_dataclass_not_in_allowlist():
def test_from_spec_rejects_bad_source_type():
with pytest.raises(PretabSerializationError):
Preprocessor.from_spec(12345)


# --- object arrays and builtin dtypes (issue #66) ---------------------------------


@pytest.fixture
def numpy_scalar_frame():
rng = np.random.default_rng(0)
n = 80
income = np.where(rng.random(n) < 0.1, np.nan, rng.normal(50, 10, n))
frame = pd.DataFrame({"income": income, "age": rng.normal(40, 10, n)})
# An object column of np.bool_ / None, as produced by a row-wise apply.
frame["high_income"] = frame.apply(lambda r: r.income > 50 if pd.notna(r.income) else None, axis=1)
return frame, rng.normal(size=n)


def test_object_arrays_of_numpy_scalars_serialize(numpy_scalar_frame, tmp_path):
"""Regression guard for issue #66: numpy scalars in object arrays leaked into the spec."""
frame, y = numpy_scalar_frame
pre = Preprocessor(random_state=0).fit(frame, y)
json.dumps(pre.to_spec())
assert len(pre.fingerprint_) == 64
path = tmp_path / "spec.json"
pre.to_spec(path)
np.testing.assert_array_equal(Preprocessor.from_spec(path).transform(frame), pre.transform(frame))


@pytest.mark.parametrize("dtype", [float, int, np.float32])
def test_builtin_and_numpy_dtypes_round_trip(dtype):
frame = pd.DataFrame({"age": np.linspace(20.0, 60.0, 30)})
pre = Preprocessor(numerical_method="minmax", dtype=dtype).fit(frame)
loaded = Preprocessor.from_spec(json.loads(json.dumps(pre.to_spec())))
np.testing.assert_array_equal(loaded.transform(frame), pre.transform(frame))
assert loaded.fingerprint_ == pre.fingerprint_


def test_failed_save_leaves_an_existing_spec_untouched(tmp_path, monkeypatch):
import pretab.preprocessor as preprocessor_module

frame = pd.DataFrame({"age": np.linspace(20.0, 60.0, 30)})
pre = Preprocessor(numerical_method="minmax").fit(frame)
path = tmp_path / "spec.json"
pre.to_spec(path)
saved = path.read_text(encoding="utf-8")

# A spec the JSON writer rejects half-way: a streaming write into the
# already-opened file used to leave a truncated, invalid spec behind.
spec = pre.to_spec()
monkeypatch.setattr(
preprocessor_module, "preprocessor_to_spec", lambda _: {**spec, "state": {"unsupported_": object()}}
)
with pytest.raises(PretabSerializationError, match="cannot be written as JSON"):
pre.to_spec(path)
assert path.read_text(encoding="utf-8") == saved


def test_types_a_spec_cannot_load_are_rejected_at_save_time():
from pretab.compose.serialize import _encode

with pytest.raises(PretabSerializationError, match="can only reference types"):
_encode(list)


def test_object_arrays_written_before_element_encoding_still_load():
from pretab.compose.serialize import _decode

payload = {"__ndarray__": {"dtype": "|O", "shape": [2, 2], "data": [["a", None], [True, 1.5]]}}
decoded = _decode(payload)
assert isinstance(decoded, np.ndarray)
assert decoded.dtype == object
assert decoded.tolist() == [["a", None], [True, 1.5]]
Loading