diff --git a/docs/core_concepts/reproducibility.md b/docs/core_concepts/reproducibility.md index ec6f1e3..8aa6490 100644 --- a/docs/core_concepts/reproducibility.md +++ b/docs/core_concepts/reproducibility.md @@ -47,7 +47,9 @@ compatibility is not guaranteed. Supported built-in representations reproduce `transform` bit-for-bit in the same environment. Third-party representations and pretrained language models are not supported by the JSON -serializer; unsupported state raises `PretabSerializationError`. +serializer; unsupported state raises `PretabSerializationError` when the spec is built. The +spec is serialized before `to_spec(path)` opens the file, so a failed save leaves an existing +spec at that path untouched. ## Fingerprint diff --git a/pretab/compose/serialize.py b/pretab/compose/serialize.py index c4b213f..025fbef 100644 --- a/pretab/compose/serialize.py +++ b/pretab/compose/serialize.py @@ -52,7 +52,18 @@ ) +# Builtin types that are valid numpy dtype specifiers, encoded as that dtype. +_BUILTIN_DTYPES = (bool, int, float, complex) + + # --- helpers ------------------------------------------------------------- +def _map_elements(data, depth: int, func): + """Apply ``func`` to the elements of a ``depth``-level nested list (an array's ``tolist()``).""" + if depth == 0: + return func(data) + return [_map_elements(item, depth - 1, func) for item in data] + + def _qualname(obj) -> str: cls = type(obj) return f"{cls.__module__}:{cls.__qualname__}" @@ -89,10 +100,24 @@ def _encode(obj): if obj is UNSET: return {"__unset__": True} if isinstance(obj, np.ndarray): - return {"__ndarray__": {"dtype": obj.dtype.str, "shape": list(obj.shape), "data": obj.tolist()}} + data = obj.tolist() + if obj.dtype.kind == "O": + # tolist() keeps the elements of an object array as they are (e.g. + # numpy scalars such as np.bool_), so encode each one. + data = _map_elements(data, obj.ndim, _encode) + return {"__ndarray__": {"dtype": obj.dtype.str, "shape": list(obj.shape), "data": data}} if isinstance(obj, np.dtype): return {"__npdtype__": obj.str} if isinstance(obj, type): + if obj in _BUILTIN_DTYPES: + # A dtype-like builtin (e.g. ``Preprocessor(dtype=float)``) is stored as + # the numpy dtype it denotes; ``builtins`` is never importable from a spec. + return {"__npdtype__": np.dtype(obj).str} + if not _module_allowed(obj.__module__): + raise PretabSerializationError( + f"Cannot serialize the type {obj.__module__}.{obj.__qualname__}: a spec can only reference " + f"types from {sorted(_ALLOWED_TOP_LEVEL)}." + ) return {"__type__": f"{obj.__module__}:{obj.__qualname__}"} if isinstance(obj, slice): return {"__slice__": [obj.start, obj.stop, obj.step]} @@ -138,6 +163,16 @@ def _json_safe(obj): # --- decoding ------------------------------------------------------------ def _decode_ndarray(payload: dict) -> np.ndarray: dtype = np.dtype(payload["dtype"]) + if dtype.kind == "O": + # Fill element by element so a decoded element that is itself a sequence + # stays a single element instead of adding a dimension. + shape = payload["shape"] + elements: list = [] + _map_elements(payload["data"], len(shape), lambda item: elements.append(_decode(item))) + arr = np.empty(len(elements), dtype=object) + for index, element in enumerate(elements): + arr[index] = element + return arr.reshape(shape) arr = np.array(payload["data"], dtype=dtype) return arr.reshape(payload["shape"]) diff --git a/pretab/preprocessor.py b/pretab/preprocessor.py index a8193b8..d89e291 100644 --- a/pretab/preprocessor.py +++ b/pretab/preprocessor.py @@ -93,6 +93,14 @@ def _preset_numerical_method(task) -> str: } +def _spec_json(spec: dict, **kwargs) -> str: + """Serialize a spec to JSON text, raising a typed error for unsupported state.""" + try: + return json.dumps(spec, **kwargs) + except (TypeError, ValueError) as exc: + raise PretabSerializationError(f"The fitted state cannot be written as JSON: {exc}") from exc + + def _method_summary(features, *, is_numerical: bool, config: PreprocessorConfig) -> str: """Summarize the resolved method(s) actually used across ``features``. @@ -275,7 +283,8 @@ class Preprocessor(TransformerMixin, BaseEstimator): records the resolved choice and its memory footprint in ``output_report_``. dtype : numpy dtype or None, default=None Optional dtype to cast the transformed output to (e.g. ``numpy.float32`` to - halve memory). ``None`` keeps the native ``float64`` output. + halve memory). ``None`` keeps the native ``float64`` output. A builtin + specifier such as ``float`` is stored in a spec as the numpy dtype it denotes. verbose : int, default=0 Verbosity level controlling ``fit``-time logging, applied through the shared ``"pretab"`` logger so a single setting on this entry point governs the whole @@ -1015,12 +1024,23 @@ def to_spec(self, path=None) -> dict: ------- dict The spec dictionary (always returned, whether or not ``path`` is set). + + Raises + ------ + PretabSerializationError + If the fitted state holds a value or type that a spec cannot store or + load (for example a type outside ``pretab``, ``sklearn``, ``numpy`` and + ``scipy``). The error is raised before ``path`` is opened, so an + existing file at ``path`` is left untouched. """ check_is_fitted(self) spec = preprocessor_to_spec(self) + # Serialize before opening the file, so a failure never truncates an + # existing spec at ``path``. + text = _spec_json(spec, indent=2) if path is not None: with open(path, "w", encoding="utf-8") as handle: - json.dump(spec, handle, indent=2) + handle.write(text) return spec @classmethod @@ -1078,9 +1098,14 @@ def fingerprint_(self) -> str: the fitted state (knot / center / bin locations, scaler statistics, encoder categories). The digest is deterministic across processes and machines, so the same fitted state and configuration produce the same fingerprint. + + Raises + ------ + PretabSerializationError + If the fitted state cannot be written as a spec (see :meth:`to_spec`). """ check_is_fitted(self) - canonical = json.dumps(self._canonical_spec(), sort_keys=True, separators=(",", ":"), ensure_ascii=True) + canonical = _spec_json(self._canonical_spec(), sort_keys=True, separators=(",", ":"), ensure_ascii=True) return hashlib.sha256(canonical.encode("utf-8")).hexdigest() def reproducibility_report(self) -> dict: diff --git a/tests/integration/test_serialization.py b/tests/integration/test_serialization.py index 493000c..c340e05 100644 --- a/tests/integration/test_serialization.py +++ b/tests/integration/test_serialization.py @@ -274,3 +274,74 @@ def test_from_spec_refuses_dataclass_not_in_allowlist(): def test_from_spec_rejects_bad_source_type(): with pytest.raises(PretabSerializationError): Preprocessor.from_spec(12345) + + +# --- object arrays and builtin dtypes (issue #66) --------------------------------- + + +@pytest.fixture +def numpy_scalar_frame(): + rng = np.random.default_rng(0) + n = 80 + income = np.where(rng.random(n) < 0.1, np.nan, rng.normal(50, 10, n)) + frame = pd.DataFrame({"income": income, "age": rng.normal(40, 10, n)}) + # An object column of np.bool_ / None, as produced by a row-wise apply. + frame["high_income"] = frame.apply(lambda r: r.income > 50 if pd.notna(r.income) else None, axis=1) + return frame, rng.normal(size=n) + + +def test_object_arrays_of_numpy_scalars_serialize(numpy_scalar_frame, tmp_path): + """Regression guard for issue #66: numpy scalars in object arrays leaked into the spec.""" + frame, y = numpy_scalar_frame + pre = Preprocessor(random_state=0).fit(frame, y) + json.dumps(pre.to_spec()) + assert len(pre.fingerprint_) == 64 + path = tmp_path / "spec.json" + pre.to_spec(path) + np.testing.assert_array_equal(Preprocessor.from_spec(path).transform(frame), pre.transform(frame)) + + +@pytest.mark.parametrize("dtype", [float, int, np.float32]) +def test_builtin_and_numpy_dtypes_round_trip(dtype): + frame = pd.DataFrame({"age": np.linspace(20.0, 60.0, 30)}) + pre = Preprocessor(numerical_method="minmax", dtype=dtype).fit(frame) + loaded = Preprocessor.from_spec(json.loads(json.dumps(pre.to_spec()))) + np.testing.assert_array_equal(loaded.transform(frame), pre.transform(frame)) + assert loaded.fingerprint_ == pre.fingerprint_ + + +def test_failed_save_leaves_an_existing_spec_untouched(tmp_path, monkeypatch): + import pretab.preprocessor as preprocessor_module + + frame = pd.DataFrame({"age": np.linspace(20.0, 60.0, 30)}) + pre = Preprocessor(numerical_method="minmax").fit(frame) + path = tmp_path / "spec.json" + pre.to_spec(path) + saved = path.read_text(encoding="utf-8") + + # A spec the JSON writer rejects half-way: a streaming write into the + # already-opened file used to leave a truncated, invalid spec behind. + spec = pre.to_spec() + monkeypatch.setattr( + preprocessor_module, "preprocessor_to_spec", lambda _: {**spec, "state": {"unsupported_": object()}} + ) + with pytest.raises(PretabSerializationError, match="cannot be written as JSON"): + pre.to_spec(path) + assert path.read_text(encoding="utf-8") == saved + + +def test_types_a_spec_cannot_load_are_rejected_at_save_time(): + from pretab.compose.serialize import _encode + + with pytest.raises(PretabSerializationError, match="can only reference types"): + _encode(list) + + +def test_object_arrays_written_before_element_encoding_still_load(): + from pretab.compose.serialize import _decode + + payload = {"__ndarray__": {"dtype": "|O", "shape": [2, 2], "data": [["a", None], [True, 1.5]]}} + decoded = _decode(payload) + assert isinstance(decoded, np.ndarray) + assert decoded.dtype == object + assert decoded.tolist() == [["a", None], [True, 1.5]]