From c1ed2ff78b1342b901b79ae2e64e97ad94089629 Mon Sep 17 00:00:00 2001 From: Manish Kumar Date: Fri, 4 Sep 2026 18:44:39 +0200 Subject: [PATCH 01/19] fix: drop unwired policy fields and encoding/embedding NaN bugs --- docs/representations/embeddings.md | 5 +- pretab/core/policy.py | 35 ++++++------- pretab/embedding/language.py | 15 ++++-- pretab/encoding/categorical/ordinal.py | 30 ++++++++--- .../test_language_embedding_transformer.py | 24 +++++++-- .../test_continuous_ordinal_transformer.py | 50 +++++++++++++++++++ 6 files changed, 128 insertions(+), 31 deletions(-) create mode 100644 tests/encoding/categorical/test_continuous_ordinal_transformer.py diff --git a/docs/representations/embeddings.md b/docs/representations/embeddings.md index 7057b6b..b4e48d9 100644 --- a/docs/representations/embeddings.md +++ b/docs/representations/embeddings.md @@ -21,8 +21,9 @@ Constructor highlights: `model_name="paraphrase-MiniLM-L3-v2"`, or pass a preloa (useful for tests or a custom embedding backend without pulling in `sentence-transformers`). Any object with a `.encode(X)` method works for the `transform` step; `embedding_dim_` is read from `get_sentence_embedding_dimension()` when the model exposes it (as a real -`SentenceTransformer` does), falling back to a `dim` attribute, or to `0` if neither is -present. The registry key is `pretrained`. +`SentenceTransformer` does), falling back to a `dim` attribute, or, if neither is present, +probed by calling `.encode()` once on a placeholder input during `fit`. The registry key is +`pretrained`. ```{important} Language embeddings require the optional `embeddings` extra, which pulls in diff --git a/pretab/core/policy.py b/pretab/core/policy.py index 1702f0f..0609921 100644 --- a/pretab/core/policy.py +++ b/pretab/core/policy.py @@ -3,17 +3,29 @@ :class:`RepresentationPolicy` names, in one place, how every transformer reacts to the recurring edge cases that would otherwise diverge silently per family: -* ``missing`` -- how ``NaN`` inputs are treated (``"error"`` / ``"propagate"``). * ``constant`` -- zero-variance input columns (``"error"`` / ``"warn"`` / ``"allow"``). * ``out_of_range`` -- values at ``transform`` outside the fitted range (``"error"`` / ``"warn"`` / ``"clip"`` / ``"extrapolate"``). -* ``invalid`` -- non-finite ``inf`` / ``-inf`` inputs (``"error"`` / ``"propagate"``). The defaults reproduce the library's historical behaviour (constant columns pass -through, ranges extrapolate, non-finite values raise), so enabling the policy -object changes nothing until a stricter choice is requested. Transformers may -narrow specific axes through class-level override attributes without exposing a -new constructor parameter (see :class:`~pretab.core.base.BasePreTabTransformer`). +through, ranges extrapolate), so enabling the policy object changes nothing +until a stricter choice is requested. Transformers may narrow specific axes +through class-level override attributes without exposing a new constructor +parameter (see :class:`~pretab.core.base.BasePreTabTransformer`). + +Missing values and non-finite (``inf`` / ``-inf``) inputs are handled elsewhere: +see ``Preprocessor.missing_policy`` for missing-value handling, and note that +``inf`` / ``-inf`` always raise during input validation regardless of any +policy (there is no configurable axis for it). + +.. note:: + ``out_of_range`` is not yet reachable from any public API: no transformer + constructor accepts a ``policy`` argument, and ``Preprocessor.policy`` is + only used for its own top-level ``constant`` check, never threaded into + ``PreprocessorConfig`` or the transformers it builds. Wiring this through + (transformer constructors, the registry's ``allowed_args``, and + ``PreprocessorConfig``) is deferred to a follow-up; see the "Spline + expansions" section of ``dev/todo/release-1.0.0/bugfixes-1.0.0.md``. """ from __future__ import annotations @@ -32,16 +44,12 @@ "resolve_out_of_range", ] -_MISSING_CHOICES = ("error", "propagate") _CONSTANT_CHOICES = ("error", "warn", "allow") _OUT_OF_RANGE_CHOICES = ("error", "warn", "clip", "extrapolate") -_INVALID_CHOICES = ("error", "propagate") _CHOICES = { - "missing": _MISSING_CHOICES, "constant": _CONSTANT_CHOICES, "out_of_range": _OUT_OF_RANGE_CHOICES, - "invalid": _INVALID_CHOICES, } @@ -51,21 +59,14 @@ class RepresentationPolicy: Parameters ---------- - missing : {"error", "propagate"}, default="propagate" - ``"propagate"`` lets ``NaN`` pass through to a downstream imputer; - ``"error"`` raises on any missing value. constant : {"error", "warn", "allow"}, default="allow" Reaction to a zero-variance (constant) input column. out_of_range : {"error", "warn", "clip", "extrapolate"}, default="extrapolate" Reaction to ``transform``-time values outside the fitted range. - invalid : {"error", "propagate"}, default="error" - Reaction to non-finite (``inf`` / ``-inf``) input values. """ - missing: str = "propagate" constant: str = "allow" out_of_range: str = "extrapolate" - invalid: str = "error" def __post_init__(self): for name, choices in _CHOICES.items(): diff --git a/pretab/embedding/language.py b/pretab/embedding/language.py index 847ce28..e8cded9 100644 --- a/pretab/embedding/language.py +++ b/pretab/embedding/language.py @@ -79,12 +79,21 @@ def fit(self, X, y=None): X = np.asarray(X) self.n_features_in_ = X.shape[1] if X.ndim > 1 else 1 self.model_ = self._resolve_model() - # Read the embedding dim without calling encode() so call-count stays - # predictable; fall back to the 'dim' attribute used by test stubs. + # Prefer reading the embedding dim without calling encode(), so the + # common SentenceTransformer / 'dim'-stub paths keep a predictable call + # count; only fall back to probing when neither hook is available. if hasattr(self.model_, "get_sentence_embedding_dimension"): self.embedding_dim_ = int(self.model_.get_sentence_embedding_dimension()) + elif hasattr(self.model_, "dim"): + self.embedding_dim_ = int(self.model_.dim) else: - self.embedding_dim_ = int(getattr(self.model_, "dim", 0)) + # Neither introspection hook is available (e.g. a bare custom + # ``encode()``-only model): probe with a placeholder input, since + # this is the only generic way to learn the output width. Without + # this, embedding_dim_ silently defaulted to 0 while transform() + # still produced real-width output, a length mismatch. + probe = np.asarray(self.model_.encode(["__pretab_probe__"], convert_to_numpy=True)) + self.embedding_dim_ = int(probe.shape[-1]) return self def transform(self, X): diff --git a/pretab/encoding/categorical/ordinal.py b/pretab/encoding/categorical/ordinal.py index ba510bc..24621ef 100644 --- a/pretab/encoding/categorical/ordinal.py +++ b/pretab/encoding/categorical/ordinal.py @@ -6,12 +6,23 @@ from ...exceptions import PretabDataError +def _is_missing_value(value) -> bool: + """Return True for ``None`` or a NaN float; False for any other value.""" + if value is None: + return True + try: + return bool(np.isnan(value)) + except TypeError: + return False + + class ContinuousOrdinalTransformer(RepresentationSpecMixin, TransformerMixin, BaseEstimator): """Encode categorical features as continuous integer values. Each unique category within a feature is assigned an integer based on its - sorted order. Unknown or missing categories are mapped to ``0``. This is - useful for models that can only handle numerical input. + sorted order. Unknown, missing (``None`` or NaN), or unseen categories are + mapped to ``0``. This is useful for models that can only handle numerical + input. Attributes ---------- @@ -21,7 +32,7 @@ class ContinuousOrdinalTransformer(RepresentationSpecMixin, TransformerMixin, Ba Notes ----- Categories are numbered starting at ``1`` in sorted order; the value ``0`` is - reserved for categories not seen during ``fit`` (and for ``None``). + reserved for categories not seen during ``fit`` (and for ``None`` / NaN). Examples -------- @@ -55,9 +66,16 @@ def fit(self, X, y=None): X = np.asarray(X, dtype=object) if X.ndim == 1: X = X.reshape(-1, 1) - self.mapping_ = [{category: i + 1 for i, category in enumerate(np.unique(X[:, j]))} for j in range(X.shape[1])] - for mapping in self.mapping_: - mapping[None] = 0 # Assign 0 to unknown values + self.mapping_ = [] + for j in range(X.shape[1]): + column = X[:, j] + # Missing values (None / NaN) are excluded before sorting: np.unique + # cannot compare a NaN or None against a string category. + non_missing = np.array([v for v in column if not _is_missing_value(v)], dtype=object) + categories = np.unique(non_missing) if non_missing.size else np.array([], dtype=object) + mapping = {category: i + 1 for i, category in enumerate(categories)} + mapping[None] = 0 # Assign 0 to unknown/missing values + self.mapping_.append(mapping) self.n_features_in_ = len(self.mapping_) return self diff --git a/tests/embedding/test_language_embedding_transformer.py b/tests/embedding/test_language_embedding_transformer.py index 07c91fc..9142e3c 100644 --- a/tests/embedding/test_language_embedding_transformer.py +++ b/tests/embedding/test_language_embedding_transformer.py @@ -71,7 +71,9 @@ def test_fit_transform_invokes_model_encode(): transformer = LanguageEmbeddingTransformer(model=dummy) embeddings = transformer.fit_transform(np.array([["a"], ["b"], ["c"]])) assert embeddings.shape == (3, 4) - assert dummy.calls == 1 + # 1 call from fit()'s dimension probe (dummy has no 'dim'/ + # get_sentence_embedding_dimension) + 1 real encode call from transform(). + assert dummy.calls == 2 def test_transform_multi_column_preserves_row_count(): @@ -81,7 +83,21 @@ def test_transform_multi_column_preserves_row_count(): transformer = LanguageEmbeddingTransformer(model=dummy) embeddings = transformer.fit_transform(np.array([["a", "b"], ["c", "d"], ["e", "f"]])) assert embeddings.shape == (3, 8) # 2 columns x 4-dim embedding - assert dummy.calls == 2 # one encode call per column + assert dummy.calls == 3 # 1 dimension probe (fit) + 2 real encode calls (transform, one per column) + + +def test_fit_infers_embedding_dim_from_probe_without_introspection_hook(): + # Regression test: a model exposing only encode() (no 'dim' / + # get_sentence_embedding_dimension) used to leave embedding_dim_ at the + # default 0, so get_feature_names_out() returned an empty array while + # transform() still produced real-width output, a length mismatch. + dummy = _DummyModel() + transformer = LanguageEmbeddingTransformer(model=dummy) + Xt = transformer.fit(np.array([["a"], ["b"]])).transform(np.array([["a"], ["b"]])) + + assert transformer.embedding_dim_ == 4 # matches _DummyModel.encode's output width + names = transformer.get_feature_names_out() + assert len(names) == Xt.shape[1] @pytest.mark.parametrize( @@ -97,7 +113,9 @@ def test_transform_rejects_fitted_feature_count_mismatch(X_transform): with pytest.raises(PretabDataError, match="is expecting 2 features"): transformer.transform(X_transform) - assert dummy.calls == 0 + # Only fit()'s dimension probe ran; the shape check in transform() rejects + # the mismatched input before any real encode() call. + assert dummy.calls == 1 def test_fit_without_dependency_raises(monkeypatch): diff --git a/tests/encoding/categorical/test_continuous_ordinal_transformer.py b/tests/encoding/categorical/test_continuous_ordinal_transformer.py new file mode 100644 index 0000000..5f3e6aa --- /dev/null +++ b/tests/encoding/categorical/test_continuous_ordinal_transformer.py @@ -0,0 +1,50 @@ +import numpy as np +import pandas as pd + +from pretab.transformers import ContinuousOrdinalTransformer + + +def test_continuous_ordinal_fit_transform_handles_nan(): + # Regression test: np.unique() on an object column crashes when it mixes + # strings with NaN/None, despite the transformer declaring allow_nan=True. + X = np.array([["a"], ["b"], [np.nan], ["a"]], dtype=object) + transformer = ContinuousOrdinalTransformer() + Xt = transformer.fit_transform(X) + + assert Xt.ravel().tolist() == [1, 2, 0, 1] + + +def test_continuous_ordinal_fit_transform_handles_none(): + X = np.array([["a"], ["b"], [None], ["a"]], dtype=object) + transformer = ContinuousOrdinalTransformer() + Xt = transformer.fit_transform(X) + + assert Xt.ravel().tolist() == [1, 2, 0, 1] + + +def test_continuous_ordinal_handles_pandas_categorical_export(): + # A realistic pandas categorical export: object dtype, mixed strings and a + # missing marker (NaN), round-tripping through fit/transform without error. + df = pd.DataFrame({"col": pd.Categorical(["x", "y", None, "x", "z"])}) + X = df["col"].to_numpy(dtype=object).reshape(-1, 1) + + transformer = ContinuousOrdinalTransformer() + Xt = transformer.fit(X).transform(X) + + assert Xt.shape == (5, 1) + assert Xt[2, 0] == 0 # the missing row maps to the reserved code + + +def test_continuous_ordinal_allow_nan_tag_is_exercised(): + tags = ContinuousOrdinalTransformer().__sklearn_tags__() + assert tags.input_tags.allow_nan is True + + # Not just declared: actually fitting/transforming NaN must not raise. + X = np.array([["a"], [np.nan], ["b"]], dtype=object) + ContinuousOrdinalTransformer().fit_transform(X) + + +def test_continuous_ordinal_unseen_category_maps_to_zero(): + transformer = ContinuousOrdinalTransformer().fit(np.array([["a"], ["b"]], dtype=object)) + Xt = transformer.transform(np.array([["c"]], dtype=object)) + assert Xt.ravel().tolist() == [0] From 150a01273dea535c817913040793e96e652633f1 Mon Sep 17 00:00:00 2001 From: Manish Kumar Date: Fri, 4 Sep 2026 18:48:49 +0200 Subject: [PATCH 02/19] fix: use configured dtype in output-budget memory estimate --- pretab/preprocessor.py | 9 +++++++-- tests/integration/test_output_budget.py | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/pretab/preprocessor.py b/pretab/preprocessor.py index 21a8b4d..83c9537 100644 --- a/pretab/preprocessor.py +++ b/pretab/preprocessor.py @@ -741,8 +741,13 @@ def output_dims_(self) -> dict: return dims def _output_itemsize(self) -> int: - """Bytes per element of the dense transformed array (float64 for now).""" - return np.dtype(np.float64).itemsize + """Bytes per element of the dense transformed array. + + Reflects the configured ``dtype`` when set (the cast :meth:`transform` + actually applies); falls back to ``float64``, the natural output dtype + of most representations when no cast is requested. + """ + return np.dtype(self.dtype if self.dtype is not None else np.float64).itemsize def estimate_output_shape(self, X) -> tuple: """Estimate the shape of the dense transformed array for ``X``. diff --git a/tests/integration/test_output_budget.py b/tests/integration/test_output_budget.py index d267f63..fad02c8 100644 --- a/tests/integration/test_output_budget.py +++ b/tests/integration/test_output_budget.py @@ -55,6 +55,25 @@ def test_estimate_memory_is_rows_times_cols_times_itemsize(frame, y): assert pre.estimate_memory(frame) == n_rows * n_cols * np.dtype(np.float64).itemsize +def test_estimate_memory_reflects_configured_dtype(frame, y): + # Regression test: _output_itemsize() used to hardcode float64 regardless of + # `dtype`, so estimate_memory() was 2x too high for a float32 configuration. + pre = _bspline(dtype=np.float32).fit(frame, y) + out = pre.transform(frame, return_array=True) + assert isinstance(out, np.ndarray) + assert out.dtype == np.float32 + assert pre.estimate_memory(frame) == out.nbytes + + +def test_max_dense_memory_does_not_false_positive_reject_float32(frame, y): + # A budget that only fits the real float32 output (not the float64 estimate + # the old bug would have used) must still be accepted, not rejected. + pre = _bspline(dtype=np.float32).fit(frame, y) + real_out = pre.transform(frame, return_array=True) + assert isinstance(real_out, np.ndarray) + _bspline(dtype=np.float32, max_dense_memory=real_out.nbytes, overflow_policy="error").fit(frame, y) + + def test_estimate_shape_scales_with_new_rows(frame, y): pre = _bspline().fit(frame, y) bigger = pd.concat([frame] * 3, ignore_index=True) From 769e7b105b15b87e9b75949fa803c690058808c4 Mon Sep 17 00:00:00 2001 From: Manish Kumar Date: Sat, 5 Sep 2026 18:12:46 +0200 Subject: [PATCH 03/19] feat: default Preprocessor output to a single array --- README.md | 16 +- docs/core_concepts/configuration.md | 1 + docs/core_concepts/outputs_and_inspection.md | 237 +++++++++++++++++- docs/getting_started/choosing_an_interface.md | 28 ++- docs/getting_started/quickstart.md | 14 +- docs/homepage.md | 14 +- docs/tutorials/nonlinear_regression.md | 12 +- docs/tutorials/sklearn_pipeline.md | 22 +- docs/tutorials/target_aware_classification.md | 4 +- pretab/core/policy.py | 5 +- pretab/preprocessor.py | 78 ++++-- tests/compose/test_method_aliases.py | 2 +- tests/integration/test_adaptive_output_dim.py | 4 +- tests/integration/test_output_format.py | 14 +- tests/integration/test_output_structure.py | 67 +++++ tests/integration/test_preprocessor.py | 22 +- 16 files changed, 453 insertions(+), 87 deletions(-) create mode 100644 tests/integration/test_output_structure.py diff --git a/README.md b/README.md index 96facae..4a12f18 100644 --- a/README.md +++ b/README.md @@ -73,10 +73,10 @@ y = np.random.randn(100) # Global strategies: PLE for numerics, integer codes for categoricals preprocessor = Preprocessor(numerical_method="ple", categorical_method="int") -X = preprocessor.fit_transform(df, y) # dict of transformed feature blocks +X = preprocessor.fit_transform(df, y) # single stacked array, one row per sample -print({k: v.shape for k, v in X.items()}) -# {'num_age': (100, 7), 'num_income': (100, 7), 'cat_city': (100, 1)} +print(X.shape) +# (100, 15) ``` > **Note:** PreTab accepts a `pandas.DataFrame` or a `numpy.ndarray` and infers numerical @@ -209,8 +209,8 @@ preprocessor = Preprocessor( task="regression", ) -X_dict = preprocessor.fit_transform(df, y) # {"num_age": ..., "cat_city": ...} -X_array = preprocessor.transform(df, return_array=True) # single stacked ndarray +X_array = preprocessor.fit_transform(df, y) # single stacked ndarray +X_dict = preprocessor.transform(df, return_array=False) # {"num_age": ..., "cat_city": ...} preprocessor.get_feature_info(verbose=True) # inspect resolved strategies ``` @@ -227,8 +227,10 @@ experience numerical imputer -> minmax -> quantile 1 - city categorical imputer -> onehot -> to_float 4 4 ``` -> **Note:** `transform` returns a dict of feature blocks by default (keys prefixed `num_` -> and `cat_`), or a single stacked array when you pass `return_array=True`. +> **Note:** `transform` returns a single stacked array by default, so a `Preprocessor` drops +> straight into a plain `sklearn.pipeline.Pipeline`. Pass `output_structure="blocks"` (or +> `return_array=False` for a single call) for the dict-of-feature-blocks form instead (keys +> prefixed `num_` and `cat_`). ### Standalone transformers diff --git a/docs/core_concepts/configuration.md b/docs/core_concepts/configuration.md index 558c9ff..713d302 100644 --- a/docs/core_concepts/configuration.md +++ b/docs/core_concepts/configuration.md @@ -122,6 +122,7 @@ in depth. | `target_aware`, `placement_strategy` | `True`, `"cart"` | [Target awareness](target_awareness.md) | | `numerical_imputation`, `categorical_imputation`, `add_missing_indicator` | `"median"`, `"most_frequent"`, `False` | [Missing values](missing_values.md) | | `output_format`, `dtype` | `"dense"`, `None` | [Outputs and inspection](outputs_and_inspection.md) | +| `output_structure` | `"matrix"` | [Outputs and inspection](outputs_and_inspection.md) | | `random_state` | `None` | [Reproducibility](reproducibility.md) | ## Where to go next diff --git a/docs/core_concepts/outputs_and_inspection.md b/docs/core_concepts/outputs_and_inspection.md index 26950fb..620c8aa 100644 --- a/docs/core_concepts/outputs_and_inspection.md +++ b/docs/core_concepts/outputs_and_inspection.md @@ -7,22 +7,90 @@ feature names, lineage, and the output budget. ## Output shapes -`fit_transform` and `transform` return a dictionary that maps each feature to its transformed -block, with keys prefixed `num_` or `cat_`. Pass `return_array=True` to receive a single -stacked `numpy.ndarray` instead. +`fit_transform` and `transform` return a single stacked `numpy.ndarray` by default. Pass +`output_structure="blocks"` (or `return_array=False` for a single call) to receive a +dictionary that maps each feature to its transformed block instead, with keys prefixed +`num_` or `cat_`. ```python -X_dict = pre.fit_transform(df, y) # {"num_age": ..., "cat_city": ...} -X_array = pre.transform(df, return_array=True) # one stacked ndarray +X_array = pre.fit_transform(df, y) # one stacked ndarray +X_dict = Preprocessor(output_structure="blocks").fit_transform(df, y) # {"num_age": ..., "cat_city": ...} ``` ```{note} -The dict form is convenient for inspection and for feeding blocks to different model heads. -The array form is what a plain scikit-learn estimator expects. Choose per call. +The array form is what a plain scikit-learn estimator expects, and is what lets a +`Preprocessor` drop straight into a `Pipeline`. The dict form is convenient for inspection and +for feeding blocks to different model heads; pass `return_array` explicitly to override +`output_structure` for a single call without changing the estimator's configuration. +``` + +### Example: one column expands into several + +A single input column rarely maps to a single output column. Most numerical methods (splines, +feature maps, PLE) expand one feature into `output_dim` basis columns, so the matrix width +grows well beyond the number of input columns. A B-spline makes this concrete: `output_dim=6` +turns the one `age` column into 6 local basis functions, each capturing a different region of +its range. + +```python +df = pd.DataFrame({"age": [25, 40, 63, 51]}) +y = [0.1, 0.9, 0.3, 0.6] + +pre = Preprocessor(numerical_method="bspline", output_dim=6, + target_aware=False, placement_strategy="quantile").fit(df, y) +out = pre.transform(df) +out.shape +``` + +```text +(4, 6) +``` + +```text +array([[1. , 0. , 0. , 0. , 0. , 0. ], + [0. , 0.179, 0.593, 0.228, 0. , 0. ], + [0. , 0. , 0. , 0. , 0. , 1. ], + [0. , 0. , 0.165, 0.607, 0.229, 0. ]]) +``` + +4 input rows, 1 input column, but 6 output columns: every row's `age` value is spread across +the basis functions whose local support it falls into (each row sums to 1, since a B-spline +basis is a partition of unity). `get_feature_names_out()` shows exactly where each column came +from: + +```python +list(pre.get_feature_names_out()) +``` + +```text +['num_age_bs0', 'num_age_bs1', 'num_age_bs2', 'num_age_bs3', 'num_age_bs4', 'num_age_bs5'] +``` + +All six still trace back to the single `age` input, which is exactly what `output_structure= +"blocks"` reflects: one dict entry per **input** feature, holding its full expanded block, not +one entry per output column. + +```python +pre_blocks = Preprocessor(numerical_method="bspline", output_dim=6, + target_aware=False, placement_strategy="quantile", + output_structure="blocks").fit(df, y) +pre_blocks.transform(df)["num_age"].shape +``` + +```text +(4, 6) +``` + +```{tip} +This is why a wide expansion (a spline or feature map with a large `output_dim`, or several +expanded columns) can produce a much wider matrix than the input `DataFrame` had columns. +Use `get_feature_info(verbose=True)` (below) or `estimate_output_shape(df)` to see the total +width before committing, especially with several expanded columns at once. ``` ## Output format and dtype + Two parameters control the physical layout of the stacked output. `output_format` @@ -57,6 +125,161 @@ Polars output is loaded lazily. If polars is not installed, requesting it raises `OptionalDependencyError` rather than failing deep in the call stack. ``` +## Choosing your output settings + +Four things independently affect what `transform` / `fit_transform` return: `output_structure` +(top-level shape), `return_array` (a per-call override of it), `output_format` (dense vs. +sparse), and `set_output` (scikit-learn's own DataFrame protocol). This section is the +decision guide: what each one controls, how they interact, and which combination fits a given +use case, with the literal output shown for each rather than just a description of it. + +The examples below all share the same tiny, fixed input so the printed output is directly +comparable: + +```python +import numpy as np +import pandas as pd +from pretab import Preprocessor + +df = pd.DataFrame({"age": [25, 40, 63], "city": ["A", "B", "A"]}) +y = [0.1, 0.9, 0.3] +``` + +### Parameter impact at a glance + +| Parameter | Values | Controls | Set at | +| --- | --- | --- | --- | +| `output_structure` | `"matrix"` (default), `"blocks"` | Whether `transform()` returns one stacked array or a dict of per-feature blocks, when `return_array` is not passed. | Constructor | +| `return_array` | `True`, `False`, `None` (default) | Overrides `output_structure` for a single call. `None` resolves from `output_structure`. | Per call | +| `output_format` | `"dense"` (default), `"sparse"`, `"auto"` | Whether the array (or each block) is a NumPy array or a SciPy CSR matrix. `"auto"` picks sparse only when it saves memory. | Constructor | +| `dtype` | e.g. `numpy.float32`, `None` (default) | Floating-point precision of the output; also what `estimate_memory()` assumes. | Constructor | +| `set_output(transform=...)` | `"default"`, `"pandas"`, `"polars"` | Wraps the array in a DataFrame. **Takes priority over `output_structure` and `return_array` entirely**: once set, `transform()` always returns a DataFrame, never a dict or a bare array. | Method call, before `transform` | + +### Which setting should I use? + +Feeding a plain scikit-learn estimator or building a `Pipeline` +: Use the default (`output_structure="matrix"`, no `return_array` override). `Preprocessor()` + composes directly: `Pipeline([("pretab", Preprocessor(...)), ("model", Ridge())])` just + works, since `transform(X)` already returns a single array. + + ```python + pre = Preprocessor(numerical_method="minmax", categorical_method="one-hot").fit(df, y) + out = pre.transform(df) + type(out), out.shape + ``` + + ```text + (, (3, 3)) + ``` + + ```text + array([[0. , 1. , 0. ], + [0.39473684, 0. , 1. ], + [1. , 1. , 0. ]]) + ``` + + Column order matches `get_feature_names_out()`: the scaled `age`, then the one-hot `city_A` + / `city_B` columns. + +Inspecting per-feature blocks, or feeding different blocks to different model heads +: Set `output_structure="blocks"` on the constructor (so it stays the estimator's default + everywhere it's reused), or pass `return_array=False` for a one-off call without changing + the estimator's configuration. + + ```python + pre = Preprocessor(numerical_method="minmax", categorical_method="one-hot", + output_structure="blocks").fit(df, y) + out = pre.transform(df) + out + ``` + + ```text + {'num_age': array([[0. ], + [0.39473684], + [1. ]]), + 'cat_city': array([[1., 0.], + [0., 1.], + [1., 0.]])} + ``` + + The same estimator still returns an array for a single call if you ask for one: + + ```python + pre.transform(df, return_array=True) # ndarray, shape (3, 3); this call only + ``` + +Passing external `embeddings` +: **Embeddings require dict output.** They are separate named blocks, so they cannot be + stacked into a single matrix. Use `output_structure="blocks"` (or `return_array=False`) on + every `transform` call that passes `embeddings`; the default `"matrix"` raises + `IncompatibleParamsError` the moment `embeddings` is supplied. + + ```python + emb = np.array([[0.1, 0.2], [0.3, 0.4], [0.5, 0.6]]) + pre = Preprocessor(numerical_method="minmax", categorical_method="one-hot", + output_structure="blocks").fit(df, y, embeddings=emb) + out = pre.transform(df, embeddings=emb) + sorted(out.keys()), out["embedding_1"].shape + ``` + + ```text + (['cat_city', 'embedding_1', 'num_age'], (3, 2)) + ``` + +Large, mostly one-hot-encoded categorical data +: Set `output_format="sparse"` (or `"auto"` to let PreTab decide per fit). This applies + independently of `output_structure`: a sparse `"matrix"` is a single stacked + `scipy.sparse.csr_matrix`, and a sparse `"blocks"` dict holds a CSR matrix per feature. + + ```python + pre = Preprocessor(numerical_method="minmax", categorical_method="one-hot", + output_format="sparse").fit(df, y) + pre.transform(df) + ``` + + ```text + + ``` + +Halving memory on a large dataset +: Set `dtype=numpy.float32`. `estimate_memory()` and the `max_dense_memory` budget both + reflect the configured `dtype`, not a hardcoded `float64` assumption, so budgets sized for + the real, cast output are honored correctly. + + ```python + pre = Preprocessor(numerical_method="minmax", categorical_method="one-hot", + dtype=np.float32).fit(df, y) + pre.transform(df).dtype + ``` + + ```text + dtype('float32') + ``` + +Feeding a library that expects a DataFrame, or wanting column names attached to the output +: Call `pre.set_output(transform="pandas")` (or `"polars"`). This overrides everything else: + `transform()` always returns a DataFrame from that point on, regardless of `output_structure` + or any `return_array` passed to an individual call. + + ```python + pre = Preprocessor(numerical_method="minmax", categorical_method="one-hot").fit(df, y) + pre.set_output(transform="pandas").transform(df) + ``` + + ```text + num_age cat_city_A cat_city_B + 0 0.000000 1.0 0.0 + 1 0.394737 0.0 1.0 + 2 1.000000 1.0 0.0 + ``` + +```{warning} +`set_output` always wins. If a downstream step unexpectedly receives a DataFrame instead of +the array or dict you configured, check whether `set_output` was called anywhere upstream +(including by a cloned copy inside a `Pipeline`/`GridSearchCV`). +``` + ## Feature names Every representation names its output columns, and the names are stable and descriptive. diff --git a/docs/getting_started/choosing_an_interface.md b/docs/getting_started/choosing_an_interface.md index 2663b79..3dba5f4 100644 --- a/docs/getting_started/choosing_an_interface.md +++ b/docs/getting_started/choosing_an_interface.md @@ -11,7 +11,7 @@ ergonomics, not capability. This page helps you pick. :::{grid-item-card} `Preprocessor` Reads a `DataFrame`, detects numerical and categorical columns, and applies a strategy per -column from a single configuration object. Returns a dict of feature blocks by default. +column from a single configuration object. Returns a single stacked array by default. ::: :::{grid-item-card} Standalone transformers @@ -40,7 +40,7 @@ pre = Preprocessor(feature_preprocessing={ "income": "ple", "city": "one-hot", }) -X = pre.fit_transform(df, y) # dict of blocks, or return_array=True for one matrix +X = pre.fit_transform(df, y) # one stacked array, or output_structure="blocks" for a dict ``` ## Reach for standalone transformers when @@ -65,10 +65,26 @@ model = Pipeline([("features", features), ("ridge", Ridge())]) ``` ```{note} -The `Preprocessor` returns a dict by default, which is convenient for inspection but is not a -drop-in for a scikit-learn estimator that expects a single matrix. Call it with -`return_array=True`, or use the standalone transformers, when you compose one end-to-end -`Pipeline`. +The `Preprocessor` returns a single stacked array by default, so it drops directly into a +plain `Pipeline`/`ColumnTransformer` like any other scikit-learn transformer. Pass +`output_structure="blocks"` (or `return_array=False` for a single call) when you want the +dict-of-feature-blocks form instead, for inspection or per-block downstream heads. +``` + +A `Preprocessor` composes the same way as the standalone transformers above: + +```python +from sklearn.pipeline import Pipeline +from sklearn.linear_model import Ridge + +from pretab import Preprocessor + +model = Pipeline([ + ("pretab", Preprocessor(feature_preprocessing={"age": "naturalspline", "income": "ple"})), + ("ridge", Ridge()), +]) +model.fit(df, y) +model.predict(df) ``` ## A note on multivariate methods diff --git a/docs/getting_started/quickstart.md b/docs/getting_started/quickstart.md index 6adf36b..a84eb59 100644 --- a/docs/getting_started/quickstart.md +++ b/docs/getting_started/quickstart.md @@ -16,8 +16,8 @@ LightGBM-based placement. ## Fit a `Preprocessor` The `Preprocessor` inspects a `DataFrame`, decides which columns are numerical and which are -categorical, and applies a strategy per column. It returns a dictionary of feature blocks by -default, or a single stacked array on request. +categorical, and applies a strategy per column. It returns a single stacked array by default, +or a dict of per-feature blocks on request. ```python import numpy as np @@ -44,9 +44,9 @@ config = { } pre = Preprocessor(feature_preprocessing=config, task="regression", random_state=0) -# Fit and transform into a dict of feature blocks -X_dict = pre.fit_transform(df, y) -{k: v.shape for k, v in X_dict.items()} +# Fit and transform into a single stacked array +X = pre.fit_transform(df, y) +X.shape ``` ```{tip} @@ -55,10 +55,10 @@ When no per-feature config is given, the `Preprocessor` falls back to its global [Configuration](../core_concepts/configuration.md) for every knob. ``` -Ask for a single stacked matrix instead when you feed a plain estimator: +Ask for a dict of per-feature blocks instead, for inspection or per-block downstream heads: ```python -X = pre.transform(df, return_array=True) # one ndarray, one row per sample +X_dict = pre.transform(df, return_array=False) # {"num_age": ..., "cat_city": ...} ``` ## Inspect what was built diff --git a/docs/homepage.md b/docs/homepage.md index 2796698..e15bf54 100644 --- a/docs/homepage.md +++ b/docs/homepage.md @@ -56,15 +56,15 @@ y = np.random.randn(100) # One strategy per feature type: PLE for numerics, integer codes for categoricals pre = Preprocessor(numerical_method="ple", categorical_method="int") -X = pre.fit_transform(df, y) # dict of model-ready feature blocks - -{k: v.shape for k, v in X.items()} -# {'num_age': (100, 7), 'num_income': (100, 7), 'cat_city': (100, 1)} +X = pre.fit_transform(df, y) # single stacked, model-ready array +X.shape +# (100, 15) ``` -`Preprocessor` detects the column types, fits a strategy per column, and returns model-ready -arrays, either as a dict of blocks or, with `return_array=True`, a single stacked matrix. -Inspect the resolved layout at any time with `get_feature_info(verbose=True)`: +`Preprocessor` detects the column types, fits a strategy per column, and returns a single +stacked, model-ready array by default, so it drops straight into a plain +`sklearn.pipeline.Pipeline`; pass `output_structure="blocks"` for a dict of per-feature blocks +instead. Inspect the resolved layout at any time with `get_feature_info(verbose=True)`: ```text feature kind pipeline dim cats diff --git a/docs/tutorials/nonlinear_regression.md b/docs/tutorials/nonlinear_regression.md index 845199c..cd2b323 100644 --- a/docs/tutorials/nonlinear_regression.md +++ b/docs/tutorials/nonlinear_regression.md @@ -108,8 +108,8 @@ pre = Preprocessor( output_dim=12, ) -X_tr = pre.fit_transform(X_train, y_train, return_array=True) -X_te = pre.transform(X_test, return_array=True) +X_tr = pre.fit_transform(X_train, y_train) +X_te = pre.transform(X_test) model = Ridge(alpha=1.0).fit(X_tr, y_train) pred = model.predict(X_te) @@ -130,10 +130,10 @@ nonlinear structure. The $R^2$ jumps from `0.124` to `0.979` and the mean absolu from `11.20` to `1.85`. ```{tip} -`Preprocessor.transform` returns a dict of feature blocks by default. When you feed a plain -estimator, call it with `return_array=True` to get a single stacked matrix. To compose -everything inside one scikit-learn `Pipeline` instead, use the standalone transformers, shown -in the [sklearn pipeline tutorial](sklearn_pipeline.md). +`Preprocessor.transform` returns a single stacked array by default, so it drops straight into +a plain scikit-learn estimator or `Pipeline`. Pass `output_structure="blocks"` (or +`return_array=False` for a single call) for the dict-of-feature-blocks form instead. See the +[sklearn pipeline tutorial](sklearn_pipeline.md) for wiring each column's transformer by hand. ``` ## What actually changed diff --git a/docs/tutorials/sklearn_pipeline.md b/docs/tutorials/sklearn_pipeline.md index a04e2fa..98fa857 100644 --- a/docs/tutorials/sklearn_pipeline.md +++ b/docs/tutorials/sklearn_pipeline.md @@ -1,10 +1,11 @@ # Inside an sklearn Pipeline -The high-level `Preprocessor` returns a dict by default, so it is used as an explicit -feature-building step (call it with `return_array=True`, then fit the model on the arrays). -The **standalone transformers**, on the other hand, return plain arrays and follow the -`sklearn` API exactly, so they drop straight into a `ColumnTransformer` and `Pipeline`, and -work with `cross_val_score`, `GridSearchCV`, and every other `sklearn` utility. +The **standalone transformers** return plain arrays and follow the `sklearn` API exactly, so +they drop straight into a `ColumnTransformer` and `Pipeline`, and work with +`cross_val_score`, `GridSearchCV`, and every other `sklearn` utility. The high-level +`Preprocessor` composes the same way (it returns a single stacked array by default), but this +page focuses on wiring each column's transformer by hand for fine-grained, per-transformer +control: addressable hyperparameters (`step__param`), one estimator per column. This tutorial builds the regression task from the [nonlinear regression tutorial](nonlinear_regression.md) as a single, self-contained @@ -110,11 +111,12 @@ Every pretab transformer participates in the search grid just like a native `skl ## When to use which - **Standalone transformers** (this page) compose inside one `Pipeline` and integrate with - cross-validation and grid search. Reach for them when you want a single estimator object. -- **The `Preprocessor`** (the [nonlinear regression tutorial](nonlinear_regression.md)) reads - a `DataFrame`, detects feature types automatically, and configures every column from a - single config. Reach for it when you want per-column strategies without wiring each one by - hand. + cross-validation and grid search, with each column's transformer addressable individually + (`step__param`). Reach for them when you want fine control over one column's transformer. +- **The `Preprocessor`** (the [nonlinear regression tutorial](nonlinear_regression.md)) also + composes inside a `Pipeline` (it returns a single stacked array by default), reads a + `DataFrame`, detects feature types automatically, and configures every column from a single + config. Reach for it when you want per-column strategies without wiring each one by hand. ## Next steps diff --git a/docs/tutorials/target_aware_classification.md b/docs/tutorials/target_aware_classification.md index d9c4d69..25780f7 100644 --- a/docs/tutorials/target_aware_classification.md +++ b/docs/tutorials/target_aware_classification.md @@ -93,8 +93,8 @@ pre = Preprocessor( output_dim=10, ) -X_tr = pre.fit_transform(X_train, y_train, return_array=True) -X_te = pre.transform(X_test, return_array=True) +X_tr = pre.fit_transform(X_train, y_train) +X_te = pre.transform(X_test) clf = LogisticRegression(max_iter=1000).fit(X_tr, y_train) proba = clf.predict_proba(X_te)[:, 1] diff --git a/pretab/core/policy.py b/pretab/core/policy.py index 0609921..d53097c 100644 --- a/pretab/core/policy.py +++ b/pretab/core/policy.py @@ -3,8 +3,8 @@ :class:`RepresentationPolicy` names, in one place, how every transformer reacts to the recurring edge cases that would otherwise diverge silently per family: -* ``constant`` -- zero-variance input columns (``"error"`` / ``"warn"`` / ``"allow"``). -* ``out_of_range`` -- values at ``transform`` outside the fitted range +* ``constant``: zero-variance input columns (``"error"`` / ``"warn"`` / ``"allow"``). +* ``out_of_range``: values at ``transform`` outside the fitted range (``"error"`` / ``"warn"`` / ``"clip"`` / ``"extrapolate"``). The defaults reproduce the library's historical behaviour (constant columns pass @@ -28,6 +28,7 @@ expansions" section of ``dev/todo/release-1.0.0/bugfixes-1.0.0.md``. """ + from __future__ import annotations import warnings diff --git a/pretab/preprocessor.py b/pretab/preprocessor.py index 83c9537..ed265e7 100644 --- a/pretab/preprocessor.py +++ b/pretab/preprocessor.py @@ -233,6 +233,14 @@ class Preprocessor(TransformerMixin, BaseEstimator): :class:`~pretab.exceptions.OutputBudgetError`, ``"warn"`` emits a :class:`~pretab.exceptions.ConfigWarning`, ``"ignore"`` proceeds silently. Only takes effect when at least one budget parameter above is set. + output_structure : {"matrix", "blocks"}, default="matrix" + Top-level shape ``transform`` / ``fit_transform`` return when ``return_array`` is not + passed explicitly. ``"matrix"`` (the default) returns a single stacked array, so a + plain ``sklearn.pipeline.Pipeline([("pretab", Preprocessor(...)), ("model", + estimator)])`` composes like any other transformer. ``"blocks"`` returns the dict of + per-feature blocks instead (the library's pre-1.0 default), useful for inspection or + when a downstream step consumes named blocks directly. Passing ``return_array`` + explicitly to ``transform`` / ``fit_transform`` always overrides this setting. output_format : {"dense", "sparse", "auto"}, default="dense" Container used for the transformed output. ``"dense"`` (the default, for backward compatibility) returns NumPy arrays; ``"sparse"`` returns SciPy @@ -307,8 +315,10 @@ class Preprocessor(TransformerMixin, BaseEstimator): ``"dummy"`` -> ``"one-hot"``, ``"ordinal"`` / ``"label"`` -> ``"int"``, ``"poly"`` -> ``"polynomial"``, ``"thin-plate"`` -> ``"tprs"``, and ``"passthrough"`` -> ``"none"``. - ``transform`` returns a dict of per-feature blocks keyed ``num_`` / ``cat_`` by - default, or a single stacked array when ``return_array=True``. + ``transform`` returns a single stacked array by default (``output_structure="matrix"``), + or a dict of per-feature blocks keyed ``num_`` / ``cat_`` when + ``output_structure="blocks"``. Passing ``return_array`` explicitly to ``transform`` / + ``fit_transform`` always overrides ``output_structure`` for that call. Examples -------- @@ -320,8 +330,8 @@ class Preprocessor(TransformerMixin, BaseEstimator): >>> y = [0.1, 0.4, 0.9, 1.2] >>> pre = Preprocessor() >>> out = pre.fit_transform(df, y) - >>> sorted(out.keys()) - ['cat_gender', 'num_age'] + >>> out.ndim + 2 Cubic-spline basis for numerics with one-hot encoded categoricals: @@ -340,13 +350,20 @@ class Preprocessor(TransformerMixin, BaseEstimator): >>> pre = Preprocessor(feature_preprocessing={"age": "pspline", "gender": "one-hot"}) >>> out = pre.fit_transform(df, y) - Data-driven (adaptive) width, returned as a single stacked array: + Data-driven (adaptive) width: >>> pre = Preprocessor(numerical_method="ple", adaptive=True, ... min_output_dim=4, max_output_dim=12) - >>> arr = pre.fit_transform(df, y, return_array=True) + >>> arr = pre.fit_transform(df, y) >>> arr.ndim 2 + + Per-feature blocks instead of a single matrix: + + >>> pre = Preprocessor(output_structure="blocks") + >>> out = pre.fit_transform(df, y) + >>> sorted(out.keys()) + ['cat_gender', 'num_age'] """ def __init__( @@ -375,6 +392,7 @@ def __init__( max_features_per_input=None, max_dense_memory=None, overflow_policy="error", + output_structure="matrix", output_format="dense", dtype=None, verbose=0, @@ -411,6 +429,7 @@ def __init__( self.max_features_per_input = max_features_per_input self.max_dense_memory = max_dense_memory self.overflow_policy = overflow_policy + self.output_structure = output_structure self.output_format = output_format self.dtype = dtype self.verbose = verbose @@ -499,6 +518,16 @@ def fit(self, X, y=None, embeddings=None): valid=set(valid_formats), ) + valid_structures = ("matrix", "blocks") + if self.output_structure not in valid_structures: + raise invalid_param_error( + type(self).__name__, + "output_structure", + self.output_structure, + "must be one of 'matrix', 'blocks'", + valid=set(valid_structures), + ) + # Ask ColumnTransformer to assemble the representation in the requested # container whenever its component outputs permit it. In particular, # explicit sparse output must not densely stack sparse one-hot blocks. @@ -533,7 +562,7 @@ def fit(self, X, y=None, embeddings=None): return self - def transform(self, X, embeddings=None, return_array=False): + def transform(self, X, embeddings=None, return_array=None): """ Transform the input data using the fitted column transformer. @@ -543,18 +572,22 @@ def transform(self, X, embeddings=None, return_array=False): Input features to transform. embeddings : np.ndarray or list of np.ndarray, optional External embeddings to attach to dictionary output. Required when - embeddings were supplied during ``fit`` and unsupported when - ``return_array=True`` or :meth:`set_output` requests a DataFrame. - return_array : bool, default=False - If True, return a single stacked NumPy array. If False, return a dict of transformed arrays. + embeddings were supplied during ``fit`` and unsupported when the + resolved output is a single array or :meth:`set_output` requests a DataFrame. + return_array : bool or None, default=None + If True, return a single stacked NumPy array; if False, return a dict of + transformed arrays. ``None`` (the default) resolves from ``output_structure``: + ``"matrix"`` behaves like ``True``, ``"blocks"`` like ``False``. Pass this + explicitly to override ``output_structure`` for a single call. Returns ------- dict, np.ndarray, scipy.sparse matrix, or DataFrame - Transformed data. By default a dictionary of per-feature blocks; a - single stacked array when ``return_array=True``; a SciPy CSR matrix (or - CSR blocks) when ``output_format`` resolves to ``"sparse"``; or a pandas - / polars DataFrame when configured via :meth:`set_output`. + Transformed data. A single stacked array by default (``output_structure= + "matrix"``); a dict of per-feature blocks when ``output_structure="blocks"`` + (or ``return_array=False``); a SciPy CSR matrix (or CSR blocks) when + ``output_format`` resolves to ``"sparse"``; or a pandas / polars DataFrame + when configured via :meth:`set_output`. """ check_is_fitted(self) @@ -564,8 +597,10 @@ def transform(self, X, embeddings=None, return_array=False): if self.missing_policy == "error": self._reject_missing(X) + resolved_return_array = (self.output_structure == "matrix") if return_array is None else return_array + container = _get_output_config("transform", self)["dense"] - output_kind = container if container in ("pandas", "polars") else ("array" if return_array else "dict") + output_kind = container if container in ("pandas", "polars") else ("array" if resolved_return_array else "dict") validate_embedding_request(embeddings, expected=self.embeddings_, output_kind=output_kind) transformed_X = self.column_transformer_.transform(X) @@ -579,10 +614,10 @@ def transform(self, X, embeddings=None, return_array=False): if container in ("pandas", "polars"): return to_dataframe_output(transformed_X, self.get_feature_names_out(), container) - slices = None if return_array else get_output_slices(self.column_transformer_) + slices = None if resolved_return_array else get_output_slices(self.column_transformer_) return format_output( transformed_X, - return_array=return_array, + return_array=resolved_return_array, slices=slices, embeddings=embeddings, embeddings_expected=self.embeddings_, @@ -590,7 +625,7 @@ def transform(self, X, embeddings=None, return_array=False): output_format=fmt, ) - def fit_transform(self, X, y=None, embeddings=None, return_array=False): + def fit_transform(self, X, y=None, embeddings=None, return_array=None): """ Convenience method that fits the preprocessor and transforms the data. @@ -602,8 +637,9 @@ def fit_transform(self, X, y=None, embeddings=None, return_array=False): Target values. embeddings : np.ndarray or list of np.ndarray, optional Optional embedding arrays. - return_array : bool, default=False - Whether to return a stacked NumPy array or a dictionary of arrays. + return_array : bool or None, default=None + Whether to return a stacked NumPy array or a dictionary of arrays. ``None`` + (the default) resolves from ``output_structure``. Returns ------- diff --git a/tests/compose/test_method_aliases.py b/tests/compose/test_method_aliases.py index 11c31d5..1ba1afb 100644 --- a/tests/compose/test_method_aliases.py +++ b/tests/compose/test_method_aliases.py @@ -148,7 +148,7 @@ def test_categorical_alias_matches_canonical_output(sample_data, alias, canonica def test_alias_in_feature_preprocessing(sample_data): X, y = sample_data - pre = Preprocessor(feature_preprocessing={"num1": "STD", "cat1": "OneHot"}) + pre = Preprocessor(feature_preprocessing={"num1": "STD", "cat1": "OneHot"}, output_structure="blocks") out = pre.fit_transform(X, y) assert "num_num1" in out assert "cat_cat1" in out diff --git a/tests/integration/test_adaptive_output_dim.py b/tests/integration/test_adaptive_output_dim.py index 2e834ba..3721e13 100644 --- a/tests/integration/test_adaptive_output_dim.py +++ b/tests/integration/test_adaptive_output_dim.py @@ -97,7 +97,7 @@ def data(): def _num_width(X, y, method, **kwargs): """Fit a Preprocessor on one numerical feature and return its block width.""" pre = Preprocessor(numerical_method=method, categorical_method="none", **kwargs) - out = cast("dict[str, np.ndarray]", pre.fit(X, y).transform(X)) + out = cast("dict[str, np.ndarray]", pre.fit(X, y).transform(X, return_array=False)) return out["num_x"].shape[1] @@ -140,7 +140,7 @@ def test_custombin_respects_bin_count(data, output_dim): """Numerical ``custombin`` yields a single column of at most output_dim codes.""" X, y = data pre = Preprocessor(numerical_method="custombin", categorical_method="none", output_dim=output_dim) - block = cast("dict[str, np.ndarray]", pre.fit(X, y).transform(X))["num_x"] + block = cast("dict[str, np.ndarray]", pre.fit(X, y).transform(X, return_array=False))["num_x"] assert block.shape[1] == 1 assert int(block.max()) < output_dim assert len(np.unique(block)) <= output_dim diff --git a/tests/integration/test_output_format.py b/tests/integration/test_output_format.py index 540c39e..c169571 100644 --- a/tests/integration/test_output_format.py +++ b/tests/integration/test_output_format.py @@ -48,7 +48,7 @@ def test_default_output_format_is_dense(frame, y): def test_default_dict_blocks_are_dense(frame, y): p = _bspline().fit(frame, y) - out = p.transform(frame) + out = p.transform(frame, return_array=False) assert isinstance(out, dict) assert all(isinstance(v, np.ndarray) for v in out.values()) @@ -69,7 +69,7 @@ def test_sparse_return_array_is_csr(frame, y): def test_sparse_dict_blocks_are_csr(frame, y): p = _bspline(output_format="sparse").fit(frame, y) - out = p.transform(frame) + out = p.transform(frame, return_array=False) assert isinstance(out, dict) assert all(sp.issparse(v) for v in out.values()) @@ -184,9 +184,17 @@ def test_set_output_pandas_fit_transform(frame, y): assert out.shape[1] == p.total_output_dim_ -def test_set_output_default_still_dict(frame, y): +def test_set_output_default_still_array(frame, y): + # set_output(transform="default") only opts out of pandas/polars wrapping; it + # does not override output_structure, which still resolves to an array. p = _bspline().fit(frame, y).set_output(transform="default") out = p.transform(frame) + assert isinstance(out, np.ndarray) + + +def test_set_output_default_with_blocks_structure_is_dict(frame, y): + p = _bspline(output_structure="blocks").fit(frame, y).set_output(transform="default") + out = p.transform(frame) assert isinstance(out, dict) diff --git a/tests/integration/test_output_structure.py b/tests/integration/test_output_structure.py new file mode 100644 index 0000000..92f7707 --- /dev/null +++ b/tests/integration/test_output_structure.py @@ -0,0 +1,67 @@ +"""``output_structure`` controls the top-level shape ``transform`` / +``fit_transform`` return when ``return_array`` is not passed explicitly. + +Regression coverage for the original bug: a plain ``Preprocessor`` inside a bare +``sklearn.pipeline.Pipeline`` broke the next step, because ``transform()`` +defaulted to a dict and ``Pipeline`` has no way to request ``return_array=True`` +for an intermediate step. +""" + +import numpy as np +import pandas as pd +import pytest +from sklearn.linear_model import Ridge +from sklearn.pipeline import Pipeline + +from pretab import Preprocessor +from pretab.exceptions import InvalidParamError + + +@pytest.fixture +def frame(): + rng = np.random.default_rng(0) + return pd.DataFrame({"a": rng.normal(size=40), "b": rng.normal(size=40)}) + + +@pytest.fixture +def y(): + return np.random.default_rng(1).normal(size=40) + + +def test_default_output_structure_is_matrix(frame, y): + pre = Preprocessor().fit(frame, y) + out = pre.transform(frame) + assert isinstance(out, np.ndarray) + assert out.shape == (len(frame), pre.total_output_dim_) + + +def test_output_structure_blocks_returns_dict(frame, y): + pre = Preprocessor(output_structure="blocks").fit(frame, y) + out = pre.transform(frame) + assert isinstance(out, dict) + assert all(isinstance(v, np.ndarray) for v in out.values()) + + +def test_explicit_return_array_overrides_output_structure(frame, y): + # output_structure="matrix" but return_array=False explicitly -> dict. + matrix_pre = Preprocessor(output_structure="matrix").fit(frame, y) + assert isinstance(matrix_pre.transform(frame, return_array=False), dict) + + # output_structure="blocks" but return_array=True explicitly -> array. + blocks_pre = Preprocessor(output_structure="blocks").fit(frame, y) + assert isinstance(blocks_pre.transform(frame, return_array=True), np.ndarray) + + +def test_invalid_output_structure_raises(frame, y): + with pytest.raises(InvalidParamError, match="output_structure"): + Preprocessor(output_structure="nope").fit(frame, y) + + +def test_preprocessor_composes_in_a_plain_pipeline(frame, y): + # Regression test: Pipeline always calls transform(X) with no return_array + # kwarg, so a dict-by-default Preprocessor broke the next step with + # "TypeError: float() argument must be a string or a real number, not 'dict'". + model = Pipeline([("pretab", Preprocessor()), ("model", Ridge())]) + model.fit(frame, y) + preds = model.predict(frame) + assert preds.shape == (len(frame),) diff --git a/tests/integration/test_preprocessor.py b/tests/integration/test_preprocessor.py index 8c7e291..b7f0e35 100644 --- a/tests/integration/test_preprocessor.py +++ b/tests/integration/test_preprocessor.py @@ -23,10 +23,19 @@ def sample_data(): return df, y -def test_fit_transform_returns_dict(sample_data): +def test_fit_transform_returns_array_by_default(sample_data): X, y = sample_data pre = Preprocessor() out = pre.fit_transform(X, y) + assert isinstance(out, np.ndarray) + assert out.shape[0] == len(X) + assert out.ndim == 2 + + +def test_fit_transform_returns_dict_with_blocks_structure(sample_data): + X, y = sample_data + pre = Preprocessor(output_structure="blocks") + out = pre.fit_transform(X, y) assert isinstance(out, dict) assert all(isinstance(k, str) for k in out) assert all(isinstance(v, np.ndarray) for v in out.values()) @@ -53,7 +62,7 @@ def test_transform_raises_before_fit(sample_data): def test_embedding_integration(sample_data): X, y = sample_data embed = np.random.rand(len(X), 10) - pre = Preprocessor() + pre = Preprocessor(output_structure="blocks") out = pre.fit_transform(X, y, embeddings=embed) assert "embedding_1" in out assert out["embedding_1"].shape == (len(X), 10) @@ -62,7 +71,7 @@ def test_embedding_integration(sample_data): def test_multiple_embeddings(sample_data): X, y = sample_data embeds = [np.random.rand(len(X), 3), np.random.rand(len(X), 7)] - pre = Preprocessor() + pre = Preprocessor(output_structure="blocks") out = pre.fit_transform(X, y, embeddings=embeds) assert "embedding_1" in out and "embedding_2" in out assert out["embedding_1"].shape[1] == 3 @@ -135,7 +144,7 @@ def test_feature_info_returns_three_dicts(sample_data): def test_dict_output_shapes_add_up(sample_data): X, y = sample_data - pre = Preprocessor() + pre = Preprocessor(output_structure="blocks") out = pre.fit_transform(X, y) assert isinstance(out, dict) shapes = [v.shape for v in out.values()] @@ -144,7 +153,7 @@ def test_dict_output_shapes_add_up(sample_data): def test_dict_keys_reflect_column_names(sample_data): X, y = sample_data - pre = Preprocessor() + pre = Preprocessor(output_structure="blocks") out = pre.fit_transform(X, y) assert isinstance(out, dict) expected_prefixes = ["num_", "cat_"] @@ -180,6 +189,7 @@ def test_dict_keys_reflect_column_names(sample_data): "max_features_per_input", "max_dense_memory", "overflow_policy", + "output_structure", "output_format", "dtype", "verbose", @@ -265,7 +275,7 @@ def test_get_feature_names_out_does_not_duplicate_feature_name(sample_data): def test_lowercase_and_none_method_resolution(sample_data): X, y = sample_data # Mixed-case / None methods are resolved at fit time, not stored on the instance. - pre = Preprocessor(numerical_method="PLE", categorical_method=None) # type: ignore[arg-type] + pre = Preprocessor(numerical_method="PLE", categorical_method=None, output_structure="blocks") # type: ignore[arg-type] out = pre.fit_transform(X, y) assert isinstance(out, dict) assert pre.numerical_method == "PLE" # unchanged on the instance From f53673d178ad8b2092761252eed810b6554b564b Mon Sep 17 00:00:00 2001 From: Manish Kumar Date: Sat, 5 Sep 2026 18:41:58 +0200 Subject: [PATCH 04/19] ci: CI before publish and test minimum dependencies --- .github/workflows/ci.yml | 33 ++++++++++++++++++++++++++ .github/workflows/publish-pypi.yml | 6 +++++ .github/workflows/publish-testpypi.yml | 6 +++++ 3 files changed, 45 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 766dca8..db783df 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,6 +2,7 @@ name: CI on: workflow_dispatch: + workflow_call: push: branches: - main @@ -150,6 +151,38 @@ jobs: - name: Run unit tests run: poetry run pytest tests/ -v + min-deps: + name: Minimum supported dependencies + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Install Poetry + run: pipx install poetry + + - name: Configure Poetry + run: poetry config virtualenvs.in-project true + + - name: Install dependencies + run: poetry install + + - name: Force the declared minimum runtime dependency versions + run: | + poetry run pip install \ + "numpy==1.24.*" \ + "pandas==2.0.*" \ + "scipy==1.10.*" \ + "scikit-learn==1.6.*" + + - name: Run unit tests against the minimum versions + run: poetry run pytest tests/ -q + smoke: name: Smoke tests (Python 3.12, ubuntu) runs-on: ubuntu-latest diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index f9ec093..56fe8ed 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -15,8 +15,14 @@ permissions: id-token: write jobs: + ci: + name: Full CI (required before publish) + uses: ./.github/workflows/ci.yml + secrets: inherit + publish: runs-on: ubuntu-latest + needs: ci environment: pypi-publish # The "v*.*.*" trigger also matches RC tags (e.g. v2.0.0rc2), so guard # against publishing pre-releases to real PyPI. RC tags are handled by diff --git a/.github/workflows/publish-testpypi.yml b/.github/workflows/publish-testpypi.yml index ae9c292..33a766b 100644 --- a/.github/workflows/publish-testpypi.yml +++ b/.github/workflows/publish-testpypi.yml @@ -16,8 +16,14 @@ permissions: id-token: write jobs: + ci: + name: Full CI (required before publish) + uses: ./.github/workflows/ci.yml + secrets: inherit + publish-rc: runs-on: ubuntu-latest + needs: ci environment: testpypi-publish steps: From 4641ab68cb3a03926d521bd23df6849b5c738bac Mon Sep 17 00:00:00 2001 From: Manish Kumar Date: Sat, 5 Sep 2026 18:43:17 +0200 Subject: [PATCH 05/19] fix: raise scikit-learn minimum and use scipy trapezoid --- docs/getting_started/installation.md | 15 ++++++++++++++- poetry.lock | 2 +- pretab/expansion/spline/natural_cubic.py | 3 ++- pyproject.toml | 2 +- 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/docs/getting_started/installation.md b/docs/getting_started/installation.md index f13b17e..1263b38 100644 --- a/docs/getting_started/installation.md +++ b/docs/getting_started/installation.md @@ -1,6 +1,19 @@ # Installation -pretab supports Python 3.10 to 3.13. +pretab supports Python 3.10 to 3.13, with the following minimum core dependency versions: + +| Dependency | Minimum | +| --- | --- | +| `numpy` | 1.24 | +| `pandas` | 2.0 | +| `scipy` | 1.10 | +| `scikit-learn` | 1.6 | + +```{note} +`scikit-learn>=1.6` is required for the `__sklearn_tags__` tag-dispatch API pretab's +transformers use. A dedicated CI job installs exactly these minimum versions and runs the +test suite against them, so this floor is verified, not just declared. +``` ## From PyPI diff --git a/poetry.lock b/poetry.lock index cf14d69..2c9a3b1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -4938,4 +4938,4 @@ lightgbm = ["lightgbm"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.14" -content-hash = "2a0bad6485988b0c36e131940e3f5df70bb2624604cfecb76ddd25b49eb1cab6" +content-hash = "2ecd3046883c17c5faaa826140973e7ca64106ce4081afae5a550197fc2268b8" diff --git a/pretab/expansion/spline/natural_cubic.py b/pretab/expansion/spline/natural_cubic.py index 372d524..9b88929 100644 --- a/pretab/expansion/spline/natural_cubic.py +++ b/pretab/expansion/spline/natural_cubic.py @@ -1,4 +1,5 @@ import numpy as np +from scipy.integrate import trapezoid from sklearn.base import BaseEstimator, TransformerMixin from sklearn.utils.validation import check_is_fitted @@ -244,6 +245,6 @@ def get_penalty_matrix(self, feature_index=0): for i in range(offset, n_basis): for j in range(offset, n_basis): integrand = B_dd[:, i] * B_dd[:, j] - P[i, j] = np.trapezoid(integrand, x_grid) + P[i, j] = trapezoid(integrand, x_grid) return P diff --git a/pyproject.toml b/pyproject.toml index 48c2607..c2498ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry.dependencies] numpy = ">=1.24,<3.0" pandas = ">=2.0,<3.0" -scikit-learn = ">=1.3,<2.0" +scikit-learn = ">=1.6,<2.0" scipy = ">=1.10,<2.0" [tool.poetry.group.dev.dependencies] From 9cc5316ad5301ce7ade84a62889934260b18a96c Mon Sep 17 00:00:00 2001 From: Manish Kumar Date: Sat, 5 Sep 2026 18:43:30 +0200 Subject: [PATCH 06/19] chore: formatting --- pretab/compose/serialize.py | 7 ++++--- pretab/core/policy.py | 1 - pretab/expansion/spline/cubic_regression.py | 1 - 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/pretab/compose/serialize.py b/pretab/compose/serialize.py index f6e78c2..de935de 100644 --- a/pretab/compose/serialize.py +++ b/pretab/compose/serialize.py @@ -47,7 +47,9 @@ # "any dataclass importable under an allowed module" -- the latter would still # let a spec construct an arbitrary sklearn/numpy/scipy dataclass with # attacker-controlled fields. -_ALLOWED_DATACLASSES = frozenset({RepresentationPolicy, RepresentationSpec, FeatureLineage, PlacementResult, TransformerSpec}) +_ALLOWED_DATACLASSES = frozenset( + {RepresentationPolicy, RepresentationSpec, FeatureLineage, PlacementResult, TransformerSpec} +) # --- helpers ------------------------------------------------------------- @@ -180,8 +182,7 @@ def _decode_estimator(payload: dict): cls = cast(Any, _resolve(payload["class"])) if not (isinstance(cls, type) and issubclass(cls, BaseEstimator)): raise PretabSerializationError( - f"Refusing to reconstruct {payload['class']!r} as an estimator: not a " - "scikit-learn BaseEstimator subclass." + f"Refusing to reconstruct {payload['class']!r} as an estimator: not a scikit-learn BaseEstimator subclass." ) obj = cls.__new__(cls) obj.__dict__.update(_decode_mapping(payload["state"])) diff --git a/pretab/core/policy.py b/pretab/core/policy.py index d53097c..4374ab0 100644 --- a/pretab/core/policy.py +++ b/pretab/core/policy.py @@ -28,7 +28,6 @@ expansions" section of ``dev/todo/release-1.0.0/bugfixes-1.0.0.md``. """ - from __future__ import annotations import warnings diff --git a/pretab/expansion/spline/cubic_regression.py b/pretab/expansion/spline/cubic_regression.py index 318541d..a651f9a 100644 --- a/pretab/expansion/spline/cubic_regression.py +++ b/pretab/expansion/spline/cubic_regression.py @@ -265,4 +265,3 @@ def _second_derivative(self, x, basis_index, knots): return 6.0 * x # x**3 knot = knots[col - 3] return 6.0 * np.maximum(x - knot, 0.0) - From b6fc3e73b3b8dcfef7e5617e33fea4ad9578c2d2 Mon Sep 17 00:00:00 2001 From: Manish Kumar Date: Sat, 5 Sep 2026 18:58:32 +0200 Subject: [PATCH 07/19] docs: fix broken source-install command in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4a12f18..ee65b9a 100644 --- a/README.md +++ b/README.md @@ -185,7 +185,7 @@ pip install "pretab[all]" # both of the above ```bash git clone https://github.com/OpenTabular/PreTab cd PreTab -pip install -e ".[dev]" +poetry install ``` ## Usage From 8a948c5f21f4375162d4e0c676ced6011d950aa6 Mon Sep 17 00:00:00 2001 From: Manish Kumar Date: Sat, 5 Sep 2026 18:58:46 +0200 Subject: [PATCH 08/19] fix: remove unimplemented resolution stubs from public placement API --- pretab/placement/__init__.py | 4 ---- pretab/placement/resolution.py | 21 ++++++++++----------- tests/placement/test_placement.py | 12 ++++++++++++ 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/pretab/placement/__init__.py b/pretab/placement/__init__.py index 3f5ae5d..9edd64b 100644 --- a/pretab/placement/__init__.py +++ b/pretab/placement/__init__.py @@ -19,8 +19,6 @@ from .factory import create_placement_strategy from .resolution import ( BaseResolutionPolicy, - CardinalityAwareResolution, - DataSizeAwareResolution, FixedResolution, ) from .supervised import CARTPlacement, LightGBMPlacement @@ -30,8 +28,6 @@ "BasePlacementStrategy", "BaseResolutionPolicy", "CARTPlacement", - "CardinalityAwareResolution", - "DataSizeAwareResolution", "FixedResolution", "LightGBMPlacement", "PLEPlacementAdapter", diff --git a/pretab/placement/resolution.py b/pretab/placement/resolution.py index 02f2352..6d6d1aa 100644 --- a/pretab/placement/resolution.py +++ b/pretab/placement/resolution.py @@ -2,17 +2,18 @@ Every PreTab expansion exposes the same sizing vocabulary: a fixed ``output_dim`` plus an optional adaptive window ``[min_output_dim, max_output_dim]``. Resolving -that vocabulary into an inclusive ``(lo, hi)`` count window -- and validating it -against a family floor / ceiling -- is a single concern that does not depend on +that vocabulary into an inclusive ``(lo, hi)`` count window (and validating it +against a family floor / ceiling) is a single concern that does not depend on *where* the units land. Keeping it here, apart from the placement strategies in :mod:`pretab.placement.unsupervised` / :mod:`pretab.placement.supervised`, lets a family combine any resolution policy with any placement strategy. :class:`FixedResolution` implements the ``output_dim`` / ``[min, max]`` contract shared by every family today. :class:`CardinalityAwareResolution` and -:class:`DataSizeAwareResolution` are declared as forward-looking stubs (their -data-driven ``(lo, hi)`` policies are scheduled for a later phase) so the -registry and factory can name them without importing from a moving target. +:class:`DataSizeAwareResolution` are internal, not-yet-implemented stubs for a +future data-driven ``(lo, hi)`` policy; every method on both currently raises +``NotImplementedError``, and neither is part of the public ``pretab.placement`` +API (they are not re-exported from :mod:`pretab.placement`'s ``__all__``). """ from __future__ import annotations @@ -25,8 +26,6 @@ __all__ = [ "BaseResolutionPolicy", - "CardinalityAwareResolution", - "DataSizeAwareResolution", "FixedResolution", ] @@ -119,8 +118,8 @@ def resolve( class CardinalityAwareResolution(BaseResolutionPolicy): """Stub: cap the unit count by the feature's distinct-value count. - Scheduled for a later phase. Declared now so the capability registry and - placement factory can reference it by name. + Internal, not-yet-implemented placeholder for a future phase; not part of the + public ``pretab.placement`` API. """ def resolve( @@ -143,8 +142,8 @@ def clamp_to_cardinality(self, hi: int, x: np.ndarray) -> int: class DataSizeAwareResolution(BaseResolutionPolicy): """Stub: scale the unit count with the number of samples. - Scheduled for a later phase. Declared now so the capability registry and - placement factory can reference it by name. + Internal, not-yet-implemented placeholder for a future phase; not part of the + public ``pretab.placement`` API. """ def resolve( diff --git a/tests/placement/test_placement.py b/tests/placement/test_placement.py index e1c629d..d0dc463 100644 --- a/tests/placement/test_placement.py +++ b/tests/placement/test_placement.py @@ -180,6 +180,18 @@ def test_fixed_resolution_non_adaptive_conflict(): FixedResolution(adaptive=False).resolve(3, 5, None, floor=1) +def test_not_yet_implemented_resolution_stubs_are_not_public(): + # Regression guard: CardinalityAwareResolution / DataSizeAwareResolution + # always raise NotImplementedError, so they must not be part of the public + # pretab.placement API (no user-facing "usable" class should always fail). + import pretab.placement as placement_pkg + + assert "CardinalityAwareResolution" not in placement_pkg.__all__ + assert "DataSizeAwareResolution" not in placement_pkg.__all__ + assert not hasattr(placement_pkg, "CardinalityAwareResolution") + assert not hasattr(placement_pkg, "DataSizeAwareResolution") + + # --------------------------------------------------------------------------- # # Adapters # --------------------------------------------------------------------------- # From ec5e3ce0ff3bb01ec9324e25c019303e8e678192 Mon Sep 17 00:00:00 2001 From: Manish Kumar Date: Sat, 5 Sep 2026 18:58:58 +0200 Subject: [PATCH 09/19] chore: add py.typed marker for PEP 561 compliance --- pretab/py.typed | 0 tests/integration/test_public_api.py | 9 +++++++++ 2 files changed, 9 insertions(+) create mode 100644 pretab/py.typed diff --git a/pretab/py.typed b/pretab/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration/test_public_api.py b/tests/integration/test_public_api.py index 16cc7b6..095f723 100644 --- a/tests/integration/test_public_api.py +++ b/tests/integration/test_public_api.py @@ -85,3 +85,12 @@ def test_legacy_pipeline_package_is_removed(): def test_compose_subsystem_is_importable(): for module in ("config", "registry", "factory", "output", "inspection", "feature_detection"): importlib.import_module(f"pretab.compose.{module}") + + +def test_py_typed_marker_is_present(): + # PEP 561 marker: without this, type checkers treat an installed pretab as + # untyped by default, losing the benefit of the project's own annotations. + import pathlib + + pretab_dir = pathlib.Path(pretab.__file__).parent + assert (pretab_dir / "py.typed").is_file() From 4fe150dc1148f92223f1f7fedae18609ab911db5 Mon Sep 17 00:00:00 2001 From: Manish Kumar Date: Sat, 5 Sep 2026 19:00:03 +0200 Subject: [PATCH 10/19] docs: remove custombin from categorical_method docstring --- pretab/preprocessor.py | 8 ++++---- tests/integration/test_preprocessor.py | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/pretab/preprocessor.py b/pretab/preprocessor.py index ed265e7..6b8addd 100644 --- a/pretab/preprocessor.py +++ b/pretab/preprocessor.py @@ -115,9 +115,9 @@ class Preprocessor(TransformerMixin, BaseEstimator): Preprocessing strategy applied to every categorical column unless overridden per feature. Choices: ``"int"`` (contiguous integer codes), ``"one-hot"`` (dummy columns), ``"onehot_from_ordinal"`` (one-hot from an already integer-coded column; raises if the - input is not already ordinal-encoded), ``"pretrained"`` (sentence-transformer language - embeddings), and ``"custombin"`` (discretized bin codes). Pass ``None`` (resolved to - ``"none"``) to leave categorical columns unchanged. + input is not already ordinal-encoded), and ``"pretrained"`` (sentence-transformer + language embeddings). Pass ``None`` (resolved to ``"none"``) to leave categorical + columns unchanged. feature_preprocessing : dict, optional Mapping of individual column names to a method, overriding the global ``numerical_method`` / ``categorical_method`` for those columns only, e.g. @@ -306,7 +306,7 @@ class Preprocessor(TransformerMixin, BaseEstimator): ``"mspline"``, ``"ispline"``. Available ``categorical_method`` values: ``"int"``, ``"one-hot"``, ``"onehot_from_ordinal"``, - ``"pretrained"``, ``"custombin"``, ``"none"``. The ``"pretrained"`` method requires the optional + ``"pretrained"``, ``"none"``. The ``"pretrained"`` method requires the optional ``sentence-transformers`` dependency (``pip install "pretab[embeddings]"``). Method names are resolved case-insensitively and ignore ``-`` / ``_`` / space separators, so diff --git a/tests/integration/test_preprocessor.py b/tests/integration/test_preprocessor.py index b7f0e35..22c17cf 100644 --- a/tests/integration/test_preprocessor.py +++ b/tests/integration/test_preprocessor.py @@ -343,3 +343,21 @@ def test_output_dims_and_total_before_fit_raise(): _ = Preprocessor().output_dims_ with pytest.raises(NotFittedError): _ = Preprocessor().total_output_dim_ + + +# --- Documented categorical_method values are actually accepted (docstring/registry drift) --- + +_DOCUMENTED_CATEGORICAL_METHODS = ["int", "one-hot", "onehot_from_ordinal", "pretrained", "none"] + + +@pytest.mark.parametrize("method", _DOCUMENTED_CATEGORICAL_METHODS) +def test_documented_categorical_method_is_accepted(sample_data, method): + # Regression guard: the docstring's categorical_method list once claimed + # "custombin" was valid, but the registry rejected it (numerical-only). + if method == "pretrained": + pytest.importorskip("sentence_transformers") + X, y = sample_data + if method == "onehot_from_ordinal": + # Requires already-integer-coded categorical input. + X = X.assign(cat1=pd.factorize(X["cat1"])[0], cat2=pd.factorize(X["cat2"])[0]) + Preprocessor(numerical_method="none", categorical_method=method).fit(X, y) From 824102e228327319f2e429f2649a2e6717e7b906 Mon Sep 17 00:00:00 2001 From: Manish Kumar Date: Sat, 5 Sep 2026 19:00:18 +0200 Subject: [PATCH 11/19] fix: validate feature_preprocessing keys against input columns --- pretab/preprocessor.py | 11 +++++++++++ tests/integration/test_preprocessor.py | 8 +++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/pretab/preprocessor.py b/pretab/preprocessor.py index 6b8addd..9509952 100644 --- a/pretab/preprocessor.py +++ b/pretab/preprocessor.py @@ -485,6 +485,17 @@ def fit(self, X, y=None, embeddings=None): X = to_dataframe(X) + if self.feature_preprocessing: + unknown_features = set(self.feature_preprocessing) - set(X.columns) + if unknown_features: + raise invalid_param_error( + type(self).__name__, + "feature_preprocessing", + sorted(unknown_features), + "every key must name a column present in X", + valid=X.columns, + ) + if self.missing_policy == "error": self._reject_missing(X) diff --git a/tests/integration/test_preprocessor.py b/tests/integration/test_preprocessor.py index 22c17cf..dad49d4 100644 --- a/tests/integration/test_preprocessor.py +++ b/tests/integration/test_preprocessor.py @@ -5,7 +5,7 @@ from sklearn.exceptions import NotFittedError from sklearn.utils.validation import check_is_fitted -from pretab.exceptions import IncompatibleParamsError, PretabDataError +from pretab.exceptions import IncompatibleParamsError, InvalidParamError, PretabDataError from pretab.preprocessor import Preprocessor # Adjust the import as needed @@ -361,3 +361,9 @@ def test_documented_categorical_method_is_accepted(sample_data, method): # Requires already-integer-coded categorical input. X = X.assign(cat1=pd.factorize(X["cat1"])[0], cat2=pd.factorize(X["cat2"])[0]) Preprocessor(numerical_method="none", categorical_method=method).fit(X, y) + + +def test_unknown_feature_preprocessing_key_raises(sample_data): + X, y = sample_data + with pytest.raises(InvalidParamError, match="feature_preprocessing"): + Preprocessor(feature_preprocessing={"nnum1": "minmax"}).fit(X, y) From 0c4ccc73a79037f28283743709b9e19c1ce6761d Mon Sep 17 00:00:00 2001 From: Manish Kumar Date: Sat, 5 Sep 2026 19:00:37 +0200 Subject: [PATCH 12/19] fix: numerical_method=none no longer applies scaling --- pretab/compose/factory.py | 2 +- tests/integration/test_preprocessor.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/pretab/compose/factory.py b/pretab/compose/factory.py index f272a6d..8ced4d6 100644 --- a/pretab/compose/factory.py +++ b/pretab/compose/factory.py @@ -136,7 +136,7 @@ def get_numerical_transformer_steps( "must name a scaler or disable scaling", valid={*scalers, "none"}, ) - if scaling in scalers and scaling != method: + if scaling in scalers and scaling != method and method != "none": steps.append(scalers[scaling]) if method not in NUMERICAL_METHODS: diff --git a/tests/integration/test_preprocessor.py b/tests/integration/test_preprocessor.py index dad49d4..df9d224 100644 --- a/tests/integration/test_preprocessor.py +++ b/tests/integration/test_preprocessor.py @@ -363,6 +363,16 @@ def test_documented_categorical_method_is_accepted(sample_data, method): Preprocessor(numerical_method="none", categorical_method=method).fit(X, y) +def test_numerical_method_none_leaves_values_untouched(): + # Regression test: numerical_method="none" used to still run MinMaxScaler + # before the no-op, contradicting the documented "leave unchanged" meaning. + X = pd.DataFrame({"a": [1.0, 2.0, 3.0, 100.0]}) + pre = Preprocessor(numerical_method="none").fit(X) + out = pre.transform(X) + assert isinstance(out, np.ndarray) + np.testing.assert_array_equal(out.ravel(), X["a"].to_numpy()) + + def test_unknown_feature_preprocessing_key_raises(sample_data): X, y = sample_data with pytest.raises(InvalidParamError, match="feature_preprocessing"): From 1139b92a1ea7689838624b6f2eebb3538798ead0 Mon Sep 17 00:00:00 2001 From: Manish Kumar Date: Sat, 5 Sep 2026 20:28:20 +0200 Subject: [PATCH 13/19] fix: add missing polars optional dependency and its tests --- .github/workflows/ci.yml | 2 + docs/core_concepts/outputs_and_inspection.md | 6 +- docs/getting_started/installation.md | 14 ++++ poetry.lock | 70 +++++++++++++++++++- pyproject.toml | 3 +- tests/integration/test_output_format.py | 35 +++++++--- 6 files changed, 116 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index db783df..826165a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -281,6 +281,8 @@ jobs: module: sentence_transformers - extra: lightgbm module: lightgbm + - extra: polars + module: polars steps: - uses: actions/checkout@v4 diff --git a/docs/core_concepts/outputs_and_inspection.md b/docs/core_concepts/outputs_and_inspection.md index 620c8aa..32e95e2 100644 --- a/docs/core_concepts/outputs_and_inspection.md +++ b/docs/core_concepts/outputs_and_inspection.md @@ -121,8 +121,10 @@ pre.set_output(transform="pandas") # or "polars" ``` ```{note} -Polars output is loaded lazily. If polars is not installed, requesting it raises a clear -`OptionalDependencyError` rather than failing deep in the call stack. +Polars output requires the optional `polars` extra (`pip install "pretab[polars]"`) and is +loaded lazily: if polars is not installed, requesting it raises a clear +`OptionalDependencyError` rather than failing deep in the call stack. See +[Installation](../getting_started/installation.md#optional-extras). ``` ## Choosing your output settings diff --git a/docs/getting_started/installation.md b/docs/getting_started/installation.md index 1263b38..5d43e3f 100644 --- a/docs/getting_started/installation.md +++ b/docs/getting_started/installation.md @@ -44,6 +44,20 @@ supervised knot, center, and threshold selection: pip install "pretab[lightgbm]" ``` +The `polars` extra enables `set_output(transform="polars")`, so `Preprocessor.transform` +returns a `polars.DataFrame` instead of a NumPy array or dict: + +```bash +pip install "pretab[polars]" +``` + +```{note} +`polars` is only needed for the `set_output(transform="polars")` output path; every other +output (`output_structure="matrix"`/`"blocks"`, `output_format="dense"`/`"sparse"`, +`set_output(transform="pandas")`) works without it. Requesting `"polars"` output without the +extra installed raises a clear `OptionalDependencyError`. +``` + Use the convenience `all` extra to install every optional dependency at once: ```bash diff --git a/poetry.lock b/poetry.lock index 2c9a3b1..aa4f1d2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2874,6 +2874,71 @@ files = [ dev = ["pre-commit", "tox"] testing = ["coverage", "pytest", "pytest-benchmark"] +[[package]] +name = "polars" +version = "1.44.1" +description = "Blazingly fast DataFrame library" +optional = true +python-versions = ">=3.10" +groups = ["main"] +markers = "extra == \"polars\" or extra == \"all\"" +files = [ + {file = "polars-1.44.1-py3-none-any.whl", hash = "sha256:1fa62fc1c88fba77a68b28291b5aabdd69e5f38b34e59721a064ae3169b59bb5"}, + {file = "polars-1.44.1.tar.gz", hash = "sha256:ef3c89e9ebbbe8eb343c06873f1945683f8b6f97a1bdf001c60551c6c5e3cda1"}, +] + +[package.dependencies] +polars-runtime-32 = "1.44.1" + +[package.extras] +adbc = ["adbc-driver-manager[dbapi]", "adbc-driver-sqlite[dbapi]"] +all = ["polars[async,cloudpickle,database,deltalake,excel,fsspec,graph,iceberg,numpy,pandas,plot,pyarrow,pydantic,style,timezone]"] +async = ["gevent"] +calamine = ["fastexcel (>=0.9)"] +cloudpickle = ["cloudpickle"] +connectorx = ["connectorx (>=0.3.2)"] +database = ["polars[adbc,connectorx,sqlalchemy]"] +deltalake = ["deltalake (>=1.0.0,!=1.5.*)"] +excel = ["polars[calamine,openpyxl,xlsx2csv,xlsxwriter]"] +fsspec = ["fsspec"] +gpu = ["cudf-polars-cu12"] +graph = ["matplotlib"] +iceberg = ["pyiceberg (>=0.9.0)"] +numpy = ["numpy (>=1.16.0)"] +openpyxl = ["openpyxl (>=3.0.0)"] +pandas = ["pandas", "polars[pyarrow]"] +plot = ["altair (>=5.4.0)"] +polars-cloud = ["polars_cloud (>=0.9.0)"] +pyarrow = ["pyarrow (>=7.0.0)"] +pydantic = ["pydantic"] +rt64 = ["polars-runtime-64 (==1.44.1)"] +rtcompat = ["polars-runtime-compat (==1.44.1)"] +sqlalchemy = ["polars[pandas]", "sqlalchemy"] +style = ["great-tables (>=0.8.0)"] +timezone = ["tzdata ; platform_system == \"Windows\""] +xlsx2csv = ["xlsx2csv (>=0.8.0)"] +xlsxwriter = ["xlsxwriter"] + +[[package]] +name = "polars-runtime-32" +version = "1.44.1" +description = "Blazingly fast DataFrame library" +optional = true +python-versions = ">=3.10" +groups = ["main"] +markers = "extra == \"polars\" or extra == \"all\"" +files = [ + {file = "polars_runtime_32-1.44.1-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:1dfccb2b52aa50468a7d28e3e61c8338a13fb5bffc8646e388a649f5bdc6b463"}, + {file = "polars_runtime_32-1.44.1-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:0580807dc3eed258f0db70bb65d905dd43f0135392119ec25308033ae24258fb"}, + {file = "polars_runtime_32-1.44.1-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0627f9aa82cb869725235e5188f698862fd9ada0c8c1cf65c3dc5a49a4a0ec26"}, + {file = "polars_runtime_32-1.44.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eea4283be8e60822d890dbda20588fe59b4172b508bd5ebf3471e531ca9f50d7"}, + {file = "polars_runtime_32-1.44.1-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:04e2c0f46e7a9906fffb1897f18f23b079b74f83c56b50060bace9e7b9b49b1a"}, + {file = "polars_runtime_32-1.44.1-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:0956f0cae632d8fad3a04b4315bf2bb69b56d10c83c79a75c2c4c5a13b9ce5cc"}, + {file = "polars_runtime_32-1.44.1-cp310-abi3-win_amd64.whl", hash = "sha256:159334184e6fbb074c9f4692221ea19970a5e2bed2a479f9d7bdb00b7f3eedb9"}, + {file = "polars_runtime_32-1.44.1-cp310-abi3-win_arm64.whl", hash = "sha256:3ba28d638d0513e0b4afbcdab5c0059a85021e5f81d62b5f793e7e23badb2cf7"}, + {file = "polars_runtime_32-1.44.1.tar.gz", hash = "sha256:abd10a54ed1caff42228610fcba0f93251f9870bd7cffb0c78bc26f5e0718ce4"}, +] + [[package]] name = "pre-commit" version = "3.8.0" @@ -4931,11 +4996,12 @@ test = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more_it type = ["pytest-mypy (>=1.0.1) ; platform_python_implementation != \"PyPy\""] [extras] -all = ["lightgbm", "sentence-transformers"] +all = ["lightgbm", "polars", "sentence-transformers"] embeddings = ["sentence-transformers"] lightgbm = ["lightgbm"] +polars = ["polars"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.14" -content-hash = "2ecd3046883c17c5faaa826140973e7ca64106ce4081afae5a550197fc2268b8" +content-hash = "72350f56d6631ce5895d0ccc60dc374cd09526ccb2068569acc2c4204489589f" diff --git a/pyproject.toml b/pyproject.toml index c2498ac..3dcec3e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,8 @@ dynamic = ["dependencies"] [project.optional-dependencies] embeddings = ["sentence-transformers>=2.0"] lightgbm = ["lightgbm>=4.0"] -all = ["sentence-transformers>=2.0", "lightgbm>=4.0"] +polars = ["polars>=0.20"] +all = ["sentence-transformers>=2.0", "lightgbm>=4.0", "polars>=0.20"] [project.urls] homepage = "https://github.com/OpenTabular/PreTab" diff --git a/tests/integration/test_output_format.py b/tests/integration/test_output_format.py index c169571..5e54deb 100644 --- a/tests/integration/test_output_format.py +++ b/tests/integration/test_output_format.py @@ -199,15 +199,32 @@ def test_set_output_default_with_blocks_structure_is_dict(frame, y): def test_set_output_polars_without_polars_raises(frame, y): - import importlib.util - - p = _bspline().fit(frame, y).set_output(transform="polars") - if importlib.util.find_spec("polars") is None: - with pytest.raises(OptionalDependencyError): - p.transform(frame) - else: - out = p.transform(frame) - assert out.shape == (len(frame), p.total_output_dim_) + # Unit-test to_dataframe_output directly (rather than through the full + # Preprocessor.set_output stack): sklearn's own polars detection elsewhere + # in that stack also probes sys.modules and breaks under a blanket + # sys.modules["polars"] = None patch, unrelated to the behavior under test. + import unittest.mock + + from pretab.compose.output import to_dataframe_output + + with unittest.mock.patch.dict("sys.modules", {"polars": None}): + with pytest.raises(OptionalDependencyError, match="polars"): + to_dataframe_output(np.zeros((2, 2)), ["a", "b"], "polars") + + +def test_set_output_polars_dataframe_is_correct(frame, y): + pl = pytest.importorskip("polars") + + arr = _bspline().fit(frame, y).transform(frame, return_array=True) + assert isinstance(arr, np.ndarray) + + p = _bspline().fit(frame, y) + out = p.set_output(transform="polars").transform(frame) + + assert isinstance(out, pl.DataFrame) + assert out.shape == (len(frame), p.total_output_dim_) + assert out.columns == list(p.get_feature_names_out()) + np.testing.assert_allclose(out.to_numpy(), arr) # --- validation ---------------------------------------------------------------- From 3f42af916b8f7882f11873eeaf421dd8fcb4761d Mon Sep 17 00:00:00 2001 From: Manish Kumar Date: Sun, 6 Sep 2026 06:23:15 +0200 Subject: [PATCH 14/19] chore: test case correction --- tests/integration/test_output_format.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/tests/integration/test_output_format.py b/tests/integration/test_output_format.py index 5e54deb..abe2613 100644 --- a/tests/integration/test_output_format.py +++ b/tests/integration/test_output_format.py @@ -170,7 +170,8 @@ def test_output_report_shape_and_keys(frame, y): def test_set_output_pandas_returns_dataframe(frame, y): - p = _bspline().fit(frame, y).set_output(transform="pandas") + p = _bspline().fit(frame, y) + p.set_output(transform="pandas") out = p.transform(frame) assert isinstance(out, pd.DataFrame) assert list(out.columns) == list(p.get_feature_names_out()) @@ -178,7 +179,8 @@ def test_set_output_pandas_returns_dataframe(frame, y): def test_set_output_pandas_fit_transform(frame, y): - p = _bspline().set_output(transform="pandas") + p = _bspline() + p.set_output(transform="pandas") out = p.fit_transform(frame, y) assert isinstance(out, pd.DataFrame) assert out.shape[1] == p.total_output_dim_ @@ -187,13 +189,15 @@ def test_set_output_pandas_fit_transform(frame, y): def test_set_output_default_still_array(frame, y): # set_output(transform="default") only opts out of pandas/polars wrapping; it # does not override output_structure, which still resolves to an array. - p = _bspline().fit(frame, y).set_output(transform="default") + p = _bspline().fit(frame, y) + p.set_output(transform="default") out = p.transform(frame) assert isinstance(out, np.ndarray) def test_set_output_default_with_blocks_structure_is_dict(frame, y): - p = _bspline(output_structure="blocks").fit(frame, y).set_output(transform="default") + p = _bspline(output_structure="blocks").fit(frame, y) + p.set_output(transform="default") out = p.transform(frame) assert isinstance(out, dict) @@ -213,18 +217,20 @@ def test_set_output_polars_without_polars_raises(frame, y): def test_set_output_polars_dataframe_is_correct(frame, y): - pl = pytest.importorskip("polars") + pytest.importorskip("polars") + import polars as pl # type: ignore arr = _bspline().fit(frame, y).transform(frame, return_array=True) assert isinstance(arr, np.ndarray) p = _bspline().fit(frame, y) - out = p.set_output(transform="polars").transform(frame) + p.set_output(transform="polars") + out = p.transform(frame) assert isinstance(out, pl.DataFrame) - assert out.shape == (len(frame), p.total_output_dim_) - assert out.columns == list(p.get_feature_names_out()) - np.testing.assert_allclose(out.to_numpy(), arr) + assert out.shape == (len(frame), p.total_output_dim_) # type: ignore + assert out.columns == list(p.get_feature_names_out()) # type: ignore + np.testing.assert_allclose(out.to_numpy(), arr) # type: ignore # --- validation ---------------------------------------------------------------- From 5e81a3b36b9a8d10f37cb6424ea7fe34de46bf05 Mon Sep 17 00:00:00 2001 From: Manish Kumar Date: Sun, 6 Sep 2026 06:42:37 +0200 Subject: [PATCH 15/19] docs: note Nystroem's non-uniform pointwise approximation error --- docs/representations/kernel_approximation.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/representations/kernel_approximation.md b/docs/representations/kernel_approximation.md index af660a0..2dbfcc0 100644 --- a/docs/representations/kernel_approximation.md +++ b/docs/representations/kernel_approximation.md @@ -85,6 +85,16 @@ random basis independent of the data, while Nyström samples landmarks from the When in doubt, try both and compare with cross-validation. ``` +```{note} +Nyström's approximation error is not uniformly bounded across the kernel matrix. In +particular, the self-similarity entries $K(x, x)$ on the diagonal can be approximated far +less accurately than typical off-diagonal entries, depending on how well the sampled +landmarks happen to cover that row. This is an inherent property of the Nyström method +itself (also present in plain `sklearn.kernel_approximation.Nystroem`), not something +specific to PreTab's wrapper. If your downstream model is sensitive to diagonal accuracy, +increase `n_components` or try random Fourier features instead. +``` + ```{warning} Random Fourier features and Nyström are multivariate and operate on the whole input matrix. They are not available as a per-column `numerical_method`; fit them standalone or combine them From dc0386e16454844f4c6746cbbedfd6f62ccf73cb Mon Sep 17 00:00:00 2001 From: Manish Kumar Date: Sun, 6 Sep 2026 06:42:46 +0200 Subject: [PATCH 16/19] fix: document periodic transform wrap-around and correct binning docstring --- pretab/encoding/numerical/binning.py | 2 +- pretab/encoding/numerical/periodic.py | 7 +++++++ tests/encoding/numerical/test_periodic.py | 10 ++++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/pretab/encoding/numerical/binning.py b/pretab/encoding/numerical/binning.py index 8198608..4b1fcbf 100644 --- a/pretab/encoding/numerical/binning.py +++ b/pretab/encoding/numerical/binning.py @@ -182,7 +182,7 @@ def fit(self, X, y=None): @staticmethod def _bin_indices(column, edges): - """Assign each value to a bin using ``(a, b]`` intervals with a closed left edge.""" + """Assign each value to a bin using ``(a, b]`` intervals with a closed right edge.""" idx = np.searchsorted(edges, column, side="left") - 1 return np.clip(idx, 0, edges.size - 2).astype(int) diff --git a/pretab/encoding/numerical/periodic.py b/pretab/encoding/numerical/periodic.py index b54102e..75aaf31 100644 --- a/pretab/encoding/numerical/periodic.py +++ b/pretab/encoding/numerical/periodic.py @@ -45,6 +45,13 @@ class PeriodicEncodingTransformer(BasePreTabTransformer): (which applies one method uniformly across columns). Apply it directly to the relevant cyclical column instead. + :meth:`fit` rejects a value outside ``[0, period]``, but :meth:`transform` does not + repeat that check: the trigonometric encoding wraps any input around the cycle + automatically (``period + x`` maps to the same output as ``x``), which is the + mathematically correct result for a genuinely cyclic quantity. If you need + transform-time inputs strictly confined to ``[0, period]`` as well, validate them + before calling :meth:`transform`. + Examples -------- >>> import numpy as np diff --git a/tests/encoding/numerical/test_periodic.py b/tests/encoding/numerical/test_periodic.py index 70a155e..e0a7270 100644 --- a/tests/encoding/numerical/test_periodic.py +++ b/tests/encoding/numerical/test_periodic.py @@ -71,3 +71,13 @@ def test_cyclic_rejects_non_positive_harmonics(): X = np.array([[0], [6], [12], [18]]) with pytest.raises(InvalidParamError): PeriodicEncodingTransformer(period=24, harmonics=0).fit(X) + + +def test_cyclic_transform_wraps_out_of_range_input(): + # fit rejects out-of-range values, but transform intentionally does not + # re-validate: it wraps them around the cycle, which is the mathematically + # correct result for a cyclic quantity. + transformer = PeriodicEncodingTransformer(period=24).fit(np.array([[0], [12]])) + out_of_range = transformer.transform(np.array([[30]])) + wrapped = transformer.transform(np.array([[6]])) + np.testing.assert_allclose(out_of_range, wrapped, atol=1e-12) From 2fe4e0de453ea395cec982578ade29c77d739d14 Mon Sep 17 00:00:00 2001 From: Manish Kumar Date: Sun, 6 Sep 2026 06:42:53 +0200 Subject: [PATCH 17/19] test: add shared penalty matrix symmetry, PSD, rank, and rescaling checks --- .../spline/test_penalty_matrix_invariants.py | 162 ++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 tests/expansion/spline/test_penalty_matrix_invariants.py diff --git a/tests/expansion/spline/test_penalty_matrix_invariants.py b/tests/expansion/spline/test_penalty_matrix_invariants.py new file mode 100644 index 0000000..38c9236 --- /dev/null +++ b/tests/expansion/spline/test_penalty_matrix_invariants.py @@ -0,0 +1,162 @@ +"""Shared invariants for every spline family's penalty matrix. + +Closes a gap flagged during the v1.0.0 hardening review: symmetry/PSD, rank/nullity, +and affine-rescaling behavior were previously checked ad hoc per family (if at all) +rather than through one shared, reusable check that every family can be run against. +""" + +import numpy as np +import pytest + +from pretab.exceptions import ConfigWarning +from pretab.transformers import ( + BSplineTransformer, + CubicRegressionSplineTransformer, + ISplineTransformer, + MSplineTransformer, + NaturalCubicSplineTransformer, + PSplineTransformer, + TensorProductSplineTransformer, + ThinPlateSplineTransformer, +) + +# Families whose penalty is the pure D^T D finite-difference operator: it depends only +# on the basis width and `diff_order`, never on the fitted data's values. +DIFFERENCE_PENALTY_FAMILIES = [BSplineTransformer, MSplineTransformer, ISplineTransformer] + +ALL_SPLINE_FAMILIES = [ + (BSplineTransformer, {"output_dim": 8}, 1), + (MSplineTransformer, {"output_dim": 8}, 1), + (ISplineTransformer, {"output_dim": 8}, 1), + (PSplineTransformer, {"output_dim": 8}, 1), + (NaturalCubicSplineTransformer, {"output_dim": 6}, 1), + (CubicRegressionSplineTransformer, {"output_dim": 8}, 1), + (TensorProductSplineTransformer, {"output_dim": 4}, 2), +] + + +def assert_valid_penalty(P, *, expect_psd=True, atol=1e-8): + """Assert a penalty matrix is square, symmetric, and (optionally) positive semi-definite.""" + assert P.ndim == 2 + assert P.shape[0] == P.shape[1] + np.testing.assert_allclose(P, P.T, atol=atol) + if expect_psd: + assert np.linalg.eigvalsh(P).min() >= -atol + + +@pytest.fixture +def X_uniform(): + rng = np.random.default_rng(0) + return rng.uniform(0, 1, size=(200, 1)) + + +@pytest.fixture +def X_multi(): + rng = np.random.default_rng(0) + return rng.uniform(0, 1, size=(200, 2)) + + +@pytest.mark.parametrize("cls", DIFFERENCE_PENALTY_FAMILIES) +def test_difference_penalty_is_symmetric_psd(cls, X_uniform): + assert_valid_penalty(cls(output_dim=8).fit(X_uniform).get_penalty_matrix()) + + +def test_pspline_penalty_is_symmetric_psd(X_uniform): + assert_valid_penalty(PSplineTransformer(output_dim=8).fit(X_uniform).get_penalty_matrix()) + + +def test_natural_cubic_penalty_is_symmetric_psd(X_uniform): + assert_valid_penalty(NaturalCubicSplineTransformer(output_dim=6).fit(X_uniform).get_penalty_matrix()) + + +def test_cubic_regression_penalty_is_symmetric_psd(X_uniform): + assert_valid_penalty(CubicRegressionSplineTransformer(output_dim=8).fit(X_uniform).get_penalty_matrix()) + + +def test_tensor_product_penalty_is_symmetric_psd(X_multi): + transformer = TensorProductSplineTransformer(output_dim=4).fit(X_multi) + for P in transformer.get_penalty_matrices(): + assert_valid_penalty(P) + + +def test_thinplate_penalty_is_symmetric_but_not_guaranteed_psd(): + X = np.linspace(0, 1, 40).reshape(-1, 1) + transformer = ThinPlateSplineTransformer(n_components=6, random_state=0).fit(X) + with pytest.warns(ConfigWarning, match="experimental"): + P = transformer.get_penalty_matrix() + # PSD is explicitly not guaranteed here (see the class docstring); only symmetry is. + assert_valid_penalty(P, expect_psd=False) + + +@pytest.mark.parametrize("cls", DIFFERENCE_PENALTY_FAMILIES) +@pytest.mark.parametrize("diff_order", [1, 2, 3]) +def test_difference_penalty_rank_matches_null_space_formula(cls, diff_order, X_uniform): + # D^T D built from a diff_order-th difference operator has rank n_basis - diff_order; + # its null space is exactly the discrete polynomials of degree < diff_order. + transformer = cls(output_dim=8, include_bias=False).fit(X_uniform) + n_basis = transformer.n_basis_[0] + P = transformer.get_penalty_matrix(diff_order=diff_order) + assert np.linalg.matrix_rank(P) == n_basis - diff_order + + +@pytest.mark.parametrize("diff_order", [1, 2, 3]) +def test_pspline_penalty_rank_matches_configured_diff_order(diff_order, X_uniform): + transformer = PSplineTransformer(output_dim=8, diff_order=diff_order, include_bias=False).fit(X_uniform) + n_basis = transformer.n_basis_[0] + P = transformer.get_penalty_matrix() + assert np.linalg.matrix_rank(P) == n_basis - diff_order + + +@pytest.mark.parametrize("diff_order", [1, 2]) +def test_tensor_product_marginal_penalty_rank_matches_diff_order(diff_order, X_multi): + transformer = TensorProductSplineTransformer(output_dim=4, diff_order=diff_order, include_bias=False).fit(X_multi) + for i, n_basis in enumerate(transformer.marginal_sizes_): + P = transformer.get_penalty_matrix(feature_index=i) + assert np.linalg.matrix_rank(P) == n_basis - diff_order + + +@pytest.mark.parametrize("cls", DIFFERENCE_PENALTY_FAMILIES) +def test_difference_penalty_is_invariant_to_affine_rescaling(cls, X_uniform): + P_original = cls(output_dim=8).fit(X_uniform).get_penalty_matrix() + P_rescaled = cls(output_dim=8).fit(3.0 * X_uniform + 5.0).get_penalty_matrix() + np.testing.assert_array_equal(P_original, P_rescaled) + + +def test_pspline_penalty_is_invariant_to_affine_rescaling(X_uniform): + P_original = PSplineTransformer(output_dim=8).fit(X_uniform).get_penalty_matrix() + P_rescaled = PSplineTransformer(output_dim=8).fit(3.0 * X_uniform + 5.0).get_penalty_matrix() + np.testing.assert_array_equal(P_original, P_rescaled) + + +def test_tensor_product_penalty_is_invariant_to_affine_rescaling(X_multi): + P_original = TensorProductSplineTransformer(output_dim=4).fit(X_multi).get_penalty_matrix(feature_index=0) + P_rescaled = ( + TensorProductSplineTransformer(output_dim=4).fit(3.0 * X_multi + 5.0).get_penalty_matrix(feature_index=0) + ) + np.testing.assert_array_equal(P_original, P_rescaled) + + +def test_natural_cubic_penalty_scales_as_cube_of_domain_scale(X_uniform): + # Unlike the difference penalties above, an integrated-squared-second-derivative + # penalty is not scale invariant: rescaling the fitted domain by `a` rescales every + # penalty entry by exactly a**3 (verified numerically against a from-scratch fit). + a = 3.0 + P_original = NaturalCubicSplineTransformer(output_dim=6).fit(X_uniform).get_penalty_matrix() + P_rescaled = NaturalCubicSplineTransformer(output_dim=6).fit(a * X_uniform).get_penalty_matrix() + np.testing.assert_allclose(P_rescaled, P_original * a**3, rtol=1e-6) + + +@pytest.mark.parametrize(("cls", "kwargs", "n_features"), ALL_SPLINE_FAMILIES) +def test_feature_names_out_length_matches_transform_width(cls, kwargs, n_features): + rng = np.random.default_rng(0) + X = rng.uniform(0, 1, size=(200, n_features)) + transformer = cls(**kwargs).fit(X) + Xt = transformer.transform(X) + assert len(transformer.get_feature_names_out()) == Xt.shape[1] + + +def test_thinplate_feature_names_out_length_matches_transform_width(): + X = np.linspace(0, 1, 40).reshape(-1, 1) + transformer = ThinPlateSplineTransformer(n_components=6, random_state=0).fit(X) + Xt = transformer.transform(X) + assert len(transformer.get_feature_names_out()) == Xt.shape[1] From 948bb145b9c56071dc9d50afe3bc954db99c41be Mon Sep 17 00:00:00 2001 From: Manish Kumar Date: Sun, 6 Sep 2026 07:39:33 +0200 Subject: [PATCH 18/19] docs: remove stale claim from reproducibility note --- docs/core_concepts/reproducibility.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/core_concepts/reproducibility.md b/docs/core_concepts/reproducibility.md index 712c879..3d7f510 100644 --- a/docs/core_concepts/reproducibility.md +++ b/docs/core_concepts/reproducibility.md @@ -38,8 +38,9 @@ restored = Preprocessor.from_spec("representation.json") ```{important} `from_spec` is a safe alternative to pickle. Reconstruction imports only from `pretab`, -`scikit-learn`, `numpy`, `scipy`, and builtins, and it never executes arbitrary estimator -code. A spec from an untrusted source cannot run code the way an untrusted pickle can. +`scikit-learn`, `numpy`, and `scipy`, reconstructs dataclasses only from an exact, +closed allow-list, and never executes arbitrary estimator code. A spec from an untrusted +source cannot run code the way an untrusted pickle can. ``` A round-trip reproduces `transform` bit-for-bit, so a spec is a faithful, human-readable From 0508e56fff56cee5a7cdba04b1fe85c6d049d7d1 Mon Sep 17 00:00:00 2001 From: Manish Kumar Date: Sun, 6 Sep 2026 07:40:00 +0200 Subject: [PATCH 19/19] fix: clip P-spline and tensor-product out-of-range transforms via policy --- docs/representations/choosing_a_method.md | 24 ++-- pretab/core/base.py | 14 +- pretab/core/policy.py | 26 ++-- pretab/expansion/spline/b_spline.py | 3 + pretab/expansion/spline/base.py | 15 ++- pretab/expansion/spline/cubic_regression.py | 12 +- pretab/expansion/spline/i_spline.py | 3 + pretab/expansion/spline/m_spline.py | 3 + .../spline/multivariate/tensor_product.py | 15 ++- pretab/expansion/spline/natural_cubic.py | 16 ++- pretab/expansion/spline/p_spline.py | 17 ++- tests/core/test_policy.py | 127 ++++++++++++++++++ .../spline/test_pspline_transformer.py | 12 ++ .../spline/test_tensorproduct_transformer.py | 13 ++ 14 files changed, 270 insertions(+), 30 deletions(-) create mode 100644 tests/core/test_policy.py diff --git a/docs/representations/choosing_a_method.md b/docs/representations/choosing_a_method.md index 42570c8..4a168a8 100644 --- a/docs/representations/choosing_a_method.md +++ b/docs/representations/choosing_a_method.md @@ -74,12 +74,15 @@ Very small samples and rely on a scaled input. Extrapolation beyond the fitted range -: Bases are fitted on the training range, and PreTab's default policy is to extrapolate: a - value beyond the fitted range is passed straight into the basis rather than clamped, so the - spline or feature map keeps evaluating past its knots or centers. If your test data lies well - beyond training, that extrapolated value carries no real signal. Set - `policy=RepresentationPolicy(out_of_range="clip")` (or `"warn"` / `"error"`) if you would - rather cap or catch out-of-range inputs. See the edge-case behaviour below. +: Bases are fitted on the training range, and each spline family has its own default + transform-time behavior for values beyond it: B/M/I-spline, P-spline, and tensor-product + clip to the fitted range, while natural-cubic and cubic-regression extrapolate smoothly + (their basis is defined for any input). If your test data lies well beyond training, an + extrapolated value carries no real signal. Pass + `policy=RepresentationPolicy(out_of_range="clip")` (or `"warn"` / `"error"`) directly to any + of these spline transformers to override their default for that instance. This is + standalone-transformer-only: it is not yet threaded through `Preprocessor`. See the + edge-case behaviour below. Pure noise features : Expanding a feature that carries no signal only gives the model more ways to fit noise. Drop @@ -98,9 +101,12 @@ PreTab is explicit about degenerate inputs rather than failing silently. tensor-product) raise a typed error rather than fitting a meaningless basis. Numeric binning falls back to a single bin instead of raising, and PLE and the feature maps place all their bins or centers at the same value, a degenerate but still valid basis. -- **Out-of-range input at transform**: by default, values beyond the fitted range extrapolate - (the basis keeps evaluating past its knots or centers); set the `out_of_range` policy to - `"clip"`, `"warn"`, or `"error"` for a different, explicit behaviour. +- **Out-of-range input at transform**: B/M/I-spline, P-spline, and tensor-product clip + values beyond the fitted range by default; natural-cubic and cubic-regression extrapolate + smoothly by default. Pass `policy=RepresentationPolicy(out_of_range=...)` directly to any + of these transformers to override its default with `"clip"`, `"warn"`, or `"error"` + instead. This is standalone-transformer-only for now, not yet available through + `Preprocessor`. - **Unseen category**: `ContinuousOrdinalTransformer` (`"int"`) maps an unseen category to a reserved code rather than raising; one-hot encoding (`"one-hot"`) instead emits an all-zero row for it. diff --git a/pretab/core/base.py b/pretab/core/base.py index 6c7a10b..284a0da 100644 --- a/pretab/core/base.py +++ b/pretab/core/base.py @@ -35,6 +35,8 @@ class BasePreTabTransformer( (``_policy``). A family narrows individual axes through the ``_constant_policy`` / ``_out_of_range_policy`` / ``_duplicate_policy`` class attributes (``None`` means "inherit the shared policy"); :meth:`_resolved_policy` merges them. + Transformers that expose their own ``policy`` constructor parameter let a + caller override those defaults per-instance; see :meth:`_resolved_policy`. """ _allow_nan: bool = True @@ -54,7 +56,17 @@ class BasePreTabTransformer( n_features_in_: int def _resolved_policy(self) -> RepresentationPolicy: - """Return the shared policy narrowed by this family's override attributes.""" + """Return the effective edge-case policy for this instance. + + An explicit ``policy`` constructor argument, on the transformers that expose + one, always wins verbatim over the class-level defaults below. Otherwise, the + shared default policy is narrowed by this family's ``_constant_policy`` / + ``_out_of_range_policy`` class attributes, which record each family's + historical, non-configurable default behavior. + """ + instance_policy = getattr(self, "policy", None) + if instance_policy is not None: + return RepresentationPolicy.resolve(instance_policy) return self._policy.merge( constant=self._constant_policy, out_of_range=self._out_of_range_policy, diff --git a/pretab/core/policy.py b/pretab/core/policy.py index 4374ab0..10a27ec 100644 --- a/pretab/core/policy.py +++ b/pretab/core/policy.py @@ -7,11 +7,11 @@ * ``out_of_range``: values at ``transform`` outside the fitted range (``"error"`` / ``"warn"`` / ``"clip"`` / ``"extrapolate"``). -The defaults reproduce the library's historical behaviour (constant columns pass -through, ranges extrapolate), so enabling the policy object changes nothing -until a stricter choice is requested. Transformers may narrow specific axes -through class-level override attributes without exposing a new constructor -parameter (see :class:`~pretab.core.base.BasePreTabTransformer`). +The defaults reproduce each family's historical behaviour (B/M/I-spline, P-spline, and +tensor-product clip out-of-range inputs; natural-cubic and cubic-regression extrapolate; +constant columns pass through everywhere), so leaving ``policy`` unset changes nothing. +Transformers may narrow specific axes through class-level override attributes without +exposing a new constructor parameter (see :class:`~pretab.core.base.BasePreTabTransformer`). Missing values and non-finite (``inf`` / ``-inf``) inputs are handled elsewhere: see ``Preprocessor.missing_policy`` for missing-value handling, and note that @@ -19,13 +19,17 @@ policy (there is no configurable axis for it). .. note:: - ``out_of_range`` is not yet reachable from any public API: no transformer - constructor accepts a ``policy`` argument, and ``Preprocessor.policy`` is - only used for its own top-level ``constant`` check, never threaded into - ``PreprocessorConfig`` or the transformers it builds. Wiring this through - (transformer constructors, the registry's ``allowed_args``, and - ``PreprocessorConfig``) is deferred to a follow-up; see the "Spline + ``out_of_range`` is reachable standalone: ``BSplineTransformer``, ``MSplineTransformer``, + ``ISplineTransformer``, ``PSplineTransformer``, ``TensorProductSplineTransformer``, + ``NaturalCubicSplineTransformer``, and ``CubicRegressionSplineTransformer`` all accept a + ``policy`` constructor parameter that is genuinely respected at ``transform`` time. It is + not yet threaded through ``Preprocessor``: ``Preprocessor.policy`` is still only used for + its own top-level ``constant`` check, never passed into ``PreprocessorConfig`` or the + transformers it builds. Wiring it through the registry's ``allowed_args`` and + ``PreprocessorConfig`` remains a separately-tracked follow-up; see the "Spline expansions" section of ``dev/todo/release-1.0.0/bugfixes-1.0.0.md``. + ``ThinPlateSplineTransformer`` is not wired (its penalty is already experimental and it + has no single-feature knot range in the same sense). """ from __future__ import annotations diff --git a/pretab/expansion/spline/b_spline.py b/pretab/expansion/spline/b_spline.py index 8c0e70f..293471c 100644 --- a/pretab/expansion/spline/b_spline.py +++ b/pretab/expansion/spline/b_spline.py @@ -11,6 +11,7 @@ from scipy.interpolate import BSpline from ...core.parameters import UNSET +from ...core.policy import RepresentationPolicy from .base import BaseSplineTransformer @@ -54,6 +55,7 @@ def __init__( min_output_dim=UNSET, max_output_dim=UNSET, random_state: int | None = None, + policy: RepresentationPolicy | dict | None = None, ): super().__init__( output_dim=output_dim, @@ -67,6 +69,7 @@ def __init__( min_output_dim=min_output_dim, max_output_dim=max_output_dim, random_state=random_state, + policy=policy, ) def _feature_suffix(self) -> str: diff --git a/pretab/expansion/spline/base.py b/pretab/expansion/spline/base.py index f9a3826..ab365bf 100644 --- a/pretab/expansion/spline/base.py +++ b/pretab/expansion/spline/base.py @@ -28,6 +28,7 @@ uniform_knots, ) from ...core.parameters import UNSET, validate_placement +from ...core.policy import RepresentationPolicy, resolve_out_of_range from ...core.supervised import warn_target_leakage from ...exceptions import ( IncompatibleParamsError, @@ -93,6 +94,13 @@ class BaseSplineTransformer(BasePreTabTransformer): random_state : int or None, default=None Random state forwarded to the target-aware selector for reproducibility. + policy : RepresentationPolicy, dict, or None, default=None + Overrides this family's default ``out_of_range`` behavior (unconditional + clipping to the fitted knot range). Pass e.g. + ``RepresentationPolicy(out_of_range="error")`` to raise instead, or + ``"warn"`` / ``"extrapolate"`` for the other supported reactions. Leaving + this at ``None`` preserves the historical clip-on-transform behavior. + Attributes ---------- knots_ : list of ndarray @@ -131,6 +139,7 @@ class BaseSplineTransformer(BasePreTabTransformer): _representation_component_kind = "basis" _representation_supervision = "optional" _representation_local_support = True + _out_of_range_policy: ClassVar[str | None] = "clip" def __init__( self, @@ -145,6 +154,7 @@ def __init__( min_output_dim=UNSET, max_output_dim=UNSET, random_state: int | None = None, + policy: RepresentationPolicy | dict | None = None, ): self.output_dim = output_dim self.degree = degree @@ -157,6 +167,7 @@ def __init__( self.min_output_dim = min_output_dim self.max_output_dim = max_output_dim self.random_state = random_state + self.policy = policy _selector_spline_type: ClassVar[Literal["bspline", "mspline", "ispline"]] = "bspline" @@ -311,8 +322,8 @@ def transform(self, X): transformed = [] for i in range(X.shape[1]): knots = self.knots_[i] - xi_clipped = np.clip(X[:, i], knots[0], knots[-1]) - design = self._design_matrix(xi_clipped, knots) + xi = resolve_out_of_range(X[:, i], knots[0], knots[-1], self._resolved_policy(), estimator=self) + design = self._design_matrix(xi, knots) if self.include_bias: design = np.hstack([np.ones((design.shape[0], 1)), design]) transformed.append(design) diff --git a/pretab/expansion/spline/cubic_regression.py b/pretab/expansion/spline/cubic_regression.py index a651f9a..392a99d 100644 --- a/pretab/expansion/spline/cubic_regression.py +++ b/pretab/expansion/spline/cubic_regression.py @@ -5,6 +5,7 @@ from sklearn.utils.validation import check_is_fitted from ...core.parameters import UNSET, validate_placement +from ...core.policy import RepresentationPolicy, resolve_out_of_range from ...core.supervised import warn_target_leakage from ...exceptions import InvalidParamError from ...placement.adapters import SplinePlacementAdapter @@ -71,6 +72,12 @@ class CubicRegressionSplineTransformer(SplineBasisMixin, TransformerMixin, BaseE random_state : int or None, default=None Random state forwarded to the target-aware selector for reproducibility. + policy : RepresentationPolicy, dict, or None, default=None + Overrides this family's default ``out_of_range`` behavior (smooth polynomial + extrapolation beyond the fitted range). Pass e.g. + ``RepresentationPolicy(out_of_range="clip")`` to clamp instead. Leaving + this at ``None`` preserves the historical extrapolation behavior. + Attributes ---------- knots_ : list of ndarray @@ -128,6 +135,7 @@ def __init__( min_output_dim=UNSET, max_output_dim=UNSET, random_state: int | None = None, + policy: RepresentationPolicy | dict | None = None, ): self.output_dim = output_dim self.degree = degree @@ -139,6 +147,7 @@ def __init__( self.min_output_dim = min_output_dim self.max_output_dim = max_output_dim self.random_state = random_state + self.policy = policy def _bspline_basis(self, x, knots): x = np.asarray(x).reshape(-1, 1) @@ -201,9 +210,10 @@ def transform(self, X): check_is_fitted(self, "n_basis_") X = self._validate_allow_nan(X, reset=False) + policy = self._resolved_policy() transformed = [] for i in range(X.shape[1]): - xi = X[:, i] + xi = resolve_out_of_range(X[:, i], self.x_min_[i], self.x_max_[i], policy, estimator=self) design = self._bspline_basis(xi, self.knots_[i]) transformed.append(design) diff --git a/pretab/expansion/spline/i_spline.py b/pretab/expansion/spline/i_spline.py index cc1130e..85f374c 100644 --- a/pretab/expansion/spline/i_spline.py +++ b/pretab/expansion/spline/i_spline.py @@ -12,6 +12,7 @@ from scipy.interpolate import BSpline from ...core.parameters import UNSET +from ...core.policy import RepresentationPolicy from .base import BaseSplineTransformer @@ -54,6 +55,7 @@ def __init__( min_output_dim=UNSET, max_output_dim=UNSET, random_state: int | None = None, + policy: RepresentationPolicy | dict | None = None, ): super().__init__( output_dim=output_dim, @@ -67,6 +69,7 @@ def __init__( min_output_dim=min_output_dim, max_output_dim=max_output_dim, random_state=random_state, + policy=policy, ) def _feature_suffix(self) -> str: diff --git a/pretab/expansion/spline/m_spline.py b/pretab/expansion/spline/m_spline.py index ed8725f..f98682e 100644 --- a/pretab/expansion/spline/m_spline.py +++ b/pretab/expansion/spline/m_spline.py @@ -12,6 +12,7 @@ from scipy.interpolate import BSpline from ...core.parameters import UNSET +from ...core.policy import RepresentationPolicy from .base import BaseSplineTransformer @@ -52,6 +53,7 @@ def __init__( min_output_dim=UNSET, max_output_dim=UNSET, random_state: int | None = None, + policy: RepresentationPolicy | dict | None = None, ): super().__init__( output_dim=output_dim, @@ -65,6 +67,7 @@ def __init__( min_output_dim=min_output_dim, max_output_dim=max_output_dim, random_state=random_state, + policy=policy, ) def _feature_suffix(self) -> str: diff --git a/pretab/expansion/spline/multivariate/tensor_product.py b/pretab/expansion/spline/multivariate/tensor_product.py index 4eaaccf..2be256b 100644 --- a/pretab/expansion/spline/multivariate/tensor_product.py +++ b/pretab/expansion/spline/multivariate/tensor_product.py @@ -4,6 +4,7 @@ from ....core.knots import bspline_basis from ....core.parameters import UNSET +from ....core.policy import RepresentationPolicy, resolve_out_of_range from ....exceptions import InvalidParamError from ..mixins import SplineBasisMixin @@ -70,6 +71,12 @@ class TensorProductSplineTransformer(SplineBasisMixin, TransformerMixin, BaseEst max_output_dim : int or None, default=None Unused for this unsupervised-only family (kept for API parity). + policy : RepresentationPolicy, dict, or None, default=None + Overrides this family's default ``out_of_range`` behavior (unconditional + clipping to each marginal dimension's fitted knot range). Pass e.g. + ``RepresentationPolicy(out_of_range="error")`` to raise instead. Leaving + this at ``None`` preserves the historical clip-on-transform behavior. + Attributes ---------- dim_ : int @@ -126,6 +133,7 @@ class TensorProductSplineTransformer(SplineBasisMixin, TransformerMixin, BaseEst _representation_family = "tensorspline" _representation_scope = "multivariate" _representation_local_support = True + _out_of_range_policy = "clip" def __init__( self, @@ -137,6 +145,7 @@ def __init__( adaptive: bool = False, min_output_dim=UNSET, max_output_dim=UNSET, + policy: RepresentationPolicy | dict | None = None, ): self.output_dim = output_dim self.degree = degree @@ -146,6 +155,7 @@ def __init__( self.adaptive = adaptive self.min_output_dim = min_output_dim self.max_output_dim = max_output_dim + self.policy = policy def _pad_knots(self, inner): return np.concatenate((np.repeat(inner[0], self.degree), inner, np.repeat(inner[-1], self.degree))) @@ -219,9 +229,12 @@ def transform(self, X): check_is_fitted(self, "marginal_sizes_") X = self._validate_allow_nan(X, reset=False) + policy = self._resolved_policy() bases = [] for d in range(self.dim_): - basis = self._basis_matrix(X[:, d], self.knots_[d]) + knots = self.knots_[d] + xd = resolve_out_of_range(X[:, d], knots[0], knots[-1], policy, estimator=self) + basis = self._basis_matrix(xd, knots) bases.append(basis) n_samples = X.shape[0] diff --git a/pretab/expansion/spline/natural_cubic.py b/pretab/expansion/spline/natural_cubic.py index 9b88929..21c3f45 100644 --- a/pretab/expansion/spline/natural_cubic.py +++ b/pretab/expansion/spline/natural_cubic.py @@ -4,6 +4,7 @@ from sklearn.utils.validation import check_is_fitted from ...core.parameters import UNSET, validate_placement +from ...core.policy import RepresentationPolicy, resolve_out_of_range from ...core.supervised import warn_target_leakage from ...exceptions import InvalidParamError from ...placement.adapters import SplinePlacementAdapter @@ -73,6 +74,13 @@ class NaturalCubicSplineTransformer(SplineBasisMixin, TransformerMixin, BaseEsti random_state : int or None, default=None Random state forwarded to the target-aware selector for reproducibility. + policy : RepresentationPolicy, dict, or None, default=None + Overrides this family's default ``out_of_range`` behavior (smooth linear + extrapolation beyond the fitted range, the natural-spline boundary + constraint this family is named for). Pass e.g. + ``RepresentationPolicy(out_of_range="clip")`` to clamp instead. Leaving + this at ``None`` preserves the historical extrapolation behavior. + Attributes ---------- knots_ : list of ndarray @@ -131,6 +139,7 @@ def __init__( min_output_dim=UNSET, max_output_dim=UNSET, random_state: int | None = None, + policy: RepresentationPolicy | dict | None = None, ): self.output_dim = output_dim self.degree = degree @@ -142,6 +151,7 @@ def __init__( self.min_output_dim = min_output_dim self.max_output_dim = max_output_dim self.random_state = random_state + self.policy = policy def _basis(self, x, knots): x = np.asarray(x).reshape(-1, 1) @@ -206,10 +216,12 @@ def transform(self, X): check_is_fitted(self, "n_basis_") X = self._validate_allow_nan(X, reset=False) + policy = self._resolved_policy() transformed = [] for i in range(X.shape[1]): - xi = X[:, i] - basis = self._basis(xi, self.knots_[i]) + knots = self.knots_[i] + xi = resolve_out_of_range(X[:, i], knots[0], knots[-1], policy, estimator=self) + basis = self._basis(xi, knots) transformed.append(basis) return np.hstack(transformed) diff --git a/pretab/expansion/spline/p_spline.py b/pretab/expansion/spline/p_spline.py index 13260dc..8328484 100644 --- a/pretab/expansion/spline/p_spline.py +++ b/pretab/expansion/spline/p_spline.py @@ -4,6 +4,7 @@ from ...core.knots import bspline_basis from ...core.parameters import UNSET +from ...core.policy import RepresentationPolicy, resolve_out_of_range from ...exceptions import InvalidParamError from .mixins import SplineBasisMixin @@ -67,6 +68,12 @@ class PSplineTransformer(SplineBasisMixin, TransformerMixin, BaseEstimator): max_output_dim : int or None, default=None Unused for this unsupervised-only family (kept for API parity). + policy : RepresentationPolicy, dict, or None, default=None + Overrides this family's default ``out_of_range`` behavior (unconditional + clipping to the fitted knot range). Pass e.g. + ``RepresentationPolicy(out_of_range="error")`` to raise instead. Leaving + this at ``None`` preserves the historical clip-on-transform behavior. + Attributes ---------- knots_ : list of ndarray @@ -115,6 +122,7 @@ class PSplineTransformer(SplineBasisMixin, TransformerMixin, BaseEstimator): _feature_suffix_value = "ps" _representation_family = "pspline" _representation_local_support = True + _out_of_range_policy = "clip" def __init__( self, @@ -126,6 +134,7 @@ def __init__( adaptive: bool = False, min_output_dim=UNSET, max_output_dim=UNSET, + policy: RepresentationPolicy | dict | None = None, ): self.output_dim = output_dim self.degree = degree @@ -135,6 +144,7 @@ def __init__( self.adaptive = adaptive self.min_output_dim = min_output_dim self.max_output_dim = max_output_dim + self.policy = policy def fit(self, X, y=None): X = self._validate_allow_nan(X, reset=True) @@ -194,11 +204,12 @@ def transform(self, X): all_basis = [] for i in range(X.shape[1]): - x = X[:, i] - nb = len(self.knots_[i]) - self.degree - 1 + knots = self.knots_[i] + x = resolve_out_of_range(X[:, i], knots[0], knots[-1], self._resolved_policy(), estimator=self) + nb = len(knots) - self.degree - 1 basis = np.zeros((len(x), nb)) for j in range(nb): - basis[:, j] = bspline_basis(x, self.knots_[i], self.degree, j) + basis[:, j] = bspline_basis(x, knots, self.degree, j) if self.include_bias: basis = np.hstack([np.ones((len(x), 1)), basis]) all_basis.append(basis) diff --git a/tests/core/test_policy.py b/tests/core/test_policy.py new file mode 100644 index 0000000..0b99b20 --- /dev/null +++ b/tests/core/test_policy.py @@ -0,0 +1,127 @@ +"""Coverage for ``RepresentationPolicy.out_of_range`` actually reaching every spline family. + +Each knot/range-based spline transformer (B/M/I-spline, P-spline, tensor-product, +natural-cubic, cubic-regression) accepts its own ``policy`` constructor parameter. +This is standalone-only wiring: the policy is genuinely respected by the transformer +itself, but it is not threaded through ``Preprocessor``/the registry (that remains a +separately-tracked gap; see ``dev/todo/release-1.0.0/bugfixes-1.0.0.md``). + +These tests close the gap flagged in the v1.0.0 hardening review: no test previously +exercised ``RepresentationPolicy(out_of_range=...)`` through any spline transformer at +all, since ``resolve_out_of_range`` had zero call sites anywhere in the library. +``ThinPlateSplineTransformer`` is intentionally excluded: it has no single-feature +knot range in the same sense (a kernel evaluated at any point) and is out of scope for +this pass. +""" + +import numpy as np +import pytest + +from pretab.core.policy import RepresentationPolicy +from pretab.exceptions import DataWarning, PretabDataError +from pretab.transformers import ( + BSplineTransformer, + CubicRegressionSplineTransformer, + ISplineTransformer, + MSplineTransformer, + NaturalCubicSplineTransformer, + PSplineTransformer, + TensorProductSplineTransformer, +) + +SINGLE_FEATURE_FAMILIES = [ + (BSplineTransformer, {"output_dim": 8}), + (MSplineTransformer, {"output_dim": 8}), + (ISplineTransformer, {"output_dim": 8}), + (PSplineTransformer, {"output_dim": 8}), + (NaturalCubicSplineTransformer, {"output_dim": 6}), + (CubicRegressionSplineTransformer, {"output_dim": 8}), +] + + +@pytest.mark.parametrize(("cls", "kwargs"), SINGLE_FEATURE_FAMILIES) +def test_out_of_range_error_policy_raises(cls, kwargs): + X = np.linspace(0, 10, 100).reshape(-1, 1) + transformer = cls(policy=RepresentationPolicy(out_of_range="error"), **kwargs).fit(X) + with pytest.raises(PretabDataError): + transformer.transform(np.array([[20.0]])) + + +@pytest.mark.parametrize(("cls", "kwargs"), SINGLE_FEATURE_FAMILIES) +def test_out_of_range_warn_policy_warns(cls, kwargs): + X = np.linspace(0, 10, 100).reshape(-1, 1) + transformer = cls(policy=RepresentationPolicy(out_of_range="warn"), **kwargs).fit(X) + with pytest.warns(DataWarning): + transformer.transform(np.array([[20.0]])) + + +@pytest.mark.parametrize(("cls", "kwargs"), SINGLE_FEATURE_FAMILIES) +def test_out_of_range_clip_policy_matches_boundary_value(cls, kwargs): + X = np.linspace(0, 10, 100).reshape(-1, 1) + transformer = cls(policy=RepresentationPolicy(out_of_range="clip"), **kwargs).fit(X) + at_max = transformer.transform(np.array([[10.0]])) + past_max = transformer.transform(np.array([[20.0]])) + np.testing.assert_allclose(past_max, at_max, atol=1e-6) + + +def test_tensorproduct_out_of_range_error_policy_raises(): + X = np.linspace(0, 10, 100).reshape(-1, 1) + X = np.hstack([X, X]) + transformer = TensorProductSplineTransformer(output_dim=4, policy=RepresentationPolicy(out_of_range="error")).fit(X) + with pytest.raises(PretabDataError): + transformer.transform(np.array([[20.0, 20.0]])) + + +def test_tensorproduct_out_of_range_clip_policy_matches_boundary_value(): + X = np.linspace(0, 10, 100).reshape(-1, 1) + X = np.hstack([X, X]) + transformer = TensorProductSplineTransformer(output_dim=4, policy=RepresentationPolicy(out_of_range="clip")).fit(X) + at_max = transformer.transform(np.array([[10.0, 10.0]])) + past_max = transformer.transform(np.array([[20.0, 20.0]])) + np.testing.assert_allclose(past_max, at_max, atol=1e-6) + + +def test_natural_cubic_and_cubic_regression_default_to_extrapolate(): + # Unlike B/M/I/P-spline/tensor-product (which default to "clip"), these two + # families extrapolate smoothly by design and only clip on request. + X = np.linspace(0, 10, 100).reshape(-1, 1) + for cls, kwargs in [ + (NaturalCubicSplineTransformer, {"output_dim": 6}), + (CubicRegressionSplineTransformer, {"output_dim": 8}), + ]: + default = cls(**kwargs).fit(X) + clipped = cls(policy=RepresentationPolicy(out_of_range="clip"), **kwargs).fit(X) + at_max = default.transform(np.array([[10.0]])) + default_past_max = default.transform(np.array([[20.0]])) + clipped_past_max = clipped.transform(np.array([[20.0]])) + assert not np.allclose(default_past_max, at_max) + np.testing.assert_allclose(clipped_past_max, at_max, atol=1e-6) + + +def test_bmi_and_pspline_and_tensor_default_to_clip(): + # These families have always clipped unconditionally; confirm the default + # (policy=None) still does, now that it's routed through resolve_out_of_range. + X = np.linspace(0, 10, 100).reshape(-1, 1) + for cls, kwargs in [ + (BSplineTransformer, {"output_dim": 8}), + (MSplineTransformer, {"output_dim": 8}), + (ISplineTransformer, {"output_dim": 8}), + (PSplineTransformer, {"output_dim": 8}), + ]: + transformer = cls(**kwargs).fit(X) + at_max = transformer.transform(np.array([[10.0]])) + past_max = transformer.transform(np.array([[20.0]])) + np.testing.assert_allclose(past_max, at_max, atol=1e-6) + + X2 = np.hstack([X, X]) + transformer = TensorProductSplineTransformer(output_dim=4).fit(X2) + at_max = transformer.transform(np.array([[10.0, 10.0]])) + past_max = transformer.transform(np.array([[20.0, 20.0]])) + np.testing.assert_allclose(past_max, at_max, atol=1e-6) + + +def test_policy_dict_is_accepted(): + X = np.linspace(0, 10, 100).reshape(-1, 1) + transformer = BSplineTransformer(output_dim=8, policy={"out_of_range": "error"}).fit(X) + with pytest.raises(PretabDataError): + transformer.transform(np.array([[20.0]])) diff --git a/tests/expansion/spline/test_pspline_transformer.py b/tests/expansion/spline/test_pspline_transformer.py index d89fa2e..c7ebac1 100644 --- a/tests/expansion/spline/test_pspline_transformer.py +++ b/tests/expansion/spline/test_pspline_transformer.py @@ -95,3 +95,15 @@ def test_pspline_transform_requires_fit(): transformer.transform(np.random.rand(5, 1)) with pytest.raises(NotFittedError): transformer.get_penalty_matrix() + + +def test_pspline_out_of_range_transform_clips_to_boundary(): + # Regression test: out-of-range transform inputs used to produce an abrupt + # all-zero row instead of clipping like the B/M/I splines already do. + X = np.linspace(0, 10, 100).reshape(-1, 1) + transformer = PSplineTransformer(output_dim=8).fit(X) + at_max = transformer.transform(np.array([[10.0]])) + just_past_max = transformer.transform(np.array([[10.0 + 1e-4]])) + far_past_max = transformer.transform(np.array([[20.0]])) + np.testing.assert_allclose(just_past_max, at_max, atol=1e-6) + np.testing.assert_allclose(far_past_max, at_max, atol=1e-6) diff --git a/tests/expansion/spline/test_tensorproduct_transformer.py b/tests/expansion/spline/test_tensorproduct_transformer.py index f5ca12c..617b9a5 100644 --- a/tests/expansion/spline/test_tensorproduct_transformer.py +++ b/tests/expansion/spline/test_tensorproduct_transformer.py @@ -69,6 +69,19 @@ def test_tensorproduct_spline_penalty_matrices_match_true_quadratic_form(): assert lib_value == pytest.approx(true_value, rel=1e-8), f"mismatch for dim={dim}" +def test_tensorproduct_out_of_range_transform_clips_to_boundary(): + # Regression test: out-of-range transform inputs used to produce an abrupt + # all-zero row instead of clipping like the B/M/I splines already do. + X = np.linspace(0, 10, 100).reshape(-1, 1) + X = np.hstack([X, X]) + transformer = TensorProductSplineTransformer(output_dim=4).fit(X) + at_max = transformer.transform(np.array([[10.0, 10.0]])) + just_past_max = transformer.transform(np.array([[10.0 + 1e-4, 10.0 + 1e-4]])) + far_past_max = transformer.transform(np.array([[20.0, 20.0]])) + np.testing.assert_allclose(just_past_max, at_max, atol=1e-6) + np.testing.assert_allclose(far_past_max, at_max, atol=1e-6) + + @pytest.mark.parametrize("diff_order", [-1, 0]) def test_tensorproduct_rejects_nonpositive_diff_order(diff_order): X = np.random.default_rng(0).random((30, 2))