diff --git a/docs/core_concepts/target_awareness.md b/docs/core_concepts/target_awareness.md index 5411715..94a6453 100644 --- a/docs/core_concepts/target_awareness.md +++ b/docs/core_concepts/target_awareness.md @@ -79,6 +79,16 @@ held-out fold, so every training row is transformed by a model that never saw it `transform` on genuinely new data (a validation or test set) instead uses one model fit on all the training data, since there is no leakage risk there. +A wrapped `Preprocessor` is not refit from scratch on each fold: every fold reuses its all-data +fit and refits only the blocks whose representations use `y`. Column types and the blocks that +do not use `y`, including their category vocabularies, are shared. Their out-of-fold values match `transform` exactly; +target-aware values can differ because they exclude the held-out targets. + +The entire target-aware block is currently refit, including its imputer and missing-indicator +branch. Each refitted block must retain its all-data output width. If adaptive sizing or +fold-specific missing-indicator columns change that width, cross-fitting raises an error +instead of mixing incompatible feature layouts. + ```python import numpy as np diff --git a/pretab/compose/inspection.py b/pretab/compose/inspection.py index dc549ed..9c24c69 100644 --- a/pretab/compose/inspection.py +++ b/pretab/compose/inspection.py @@ -15,11 +15,13 @@ from ..core.logging import get_logger from ..core.representation import FeatureLineage +from .registry import TRANSFORMER_REGISTRY logger = get_logger(__name__) __all__ = [ "block_name", + "block_uses_target", "build_feature_info", "build_feature_lineage", "build_transformer_summary", @@ -366,20 +368,46 @@ def _resolve_block_representation(pipeline, columns): """Return ``(family, component, uses_target, is_interaction)`` for a block. The representation-bearing step is the last pipeline step exposing a - ``get_representation_spec`` (a PreTab transformer) or a known scikit-learn - step name; helper steps such as imputers and float casts are skipped. + ``get_representation_spec`` (a PreTab transformer), a known scikit-learn + step name, or a registered method that consumes the target; helper steps + such as imputers and float casts are skipped. ``columns`` names the spec's + inputs; with None the step names its own. """ steps = pipeline.steps if hasattr(pipeline, "steps") else [("_", pipeline)] for step_name, transformer in reversed(steps): if hasattr(transformer, "get_representation_spec"): - spec = transformer.get_representation_spec(input_features=list(columns)) + spec = transformer.get_representation_spec(input_features=None if columns is None else list(columns)) return spec.family, spec.component_kind, spec.uses_target, spec.is_interaction + registered = TRANSFORMER_REGISTRY.get(step_name) + if ( + registered is not None + and isinstance(transformer, registered.transformer_cls) + and registered.target_usage != "forbidden" + ): + # A registered class without a RepresentationSpec (e.g. scikit-learn's + # TargetEncoder): its declared supervision says whether it used y, so + # cross-fitting refits it per fold instead of reusing the all-data fit. + uses_target = registered.target_usage == "required" or bool(getattr(transformer, "target_aware", False)) + component = "category" if registered.is_categorical else "basis" + return step_name, component, uses_target, registered.is_multivariate if step_name in _STEP_FAMILY: family, component = _STEP_FAMILY[step_name] return family, component, False, False return "passthrough", "raw", False, False +def block_uses_target(transformer) -> bool: + """Whether a fitted per-column block's representation consumed the target ``y``. + + For a separate-state / missing-indicator union the representation branch decides; helper steps + such as imputers and scalers never use the target. The spec is read without + input names, since only its target use matters here. + """ + separate_state = _separate_state_branches(transformer) + representation = separate_state[0] if separate_state is not None else transformer + return bool(_resolve_block_representation(representation, None)[2]) + + def _passthrough_source(columns, offset, feature_names_in): """Resolve the source feature name for a passthrough / remainder column.""" column = columns[offset] if offset < len(columns) else columns[-1] diff --git a/pretab/core/supervised.py b/pretab/core/supervised.py index 5a4fa3f..44158be 100644 --- a/pretab/core/supervised.py +++ b/pretab/core/supervised.py @@ -124,9 +124,13 @@ class CrossFittedTransformer(RepresentationSpecMixin, TransformerMixin, BaseEsti ---------- transformer : estimator A supervised (target-aware) PreTab transformer to cross-fit, or an - estimator such as a :class:`~pretab.Preprocessor` or ``Pipeline`` built from - them. A pandas DataFrame ``X`` is passed to it unchanged (each fold is a row - subset), so column names, dtypes and index labels are preserved. Polars + estimator such as a :class:`~pretab.Preprocessor` or ``Pipeline`` built + from them. For a :class:`~pretab.Preprocessor`, only its target-aware blocks are refit per + fold: column types and the blocks that do not use ``y``, including their + category vocabularies, come from the all-data fit, so their out-of-fold features use exactly the + encoding of :meth:`transform`. Any other estimator is cloned and refit + from scratch on each fold. A pandas DataFrame ``X`` is passed through + unchanged (each fold is a row subset), so column names, dtypes and index labels are preserved. Polars DataFrames are converted once to pandas before selecting fold rows. n_folds : int, default=5 Number of cross-fitting folds. Must be at least 2. @@ -193,6 +197,21 @@ def transform(self, X): check_is_fitted(self, "estimator_") return self.estimator_.transform(_as_2d(X)) + def _fit_fold(self, X, y): + """Return the model that encodes one held-out fold, fit on the other rows. + + A wrapped estimator that implements ``_cross_fit_fold`` (the + :class:`~pretab.Preprocessor`) derives the fold model from ``estimator_`` + and refits only its target-aware parts, so the out-of-fold features share + the encoding ``transform`` uses. Any other estimator is cloned and refit. + """ + cross_fit_fold = getattr(self.estimator_, "_cross_fit_fold", None) + if cross_fit_fold is not None: + return cast(TransformerLike, cross_fit_fold(X, y)) + fold = cast(TransformerLike, clone(self.transformer)) + fold.fit(X, y) + return fold + def fit_transform(self, X, y=None): """Fit and return leakage-free out-of-fold features for the training data.""" X_arr, y_arr = self._fit_full(X, y) @@ -202,14 +221,14 @@ def fit_transform(self, X, y=None): token = _cross_fit_active.set(True) try: for train_idx, test_idx in splitter.split(X_arr, y_arr): - fold = cast(TransformerLike, clone(self.transformer)) - fold.fit(_take_rows(X_arr, train_idx), y_arr[train_idx]) + fold = self._fit_fold(_take_rows(X_arr, train_idx), y_arr[train_idx]) fold_out = np.asarray(fold.transform(_take_rows(X_arr, test_idx))) if fold_out.shape[1] != width: raise IncompatibleParamsError( "Cross-fitting requires a fixed output width across folds; expected " - f"{width}, got {fold_out.shape[1]}. Disable adaptive sizing on the " - "wrapped transformer." + f"{width}, got {fold_out.shape[1]}. The wrapped transformer's width " + "depends on the training rows (e.g. adaptive sizing); fix it, for " + "example by disabling adaptive sizing." ) out[test_idx] = fold_out finally: diff --git a/pretab/extension.py b/pretab/extension.py index 64af86e..7129eed 100644 --- a/pretab/extension.py +++ b/pretab/extension.py @@ -159,7 +159,13 @@ def register_representation( Inferred from ``cls`` when omitted. An ``"optional"`` class must take a ``target_aware`` constructor parameter: ``Preprocessor`` always passes its own ``target_aware``, so the class uses ``y`` only when that is True. A - class without one raises ``TypeError``. + class without one raises ``TypeError``. For a class without a + ``get_representation_spec`` (a plain scikit-learn transformer such as + ``TargetEncoder``), this declaration decides whether a fitted block used + ``y``: ``"supervised"``, or ``"optional"`` with ``target_aware`` set on the + instance, counts as target-aware. The feature lineage then reports + ``uses_target``, and :class:`~pretab.CrossFittedTransformer` refits the + block on every fold of a wrapped :class:`~pretab.Preprocessor`. allowed_args : iterable of str, default=() Constructor argument names the shared Preprocessor keyword arguments are filtered down to for this method. diff --git a/pretab/preprocessor.py b/pretab/preprocessor.py index 958f9b7..170dccc 100644 --- a/pretab/preprocessor.py +++ b/pretab/preprocessor.py @@ -1,10 +1,11 @@ +import copy import hashlib import json import logging import os import time import warnings -from typing import Any, cast +from typing import cast import numpy as np from scipy import sparse as sp @@ -23,6 +24,7 @@ ) from .compose.inspection import ( block_name, + block_uses_target, build_feature_info, build_feature_lineage, build_transformer_summary, @@ -39,6 +41,7 @@ validate_embedding_request, ) from .compose.serialize import SCHEMA_VERSION, preprocessor_from_spec, preprocessor_to_spec +from .core._typing import TransformerLike from .core.logging import configure_logging, get_logger from .core.parameters import UNSET, validate_task from .core.policy import RepresentationPolicy, apply_constant_policy @@ -46,6 +49,7 @@ ConfigWarning, DataWarning, FrozenRepresentationError, + IncompatibleParamsError, OutputBudgetError, PretabConfigError, PretabDataError, @@ -1113,6 +1117,43 @@ def estimate_memory(self, X) -> int: n_rows, n_cols = self.estimate_output_shape(X) return int(n_rows * n_cols * self._output_itemsize()) + def _cross_fit_fold(self, X, y): + """Return a copy of this fit whose target-aware blocks are refit on ``(X, y)``. + + Used by :class:`~pretab.CrossFittedTransformer` for its out-of-fold + features. Only the blocks that consume the target are refit on the fold; + column types and target-free blocks, including their category vocabularies, are shared + with this all-data fit, so an out-of-fold row is encoded exactly like + ``transform`` encodes it, except that no target-aware placement has seen + that row's target. Refitting the whole preprocessor per fold would + re-detect column types and relearn categories on fewer rows, shifting + integer codes and one-hot columns between the folds and ``transform``. + """ + check_is_fitted(self) + # Each block sees the fold rows exactly as the internal ColumnTransformer + # routed the training rows to it in ``fit``. + X_ct = self._column_transformer_input(self._align_input(X)) + column_transformer = copy.copy(self.column_transformer_) + column_transformer.transformers_ = [] + for name, transformer, columns in self.column_transformer_.transformers_: + fold_transformer = transformer + if name != "remainder" and block_uses_target(transformer): + fold_transformer = cast(TransformerLike, clone(transformer)) + fold_transformer.fit(X_ct[list(columns)], y) + span = self.column_transformer_.output_indices_[name] + expected_width = span.stop - span.start + width = len(fold_transformer.get_feature_names_out(list(columns))) + if width != expected_width: + raise IncompatibleParamsError( + f"Cross-fitting requires a fixed output width for block {block_name(name, columns)!r}; " + f"expected {expected_width}, got {width}. The block's width depends on the training rows; " + "use a fixed-width representation and stable missing-indicator columns." + ) + column_transformer.transformers_.append((name, fold_transformer, columns)) + fold = copy.copy(self) + fold.column_transformer_ = column_transformer + return fold + def _reject_missing(self, X) -> None: """Raise when ``missing_policy="error"`` but ``X`` contains missing values.""" na_columns = [col for col in X.columns if X[col].isna().any()] @@ -1144,7 +1185,7 @@ def _raise_for_empty_features(self, X, y, empty_features, config) -> None: feature = empty[columns[0]] is_numerical = feature in self.numerical_features_ try: - cast(Any, clone(transformer)).fit(X[columns], y) + cast(TransformerLike, clone(transformer)).fit(X[columns], y) except PretabConfigError: continue # a configuration problem, not one of the empty column except ValueError as exc: @@ -1155,7 +1196,7 @@ def _raise_for_empty_features(self, X, y, empty_features, config) -> None: else np.resize(np.array(["a", "b"], dtype=object), len(X)) ) try: - cast(Any, clone(transformer)).fit(stand_in, y) + cast(TransformerLike, clone(transformer)).fit(stand_in, y) except ValueError: continue # the block fails with observed values too: not the emptiness method = config.method_for(feature, is_numerical=is_numerical) diff --git a/tests/core/test_cross_fitted.py b/tests/core/test_cross_fitted.py index de4867d..a26bfbb 100644 --- a/tests/core/test_cross_fitted.py +++ b/tests/core/test_cross_fitted.py @@ -107,6 +107,171 @@ def test_invalid_task_rejected(data): CrossFittedTransformer(PLETransformer(), task="classificaton").fit_transform(X, y) +# --- a wrapped Preprocessor keeps one encoding across folds (issue #79) ------------- + + +@pytest.fixture +def array_with_rare_category(): + rng = np.random.default_rng(0) + X = np.column_stack( + [ + rng.integers(0, 100_000, 300), # many distinct integers: numerical + rng.integers(1, 9, 300), # unique ratio just below the default cat_cutoff + rng.choice([1, 2, 3], 300), # category codes + ] + ) + X[7, 2] = 0 # a category some folds never see, sorted before the others + y = (X[:, 0] / 50_000.0) ** 2 + (X[:, 2] == 2) + rng.normal(scale=0.1, size=300) + return X, y + + +def _split_columns(cross_fitted, prefix): + names = list(cross_fitted.get_feature_names_out()) + return [i for i, name in enumerate(names) if name.startswith(prefix)] + + +def test_wrapped_preprocessor_out_of_fold_codes_match_transform(array_with_rare_category): + """Each fold used to refit the whole Preprocessor: category codes shifted when a + category was missing from a fold, and column types were re-detected on fewer rows.""" + from pretab import Preprocessor + + X, y = array_with_rare_category + cross_fitted = CrossFittedTransformer(Preprocessor(output_dim=4, random_state=0), n_folds=5, random_state=0) + out = cross_fitted.fit_transform(X, y) + full = np.asarray(cross_fitted.transform(X)) + + unsupervised = _split_columns(cross_fitted, "cat_") + assert len(unsupervised) == 2 # both integer-coded columns are categorical on the full data + np.testing.assert_array_equal(out[:, unsupervised], full[:, unsupervised]) + np.testing.assert_array_equal(np.unique(out[:, unsupervised[1]]), [1.0, 2.0, 3.0, 4.0]) + + +def test_wrapped_preprocessor_target_aware_blocks_stay_out_of_fold(array_with_rare_category): + """Only the target-aware block is refit, on each fold's training rows.""" + import pandas as pd + from sklearn.base import clone + + from pretab import Preprocessor + + X, y = array_with_rare_category + cross_fitted = CrossFittedTransformer(Preprocessor(output_dim=4, random_state=0), n_folds=5, random_state=0) + out = cross_fitted.fit_transform(X, y) + supervised = _split_columns(cross_fitted, "num_feature_0_ple") + assert len(supervised) == 4 + + wrapped = cast(Preprocessor, cross_fitted.estimator_) + block = wrapped.column_transformer_.named_transformers_["num_feature_0"] + column = pd.DataFrame({"feature_0": X[:, 0]}) + for train_idx, test_idx in KFold(n_splits=5, shuffle=True, random_state=0).split(X): + fold_block = clone(block).fit(column.iloc[train_idx], y[train_idx]) + np.testing.assert_allclose(out[np.ix_(test_idx, supervised)], fold_block.transform(column.iloc[test_idx])) + assert not np.allclose(out[:, supervised], np.asarray(cross_fitted.transform(X))[:, supervised]) + + +def test_wrapped_preprocessor_with_one_hot_keeps_a_fixed_width(array_with_rare_category): + from pretab import Preprocessor + + X, y = array_with_rare_category + wrapped = Preprocessor(categorical_method="one-hot", output_dim=4, random_state=0) + cross_fitted = CrossFittedTransformer(wrapped, n_folds=5, random_state=0) + out = cross_fitted.fit_transform(X, y) + one_hot = _split_columns(cross_fitted, "cat_") + + assert out.shape == (len(X), len(cross_fitted.get_feature_names_out())) + np.testing.assert_array_equal(out[:, one_hot], np.asarray(cross_fitted.transform(X))[:, one_hot]) + + +def test_wrapped_unsupervised_preprocessor_matches_transform(array_with_rare_category): + from pretab import Preprocessor + + X, y = array_with_rare_category + wrapped = Preprocessor(numerical_method="minmax", target_aware=False, placement_strategy="quantile") + cross_fitted = CrossFittedTransformer(wrapped, n_folds=3, random_state=0) + np.testing.assert_array_equal(cross_fitted.fit_transform(X, y), np.asarray(cross_fitted.transform(X))) + + +def test_wrapped_preprocessor_keeps_the_all_data_column_types(): + """25 integer levels are categorical on 1000 rows (25/1000 < cat_cutoff) but were + re-detected as numerical on each 800-row fold, so the out-of-fold features held + min-max values where transform returned category codes.""" + from pretab import Preprocessor + + rng = np.random.default_rng(0) + X = rng.integers(0, 25, (1000, 1)) + cross_fitted = CrossFittedTransformer(Preprocessor(numerical_method="minmax"), n_folds=5, random_state=0) + out = cross_fitted.fit_transform(X, rng.normal(size=1000)) + + assert cast(Preprocessor, cross_fitted.estimator_).categorical_features_ == ["feature_0"] + np.testing.assert_array_equal(out, np.asarray(cross_fitted.transform(X))) + np.testing.assert_array_equal(np.unique(out), np.arange(1, 26)) + + +def _one_missing_value(X): + X[7, 0] = np.nan # a single missing value: the fold holding out row 7 trains on none + return X + + +@pytest.mark.parametrize( + ("X", "method"), + [ + (_one_missing_value(np.random.default_rng(0).normal(size=(200, 2))), {"numerical_method": "minmax"}), + (_one_missing_value(np.random.default_rng(0).normal(size=(200, 2))), {"numerical_method": "standardization"}), + ( + _one_missing_value(np.random.default_rng(0).normal(size=(200, 2))), + {"numerical_method": "bspline", "target_aware": False, "placement_strategy": "uniform"}, + ), + (_one_missing_value(np.random.default_rng(0).choice(["a", "b", "c"], (200, 1)).astype(object)), {}), + ( + _one_missing_value(np.random.default_rng(0).choice(["a", "b", "c"], (200, 1)).astype(object)), + {"categorical_method": "one-hot"}, + ), + ], + ids=["minmax", "standardization", "bspline-unsupervised", "categorical-int", "categorical-one-hot"], +) +def test_wrapped_preprocessor_shares_indicator_blocks_that_do_not_use_y(X, method): + """Blocks that do not use y keep their all-data fit, missing indicator included: each + fold used to refit them, and a fold whose training rows hold no missing value + dropped the indicator column.""" + from pretab import Preprocessor + + y = np.random.default_rng(1).normal(size=len(X)) + wrapped = Preprocessor(add_missing_indicator=True, output_dim=6, random_state=0, **method) + cross_fitted = CrossFittedTransformer(wrapped, n_folds=5, random_state=0) + out = cross_fitted.fit_transform(X, y) + assert out.shape[1] == len(cross_fitted.get_feature_names_out()) > X.shape[1] # indicator columns kept + np.testing.assert_array_equal(out, np.asarray(cross_fitted.transform(X))) + + +def test_wrapped_preprocessor_reads_the_target_use_of_an_indicator_block(): + """With add_missing_indicator, a representation is fitted on its column and the + indicator; deciding whether it used y must not depend on its input names.""" + from pretab import Preprocessor + + rng = np.random.default_rng(0) + X = rng.normal(size=(200, 1)) + X[::4, 0] = np.nan # every fold's training rows hold missing values + y = np.nan_to_num(X[:, 0]) + rng.normal(scale=0.1, size=200) + wrapped = Preprocessor(numerical_method="rbf", add_missing_indicator=True, output_dim=4, random_state=0) + cross_fitted = CrossFittedTransformer(wrapped, n_folds=5, random_state=0) + out = cross_fitted.fit_transform(X, y) + + full = np.asarray(cross_fitted.transform(X)) + assert out.shape == full.shape == (200, len(cross_fitted.get_feature_names_out())) + assert not np.allclose(out, full) # refit on every fold + + +def test_width_mismatch_message_does_not_blame_adaptive_sizing_only(): + from sklearn.preprocessing import OneHotEncoder + + x = np.array(["a"] * 99 + ["b"]).reshape(-1, 1) # the fold holding out "b" never sees it + y = np.arange(100.0) + cross_fitted = CrossFittedTransformer( + OneHotEncoder(handle_unknown="ignore", sparse_output=False), n_folds=5, random_state=0 + ) + with pytest.raises(IncompatibleParamsError, match="depends on the training rows"): + cross_fitted.fit_transform(x, y) + + # --- DataFrame input is passed through (issue #58) -------------------------------- @@ -220,6 +385,7 @@ def test_dataframe_fold_rows_keep_index_dtypes_and_target_alignment(mixed_frame, original = X.copy(deep=True) fits, transforms = [], [] original_fit = Preprocessor.fit + original_cross_fit_fold = Preprocessor._cross_fit_fold original_transform = Preprocessor.transform def record_fit(self, X, y=None): @@ -232,7 +398,13 @@ def record_transform(self, X): transforms.append(X.copy(deep=True)) return original_transform(self, X) + def record_cross_fit_fold(self, X, y): + assert isinstance(X, pd.DataFrame) + fits.append((X.copy(deep=True), np.asarray(y).copy())) + return original_cross_fit_fold(self, X, y) + monkeypatch.setattr(Preprocessor, "fit", record_fit) + monkeypatch.setattr(Preprocessor, "_cross_fit_fold", record_cross_fit_fold) monkeypatch.setattr(Preprocessor, "transform", record_transform) cross_fitted = CrossFittedTransformer( Preprocessor(output_dim=4, output_format="dense", random_state=0), n_folds=3, random_state=0 @@ -254,3 +426,67 @@ def record_transform(self, X): cross_fitted.transform(unseen) pd.testing.assert_frame_equal(transforms[-1], unseen) pd.testing.assert_frame_equal(X, original) + + +@pytest.mark.parametrize("method", ["ple", "bspline", "mspline", "ispline", "rbf"]) +@pytest.mark.parametrize("task", ["regression", "classification"]) +def test_cross_fitting_lightgbm_refits_only_target_blocks_and_keeps_the_full_fit(method, task): + pytest.importorskip("lightgbm") + import pandas as pd + from sklearn.base import clone + + from pretab import Preprocessor + + rng = np.random.default_rng(0) + X = pd.DataFrame({1: rng.normal(size=160), 2: rng.choice([True, False], size=160)}) + y = 2 * X[1].to_numpy() + X[2].to_numpy() + rng.normal(scale=0.1, size=len(X)) + if task == "classification": + y = np.where(y >= np.median(y), "high", "low") + preprocessor = Preprocessor( + numerical_method=method, + categorical_method="one-hot", + placement_strategy="lightgbm", + output_dim=6, + output_format="dense", + task=task, + random_state=0, + ) + cross_fitted = CrossFittedTransformer(preprocessor, n_folds=3, task=task, random_state=0) + out = cross_fitted.fit_transform(X, y) + full = cross_fitted.transform(X) + assert out.shape == full.shape == (len(X), 8) + assert not np.allclose(out[:, :6], full[:, :6]) + np.testing.assert_array_equal(out[:, 6:], full[:, 6:]) + reference = clone(preprocessor) + assert isinstance(reference, Preprocessor) + reference_out = reference.fit(X, y).transform(X) + assert isinstance(reference_out, np.ndarray) + np.testing.assert_allclose(full, reference_out) + assert len(cross_fitted.get_feature_names_out()) == 8 + fitted = cross_fitted.estimator_ + assert isinstance(fitted, Preprocessor) + assert [item.uses_target for item in fitted.get_feature_lineage()] == [True] * 6 + [False] * 2 + + +@pytest.mark.parametrize("labels", [[1, 2], [np.int64(7), np.int64(9)], [1, "flag"], ["amount", "flag"]]) +def test_cross_fitting_routes_integer_labels_and_boolean_columns_like_full_fit(labels): + import pandas as pd + + from pretab import Preprocessor + + rng = np.random.default_rng(0) + X = pd.DataFrame( + {labels[0]: rng.normal(size=200), labels[1]: rng.choice([True, False], size=200)}, + index=rng.permutation(1000)[:200], + ) + original = X.copy(deep=True) + y = np.sin(X.iloc[:, 0].to_numpy()) + X.iloc[:, 1].to_numpy() + rng.normal(scale=0.1, size=200) + + def cross_fit(frame): + return CrossFittedTransformer( + Preprocessor(output_dim=4, output_format="dense", random_state=0), n_folds=5, random_state=0 + ).fit_transform(frame, y) + + expected = cross_fit(X.set_axis(["amount", "flag"], axis=1)) + np.testing.assert_allclose(cross_fit(X), expected) + pd.testing.assert_frame_equal(X, original) diff --git a/tests/extension/test_registered_cross_fitting.py b/tests/extension/test_registered_cross_fitting.py new file mode 100644 index 0000000..c520052 --- /dev/null +++ b/tests/extension/test_registered_cross_fitting.py @@ -0,0 +1,179 @@ +"""Cross-fitting a Preprocessor that uses a registered, target-consuming class. + +Regression guard: a wrapped Preprocessor refits only its target-aware blocks per +fold, but a registered class without a ``RepresentationSpec`` (a plain +scikit-learn transformer such as ``TargetEncoder``) was never recognised as +target-aware. Its out-of-fold rows were then encoded by the all-data fit, which +had seen their own targets. +""" + +import warnings + +import numpy as np +import pandas as pd +import pytest +from sklearn.base import BaseEstimator, TransformerMixin +from sklearn.model_selection import KFold +from sklearn.preprocessing import TargetEncoder + +from pretab import CrossFittedTransformer, Preprocessor, register_representation +from pretab.exceptions import IncompatibleParamsError + + +class _MeanEncoder(TransformerMixin, BaseEstimator): + """Plain, optionally supervised encoder: category -> mean target when target-aware.""" + + def __init__(self, target_aware=True): + self.target_aware = target_aware + + def fit(self, X, y=None): + x = np.asarray(X, dtype=object)[:, 0] + if self.target_aware: + y = np.asarray(y, dtype=float) + self.mapping_ = {c: y[x == c].mean() for c in np.unique(x)} + else: + self.mapping_ = {c: float(i) for i, c in enumerate(np.unique(x))} + self.n_features_in_ = 1 + return self + + def transform(self, X): + return np.array([[self.mapping_.get(c, 0.0)] for c in np.asarray(X, dtype=object)[:, 0]]) + + def get_feature_names_out(self, input_features=None): + return np.asarray(input_features if input_features is not None else ["x0"], dtype=object) + + +class _OffByDefaultMeanEncoder(_MeanEncoder): + def __init__(self, target_aware=False): + super().__init__(target_aware=target_aware) + + +class _ChangingWidthEncoder(TransformerMixin, BaseEstimator): + def fit(self, X, y=None): + x = np.asarray(X, dtype=float) + self.n_features_in_ = x.shape[1] + self.width_ = 2 if len(x) == 100 else (3 if x.mean() > 0 else 1) + self.target_mean_ = np.asarray(y, dtype=float).mean() + return self + + def transform(self, X): + return np.repeat(np.asarray(X, dtype=float) + self.target_mean_, self.width_, axis=1) + + def get_feature_names_out(self, input_features=None): + name = "x0" if input_features is None else input_features[0] + return np.array([f"{name}_{i}" for i in range(self.width_)], dtype=object) + + +@pytest.fixture +def noise_target_data(): + """An id column with four rows per id and a target of pure noise.""" + rng = np.random.default_rng(0) + X = pd.DataFrame({"x": rng.normal(size=400), "id": np.repeat([f"id{i}" for i in range(100)], 4)}) + return X, rng.normal(size=400) + + +@pytest.mark.parametrize( + ("cls", "supervision"), + [(TargetEncoder, "supervised"), (_MeanEncoder, "optional")], +) +def test_registered_target_encoder_is_refit_per_fold(noise_target_data, cls, supervision): + X, y = noise_target_data + register_representation("registered_encoder", cls, feature_kind="categorical", supervision=supervision) + pre = Preprocessor(numerical_method="minmax", categorical_method="registered_encoder", random_state=0) + + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + out_of_fold = np.asarray(CrossFittedTransformer(pre, n_folds=5, random_state=0).fit_transform(X, y)) + lineage = {item.output_feature: item.uses_target for item in pre.fit(X, y).get_feature_lineage()} + + # An encoding fit without each row's own target cannot predict pure noise; the + # all-data fit correlated at about 0.5. + assert abs(np.corrcoef(out_of_fold[:, -1], y)[0, 1]) < 0.2 + assert lineage["cat_id"] is True + + +@pytest.mark.parametrize("target_aware", [False, True]) +@pytest.mark.parametrize("boolean_input", [False, True]) +def test_optional_class_follows_the_preprocessor_target_aware_across_folds( + noise_target_data, target_aware, boolean_input +): + X, y = noise_target_data + if boolean_input: + X = X.assign(id=np.arange(len(X)) % 2 == 0) + register_representation( + "off_by_default_encoder", _OffByDefaultMeanEncoder, feature_kind="categorical", supervision="optional" + ) + cross_fitted = CrossFittedTransformer( + Preprocessor( + numerical_method="minmax", + categorical_method="off_by_default_encoder", + target_aware=target_aware, + placement_strategy="cart" if target_aware else "quantile", + output_format="dense", + random_state=0, + ), + n_folds=5, + random_state=0, + ) + out = cross_fitted.fit_transform(X, y) + full = cross_fitted.transform(X) + fitted = cross_fitted.estimator_ + assert isinstance(fitted, Preprocessor) + encoder = fitted.column_transformer_.named_transformers_["cat_id"].steps[-1][1] + assert encoder.target_aware is target_aware + lineage = {item.output_feature: item.uses_target for item in fitted.get_feature_lineage()} + assert lineage["cat_id"] is target_aware + if not target_aware: + np.testing.assert_array_equal(out, full) + else: + expected = np.empty(len(X)) + for train_idx, test_idx in KFold(n_splits=5, shuffle=True, random_state=0).split(X): + encoder = _OffByDefaultMeanEncoder(target_aware=True).fit(X.iloc[train_idx][["id"]], y[train_idx]) + expected[test_idx] = encoder.transform(X.iloc[test_idx][["id"]]).ravel() + np.testing.assert_allclose(out[:, -1], expected) + assert abs(np.corrcoef(out[:, -1], y)[0, 1]) < 0.2 + assert not np.allclose(out[:, -1], full[:, -1]) + + +def test_registered_target_using_override_of_a_builtin_is_refit_per_fold(noise_target_data): + _, y = noise_target_data + X = pd.DataFrame({"x": np.repeat(np.arange(100, dtype=float), 4)}) + register_representation("minmax", _MeanEncoder, feature_kind="numerical", supervision="optional", override=True) + cross_fitted = CrossFittedTransformer( + Preprocessor(numerical_method="minmax", output_format="dense"), n_folds=5, random_state=0 + ) + out = cross_fitted.fit_transform(X, y) + assert abs(np.corrcoef(out[:, 0], y)[0, 1]) < 0.2 + fitted = cross_fitted.estimator_ + assert isinstance(fitted, Preprocessor) + assert fitted.get_feature_lineage()[0].uses_target is True + + +def test_registered_method_named_like_a_helper_does_not_make_the_helper_target_aware(noise_target_data): + X, y = noise_target_data + X = X.copy() + X.loc[7, "id"] = "rare" + register_representation("to_float", _MeanEncoder, feature_kind="categorical", supervision="supervised") + cross_fitted = CrossFittedTransformer( + Preprocessor(numerical_method="minmax", categorical_method="one-hot", output_format="dense"), + n_folds=5, + random_state=0, + ) + np.testing.assert_array_equal(cross_fitted.fit_transform(X, y), cross_fitted.transform(X)) + fitted = cross_fitted.estimator_ + assert isinstance(fitted, Preprocessor) + assert not any(item.uses_target for item in fitted.get_feature_lineage()) + + +def test_opposite_block_width_changes_are_rejected_even_when_total_width_is_fixed(): + register_representation("changing_width", _ChangingWidthEncoder, feature_kind="numerical", supervision="supervised") + X = pd.DataFrame({"a": np.linspace(1, 2, 100), "b": np.linspace(-2, -1, 100)}) + y = np.random.default_rng(0).normal(size=100) + cross_fitted = CrossFittedTransformer( + Preprocessor(numerical_method="changing_width", scaling="none", output_format="dense"), + n_folds=5, + random_state=0, + ) + # Full-data widths are (2, 2); fold widths (3, 1) keep the same total but move the feature boundary. + with pytest.raises(IncompatibleParamsError, match=r"block 'num_a'.*expected 2, got 3"): + cross_fitted.fit_transform(X, y)