From d7707db187677c022e5e2ba1456b80b1eb2e118d Mon Sep 17 00:00:00 2001 From: ChrisW09 <50968720+ChrisW09@users.noreply.github.com> Date: Fri, 16 Oct 2026 14:20:31 +0200 Subject: [PATCH] fix(cross-fitting): refit only target-aware Preprocessor blocks per fold CrossFittedTransformer refit a fresh clone of the wrapped estimator on each fold. For a Preprocessor that re-detected column types (the cat_cutoff ratio depends on the row count) and relearned category vocabularies on fewer rows, so the out-of-fold training features used different integer codes, a different numerical/categorical split, or a different one-hot width than transform, which crashed the fit with a message about adaptive sizing. A Preprocessor now derives each fold model from the all-data fit and refits only the blocks that consume the target; everything else is shared, so out-of-fold rows are encoded exactly like transform encodes them while target-aware placement still never sees a row's own target. CrossFittedTransformer finds this through a duck-typed _cross_fit_fold hook, so core keeps no dependency on the Preprocessor. Other estimators keep the clone-and-refit path, and the width-mismatch message no longer blames adaptive sizing only. A block is target-aware when its representation reports uses_target in its RepresentationSpec (read without input names, since with add_missing_indicator the representation is fitted on its column and the imputer's indicator). A class registered with supervision="supervised", or "optional" with target_aware set, that has no RepresentationSpec (such as scikit-learn's TargetEncoder) is resolved from its registry entry, so it is refit on every fold instead of encoding out-of-fold rows with an all-data fit that saw their targets, and the feature lineage reports uses_target for it. Fixes #79 --- docs/core_concepts/target_awareness.md | 5 + pretab/compose/inspection.py | 32 +++- pretab/core/supervised.py | 30 +++- pretab/extension.py | 8 +- pretab/preprocessor.py | 36 +++- tests/core/test_cross_fitted.py | 166 ++++++++++++++++++ .../test_registered_cross_fitting.py | 69 ++++++++ 7 files changed, 336 insertions(+), 10 deletions(-) create mode 100644 tests/extension/test_registered_cross_fitting.py diff --git a/docs/core_concepts/target_awareness.md b/docs/core_concepts/target_awareness.md index d3eccf2..249c2ae 100644 --- a/docs/core_concepts/target_awareness.md +++ b/docs/core_concepts/target_awareness.md @@ -72,6 +72,11 @@ held-out fold, so every training row is transformed by a model that never saw it `transform` on genuinely new data (a validation or test set) instead uses one model fit on all the training data, since there is no leakage risk there. +A wrapped `Preprocessor` is not refit from scratch on each fold: every fold reuses its all-data +fit and refits only the target-aware representations. Column types, category codes and the +blocks that do not use `y` are shared, so the out-of-fold features have exactly the columns and +encoding `transform` produces. + ```python import numpy as np diff --git a/pretab/compose/inspection.py b/pretab/compose/inspection.py index 7f3b9b3..7f0ced3 100644 --- a/pretab/compose/inspection.py +++ b/pretab/compose/inspection.py @@ -12,10 +12,12 @@ from ..core.logging import get_logger from ..core.representation import FeatureLineage +from .registry import TRANSFORMER_REGISTRY logger = get_logger(__name__) __all__ = [ + "block_uses_target", "build_feature_info", "build_feature_lineage", "build_transformer_summary", @@ -257,20 +259,44 @@ def _resolve_block_representation(pipeline, columns): """Return ``(family, component, uses_target, is_interaction)`` for a block. The representation-bearing step is the last pipeline step exposing a - ``get_representation_spec`` (a PreTab transformer) or a known scikit-learn - step name; helper steps such as imputers and float casts are skipped. + ``get_representation_spec`` (a PreTab transformer), a known scikit-learn + step name, or a registered method that consumes the target; helper steps + such as imputers and float casts are skipped. ``columns`` names the spec's + inputs; with None the step names its own. """ steps = pipeline.steps if hasattr(pipeline, "steps") else [("_", pipeline)] for step_name, transformer in reversed(steps): if hasattr(transformer, "get_representation_spec"): - spec = transformer.get_representation_spec(input_features=list(columns)) + spec = transformer.get_representation_spec(input_features=None if columns is None else list(columns)) return spec.family, spec.component_kind, spec.uses_target, spec.is_interaction if step_name in _STEP_FAMILY: family, component = _STEP_FAMILY[step_name] return family, component, False, False + registered = TRANSFORMER_REGISTRY.get(step_name) + if registered is not None and registered.target_usage != "forbidden": + # A registered class without a RepresentationSpec (e.g. scikit-learn's + # TargetEncoder): its declared supervision says whether it used y, so + # cross-fitting refits it per fold instead of reusing the all-data fit. + uses_target = registered.target_usage == "required" or bool(getattr(transformer, "target_aware", False)) + component = "category" if registered.is_categorical else "basis" + return step_name, component, uses_target, registered.is_multivariate return "passthrough", "raw", False, False +def block_uses_target(transformer) -> bool: + """Whether a fitted per-column block's representation consumed the target ``y``. + + For a separate-state union the representation branch decides; helper steps + such as imputers and scalers never use the target. The spec is read without + input names, since only its target use matters here and the representation + may have been fitted on more inputs than the block's columns (the imputer's + missing indicator is fed to it as a second input). + """ + separate_state = _separate_state_branches(transformer) + representation = separate_state[0] if separate_state is not None else transformer + return bool(_resolve_block_representation(representation, None)[2]) + + def _passthrough_source(columns, offset, feature_names_in): """Resolve the source feature name for a passthrough / remainder column.""" column = columns[offset] if offset < len(columns) else columns[-1] diff --git a/pretab/core/supervised.py b/pretab/core/supervised.py index f6aca3b..ce1f439 100644 --- a/pretab/core/supervised.py +++ b/pretab/core/supervised.py @@ -98,7 +98,12 @@ class CrossFittedTransformer(RepresentationSpecMixin, TransformerMixin, BaseEsti Parameters ---------- transformer : estimator - A supervised (target-aware) PreTab transformer to cross-fit. + A supervised (target-aware) PreTab transformer to cross-fit. For a + :class:`~pretab.Preprocessor`, only its target-aware blocks are refit per + fold: column types, category codes and the blocks that do not use ``y`` + come from the all-data fit, so the out-of-fold features use exactly the + encoding of :meth:`transform`. Any other estimator is cloned and refit + from scratch on each fold. n_folds : int, default=5 Number of cross-fitting folds. Must be at least 2. task : {"regression", "classification"}, default="regression" @@ -170,6 +175,21 @@ def transform(self, X): X_arr = X_arr.reshape(-1, 1) return self.estimator_.transform(X_arr) + def _fit_fold(self, X, y): + """Return the model that encodes one held-out fold, fit on the other rows. + + A wrapped estimator that implements ``_cross_fit_fold`` (the + :class:`~pretab.Preprocessor`) derives the fold model from ``estimator_`` + and refits only its target-aware parts, so the out-of-fold features share + the encoding ``transform`` uses. Any other estimator is cloned and refit. + """ + cross_fit_fold = getattr(self.estimator_, "_cross_fit_fold", None) + if cross_fit_fold is not None: + return cast(TransformerLike, cross_fit_fold(X, y)) + fold = cast(TransformerLike, clone(self.transformer)) + fold.fit(X, y) + return fold + def fit_transform(self, X, y=None): """Fit and return leakage-free out-of-fold features for the training data.""" X_arr, y_arr = self._fit_full(X, y) @@ -179,14 +199,14 @@ def fit_transform(self, X, y=None): token = _cross_fit_active.set(True) try: for train_idx, test_idx in splitter.split(X_arr, y_arr): - fold = cast(TransformerLike, clone(self.transformer)) - fold.fit(X_arr[train_idx], y_arr[train_idx]) + fold = self._fit_fold(X_arr[train_idx], y_arr[train_idx]) fold_out = np.asarray(fold.transform(X_arr[test_idx])) if fold_out.shape[1] != width: raise IncompatibleParamsError( "Cross-fitting requires a fixed output width across folds; expected " - f"{width}, got {fold_out.shape[1]}. Disable adaptive sizing on the " - "wrapped transformer." + f"{width}, got {fold_out.shape[1]}. The wrapped transformer's width " + "depends on the training rows (e.g. adaptive sizing); fix it, for " + "example by disabling adaptive sizing." ) out[test_idx] = fold_out finally: diff --git a/pretab/extension.py b/pretab/extension.py index e8107ed..86c6096 100644 --- a/pretab/extension.py +++ b/pretab/extension.py @@ -145,7 +145,13 @@ def register_representation( scope : {"univariate", "multivariate"}, optional Inferred from ``cls`` when omitted. supervision : {"unsupervised", "optional", "supervised"}, optional - Inferred from ``cls`` when omitted. + Inferred from ``cls`` when omitted. For a class without a + ``get_representation_spec`` (a plain scikit-learn transformer such as + ``TargetEncoder``), this declaration decides whether a fitted block used + ``y``: ``"supervised"``, or ``"optional"`` with ``target_aware`` set on the + instance, counts as target-aware. The feature lineage then reports + ``uses_target``, and :class:`~pretab.CrossFittedTransformer` refits the + block on every fold of a wrapped :class:`~pretab.Preprocessor`. allowed_args : iterable of str, default=() Constructor argument names the shared Preprocessor keyword arguments are filtered down to for this method. diff --git a/pretab/preprocessor.py b/pretab/preprocessor.py index a8193b8..a2c837b 100644 --- a/pretab/preprocessor.py +++ b/pretab/preprocessor.py @@ -1,13 +1,15 @@ +import copy import hashlib import json import os import time import warnings -from typing import cast +from typing import Any, cast import numpy as np from scipy import sparse as sp from sklearn.base import BaseEstimator, TransformerMixin, clone +from sklearn.utils import _safe_indexing from sklearn.utils._set_output import _get_output_config from sklearn.utils.validation import check_is_fitted @@ -15,6 +17,7 @@ from .compose.factory import build_column_transformer from .compose.feature_detection import detect_column_types, to_dataframe from .compose.inspection import ( + block_uses_target, build_feature_info, build_feature_lineage, build_transformer_summary, @@ -871,6 +874,37 @@ def estimate_memory(self, X) -> int: n_rows, n_cols = self.estimate_output_shape(X) return int(n_rows * n_cols * self._output_itemsize()) + def _cross_fit_fold(self, X, y): + """Return a copy of this fit whose target-aware blocks are refit on ``(X, y)``. + + Used by :class:`~pretab.CrossFittedTransformer` for its out-of-fold + features. Only the blocks that consume the target are refit on the fold; + column types, category vocabularies and every other block are shared + with this all-data fit, so an out-of-fold row is encoded exactly like + ``transform`` encodes it, except that no target-aware placement has seen + that row's target. Refitting the whole preprocessor per fold would + re-detect column types and relearn categories on fewer rows, shifting + integer codes and one-hot columns between the folds and ``transform``. + """ + check_is_fitted(self) + # Each block sees the fold rows exactly as the internal ColumnTransformer + # routed the training rows to it in ``fit``. + X = to_dataframe(X) + column_transformer = copy.copy(self.column_transformer_) + column_transformer.transformers_ = [ + ( + name, + cast(Any, clone(transformer)).fit(_safe_indexing(X, columns, axis=1), y) + if name != "remainder" and block_uses_target(transformer) + else transformer, + columns, + ) + for name, transformer, columns in self.column_transformer_.transformers_ + ] + fold = copy.copy(self) + fold.column_transformer_ = column_transformer + return fold + def _reject_missing(self, X) -> None: """Raise when ``missing_policy="error"`` but ``X`` contains missing values.""" na_columns = [col for col in X.columns if X[col].isna().any()] diff --git a/tests/core/test_cross_fitted.py b/tests/core/test_cross_fitted.py index 723d971..c3843c8 100644 --- a/tests/core/test_cross_fitted.py +++ b/tests/core/test_cross_fitted.py @@ -6,6 +6,7 @@ """ import warnings +from typing import cast import numpy as np import pytest @@ -104,3 +105,168 @@ def test_invalid_task_rejected(data): X, y = data with pytest.raises(InvalidParamError, match="task"): CrossFittedTransformer(PLETransformer(), task="classificaton").fit_transform(X, y) + + +# --- a wrapped Preprocessor keeps one encoding across folds (issue #79) ------------- + + +@pytest.fixture +def array_with_rare_category(): + rng = np.random.default_rng(0) + X = np.column_stack( + [ + rng.integers(0, 100_000, 300), # many distinct integers: numerical + rng.integers(1, 9, 300), # unique ratio just below the default cat_cutoff + rng.choice([1, 2, 3], 300), # category codes + ] + ) + X[7, 2] = 0 # a category some folds never see, sorted before the others + y = (X[:, 0] / 50_000.0) ** 2 + (X[:, 2] == 2) + rng.normal(scale=0.1, size=300) + return X, y + + +def _split_columns(cross_fitted, prefix): + names = list(cross_fitted.get_feature_names_out()) + return [i for i, name in enumerate(names) if name.startswith(prefix)] + + +def test_wrapped_preprocessor_out_of_fold_codes_match_transform(array_with_rare_category): + """Each fold used to refit the whole Preprocessor: category codes shifted when a + category was missing from a fold, and column types were re-detected on fewer rows.""" + from pretab import Preprocessor + + X, y = array_with_rare_category + cross_fitted = CrossFittedTransformer(Preprocessor(output_dim=4, random_state=0), n_folds=5, random_state=0) + out = cross_fitted.fit_transform(X, y) + full = np.asarray(cross_fitted.transform(X)) + + unsupervised = _split_columns(cross_fitted, "cat_") + assert len(unsupervised) == 2 # both integer-coded columns are categorical on the full data + np.testing.assert_array_equal(out[:, unsupervised], full[:, unsupervised]) + np.testing.assert_array_equal(np.unique(out[:, unsupervised[1]]), [1.0, 2.0, 3.0, 4.0]) + + +def test_wrapped_preprocessor_target_aware_blocks_stay_out_of_fold(array_with_rare_category): + """Only the target-aware block is refit, on each fold's training rows.""" + import pandas as pd + from sklearn.base import clone + + from pretab import Preprocessor + + X, y = array_with_rare_category + cross_fitted = CrossFittedTransformer(Preprocessor(output_dim=4, random_state=0), n_folds=5, random_state=0) + out = cross_fitted.fit_transform(X, y) + supervised = _split_columns(cross_fitted, "num_feature_0_ple") + assert len(supervised) == 4 + + wrapped = cast(Preprocessor, cross_fitted.estimator_) + block = wrapped.column_transformer_.named_transformers_["num_feature_0"] + column = pd.DataFrame({"feature_0": X[:, 0]}) + for train_idx, test_idx in KFold(n_splits=5, shuffle=True, random_state=0).split(X): + fold_block = clone(block).fit(column.iloc[train_idx], y[train_idx]) + np.testing.assert_allclose(out[np.ix_(test_idx, supervised)], fold_block.transform(column.iloc[test_idx])) + assert not np.allclose(out[:, supervised], np.asarray(cross_fitted.transform(X))[:, supervised]) + + +def test_wrapped_preprocessor_with_one_hot_keeps_a_fixed_width(array_with_rare_category): + from pretab import Preprocessor + + X, y = array_with_rare_category + wrapped = Preprocessor(categorical_method="one-hot", output_dim=4, random_state=0) + cross_fitted = CrossFittedTransformer(wrapped, n_folds=5, random_state=0) + out = cross_fitted.fit_transform(X, y) + one_hot = _split_columns(cross_fitted, "cat_") + + assert out.shape == (len(X), len(cross_fitted.get_feature_names_out())) + np.testing.assert_array_equal(out[:, one_hot], np.asarray(cross_fitted.transform(X))[:, one_hot]) + + +def test_wrapped_unsupervised_preprocessor_matches_transform(array_with_rare_category): + from pretab import Preprocessor + + X, y = array_with_rare_category + wrapped = Preprocessor(numerical_method="minmax", target_aware=False, placement_strategy="quantile") + cross_fitted = CrossFittedTransformer(wrapped, n_folds=3, random_state=0) + np.testing.assert_array_equal(cross_fitted.fit_transform(X, y), np.asarray(cross_fitted.transform(X))) + + +def test_wrapped_preprocessor_keeps_the_all_data_column_types(): + """25 integer levels are categorical on 1000 rows (25/1000 < cat_cutoff) but were + re-detected as numerical on each 800-row fold, so the out-of-fold features held + min-max values where transform returned category codes.""" + from pretab import Preprocessor + + rng = np.random.default_rng(0) + X = rng.integers(0, 25, (1000, 1)) + cross_fitted = CrossFittedTransformer(Preprocessor(numerical_method="minmax"), n_folds=5, random_state=0) + out = cross_fitted.fit_transform(X, rng.normal(size=1000)) + + assert cast(Preprocessor, cross_fitted.estimator_).categorical_features_ == ["feature_0"] + np.testing.assert_array_equal(out, np.asarray(cross_fitted.transform(X))) + np.testing.assert_array_equal(np.unique(out), np.arange(1, 26)) + + +def _one_missing_value(X): + X[7, 0] = np.nan # a single missing value: the fold holding out row 7 trains on none + return X + + +@pytest.mark.parametrize( + ("X", "method"), + [ + (_one_missing_value(np.random.default_rng(0).normal(size=(200, 2))), {"numerical_method": "minmax"}), + (_one_missing_value(np.random.default_rng(0).normal(size=(200, 2))), {"numerical_method": "standardization"}), + ( + _one_missing_value(np.random.default_rng(0).normal(size=(200, 2))), + {"numerical_method": "bspline", "target_aware": False, "placement_strategy": "uniform"}, + ), + (_one_missing_value(np.random.default_rng(0).choice(["a", "b", "c"], (200, 1)).astype(object)), {}), + ( + _one_missing_value(np.random.default_rng(0).choice(["a", "b", "c"], (200, 1)).astype(object)), + {"categorical_method": "one-hot"}, + ), + ], + ids=["minmax", "standardization", "bspline-unsupervised", "categorical-int", "categorical-one-hot"], +) +def test_wrapped_preprocessor_shares_indicator_blocks_that_do_not_use_y(X, method): + """Blocks that do not use y keep their all-data fit, missing indicator included: each + fold used to refit them, and a fold whose training rows hold no missing value + dropped the indicator column.""" + from pretab import Preprocessor + + y = np.random.default_rng(1).normal(size=len(X)) + wrapped = Preprocessor(add_missing_indicator=True, output_dim=6, random_state=0, **method) + cross_fitted = CrossFittedTransformer(wrapped, n_folds=5, random_state=0) + out = cross_fitted.fit_transform(X, y) + assert out.shape[1] == len(cross_fitted.get_feature_names_out()) > X.shape[1] # indicator columns kept + np.testing.assert_array_equal(out, np.asarray(cross_fitted.transform(X))) + + +def test_wrapped_preprocessor_reads_the_target_use_of_an_indicator_block(): + """With add_missing_indicator, a representation is fitted on its column and the + indicator; deciding whether it used y must not depend on its input names.""" + from pretab import Preprocessor + + rng = np.random.default_rng(0) + X = rng.normal(size=(200, 1)) + X[::4, 0] = np.nan # every fold's training rows hold missing values + y = np.nan_to_num(X[:, 0]) + rng.normal(scale=0.1, size=200) + wrapped = Preprocessor(numerical_method="rbf", add_missing_indicator=True, output_dim=4, random_state=0) + cross_fitted = CrossFittedTransformer(wrapped, n_folds=5, random_state=0) + out = cross_fitted.fit_transform(X, y) + + full = np.asarray(cross_fitted.transform(X)) + assert out.shape == full.shape == (200, len(cross_fitted.get_feature_names_out())) + assert not np.allclose(out, full) # refit on every fold + + +def test_width_mismatch_message_does_not_blame_adaptive_sizing_only(): + from sklearn.preprocessing import OneHotEncoder + + x = np.array(["a"] * 99 + ["b"]).reshape(-1, 1) # the fold holding out "b" never sees it + y = np.arange(100.0) + cross_fitted = CrossFittedTransformer( + OneHotEncoder(handle_unknown="ignore", sparse_output=False), n_folds=5, random_state=0 + ) + with pytest.raises(IncompatibleParamsError, match="depends on the training rows"): + cross_fitted.fit_transform(x, y) diff --git a/tests/extension/test_registered_cross_fitting.py b/tests/extension/test_registered_cross_fitting.py new file mode 100644 index 0000000..3e8a039 --- /dev/null +++ b/tests/extension/test_registered_cross_fitting.py @@ -0,0 +1,69 @@ +"""Cross-fitting a Preprocessor that uses a registered, target-consuming class. + +Regression guard: a wrapped Preprocessor refits only its target-aware blocks per +fold, but a registered class without a ``RepresentationSpec`` (a plain +scikit-learn transformer such as ``TargetEncoder``) was never recognised as +target-aware. Its out-of-fold rows were then encoded by the all-data fit, which +had seen their own targets. +""" + +import warnings + +import numpy as np +import pandas as pd +import pytest +from sklearn.base import BaseEstimator, TransformerMixin +from sklearn.preprocessing import TargetEncoder + +from pretab import CrossFittedTransformer, Preprocessor, register_representation + + +class _MeanEncoder(TransformerMixin, BaseEstimator): + """Plain, optionally supervised encoder: category -> mean target when target-aware.""" + + def __init__(self, target_aware=True): + self.target_aware = target_aware + + def fit(self, X, y=None): + x = np.asarray(X, dtype=object)[:, 0] + if self.target_aware: + y = np.asarray(y, dtype=float) + self.mapping_ = {c: y[x == c].mean() for c in np.unique(x)} + else: + self.mapping_ = {c: float(i) for i, c in enumerate(np.unique(x))} + self.n_features_in_ = 1 + return self + + def transform(self, X): + return np.array([[self.mapping_.get(c, 0.0)] for c in np.asarray(X, dtype=object)[:, 0]]) + + def get_feature_names_out(self, input_features=None): + return np.asarray(input_features if input_features is not None else ["x0"], dtype=object) + + +@pytest.fixture +def noise_target_data(): + """An id column with four rows per id and a target of pure noise.""" + rng = np.random.default_rng(0) + X = pd.DataFrame({"x": rng.normal(size=400), "id": np.repeat([f"id{i}" for i in range(100)], 4)}) + return X, rng.normal(size=400) + + +@pytest.mark.parametrize( + ("cls", "supervision"), + [(TargetEncoder, "supervised"), (_MeanEncoder, "optional")], +) +def test_registered_target_encoder_is_refit_per_fold(noise_target_data, cls, supervision): + X, y = noise_target_data + register_representation("registered_encoder", cls, feature_kind="categorical", supervision=supervision) + pre = Preprocessor(numerical_method="minmax", categorical_method="registered_encoder", random_state=0) + + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + out_of_fold = np.asarray(CrossFittedTransformer(pre, n_folds=5, random_state=0).fit_transform(X, y)) + lineage = {item.output_feature: item.uses_target for item in pre.fit(X, y).get_feature_lineage()} + + # An encoding fit without each row's own target cannot predict pure noise; the + # all-data fit correlated at about 0.5. + assert abs(np.corrcoef(out_of_fold[:, -1], y)[0, 1]) < 0.2 + assert lineage["cat_id"] is True