From 42e838b2446e6344d2c23cc1006210b0fd7ebbd8 Mon Sep 17 00:00:00 2001 From: ChrisW09 <50968720+ChrisW09@users.noreply.github.com> Date: Fri, 16 Oct 2026 14:16:22 +0200 Subject: [PATCH] fix(preprocessor): accept input_features after fitting on a NumPy array Preprocessor forwarded input_features to its ColumnTransformer, which was fitted on the synthetic feature_0, feature_1, ... names of array input and so required exactly those names. That broke Pipeline.get_feature_names_out() whenever an upstream step, such as SimpleImputer, outputs arrays. Following scikit-learn's convention, Preprocessor now records feature_names_in_ only when fitted on string-labelled columns and keeps validating input_features against it. Without it, any n_features_in_ names are accepted: each step's output names are rebuilt from its fitted transformer with the given labels (on shallow copies without their recorded feature names), so they match the names a fit on named columns would produce, e.g. num_age instead of num_feature_0. Behaviour change: a Preprocessor fitted on a DataFrame or dict with string column labels now has a feature_names_in_ attribute, which also changes the fingerprint_ of newly fitted preprocessors. Fixes #65 --- docs/core_concepts/outputs_and_inspection.md | 22 ++++++ pretab/compose/inspection.py | 72 ++++++++++++++++++-- pretab/preprocessor.py | 41 +++++++++-- tests/integration/test_preprocessor.py | 56 +++++++++++++++ 4 files changed, 183 insertions(+), 8 deletions(-) diff --git a/docs/core_concepts/outputs_and_inspection.md b/docs/core_concepts/outputs_and_inspection.md index b8b4d80..7be2841 100644 --- a/docs/core_concepts/outputs_and_inspection.md +++ b/docs/core_concepts/outputs_and_inspection.md @@ -290,6 +290,28 @@ Every representation names its output columns, and the names are stable and desc pre.get_feature_names_out() ``` +As in scikit-learn, a `Preprocessor` fitted on string-labelled columns (a `DataFrame` or a +dict) records them as `feature_names_in_`, and `get_feature_names_out(input_features)` accepts +only those names. One fitted on a NumPy array has no feature names: `get_feature_names_out` +then accepts any `n_features_in_` names and uses them in place of `feature_0`, `feature_1`, +.... This is what `Pipeline.get_feature_names_out()` relies on when an upstream step, such as +a `SimpleImputer`, outputs arrays: + +```python +import numpy as np +import pandas as pd +from sklearn.impute import SimpleImputer +from sklearn.pipeline import make_pipeline + +X = pd.DataFrame({"age": [25.0, np.nan, 47.0, 33.0], "income": [3.1e4, 5.2e4, np.nan, 4.4e4]}) +pipe = make_pipeline(SimpleImputer(), Preprocessor(numerical_method="minmax")) +list(pipe.fit(X, [0.0, 1.0, 1.0, 0.0]).get_feature_names_out()) +``` + +```text +['num_age', 'num_income'] +``` + Use `get_feature_info(verbose=True)` for a human-readable table of the resolved per-feature pipeline, output width, and category count. diff --git a/pretab/compose/inspection.py b/pretab/compose/inspection.py index 7f3b9b3..d30ca2b 100644 --- a/pretab/compose/inspection.py +++ b/pretab/compose/inspection.py @@ -7,8 +7,11 @@ :func:`build_transformer_summary` renders that metadata as an aligned table. """ +import copy +from typing import Any + import numpy as np -from sklearn.pipeline import FeatureUnion +from sklearn.pipeline import FeatureUnion, Pipeline from ..core.logging import get_logger from ..core.representation import FeatureLineage @@ -20,6 +23,7 @@ "build_feature_lineage", "build_transformer_summary", "clean_feature_names", + "feature_names_out", "get_output_slices", ] @@ -45,7 +49,59 @@ def get_output_slices(column_transformer): return slices -def clean_feature_names(column_transformer, names): +def feature_names_out(column_transformer, input_features=None): + """Return the cleaned output feature names of a fitted ColumnTransformer. + + ``input_features`` -- one name per input column, in input order -- replaces the + column labels the ColumnTransformer was fitted on (e.g. the synthetic + ``feature_i`` labels of an array input), following scikit-learn's contract + for an estimator fitted without feature names. Each step's names are rebuilt + from its fitted transformer with the new labels, so names that embed a label + anywhere (e.g. polynomial terms) are renamed consistently. + """ + fitted_labels = getattr(column_transformer, "feature_names_in_", None) + if input_features is None or fitted_labels is None: + raw_names = column_transformer.get_feature_names_out(input_features) + return clean_feature_names(column_transformer, [str(name) for name in raw_names]) + + rename = {str(label): str(name) for label, name in zip(fitted_labels, input_features, strict=True)} + raw_names = [] + for name, transformer, columns in column_transformer.transformers_: + if transformer == "drop" or len(columns) == 0: + continue + labels = [rename[str(column)] for column in columns] + if transformer == "passthrough": + inner = labels + else: + inner = _without_feature_names(transformer).get_feature_names_out(labels) + raw_names.extend(f"{name}__{inner_name}" for inner_name in inner) + return clean_feature_names(column_transformer, raw_names, rename=rename) + + +def _without_feature_names(estimator) -> Any: + """Return a shallow copy of a fitted step that accepts any input feature names. + + The ColumnTransformer hands each step a DataFrame column, so scikit-learn + estimators inside it record ``feature_names_in_`` and only accept those exact + names in ``get_feature_names_out``. Dropping the attribute on a shallow copy + (and on copies of any nested pipeline / union steps) makes them fall back to + the length check scikit-learn applies to estimators fitted without feature + names, while the fitted step itself is left untouched. + """ + if not hasattr(estimator, "get_params"): + return estimator + stripped = copy.copy(estimator) + stripped.__dict__.pop("feature_names_in_", None) + if isinstance(stripped, Pipeline): + stripped.steps = [(name, _without_feature_names(step)) for name, step in stripped.steps] + elif isinstance(stripped, FeatureUnion): + stripped.transformer_list = [ + (name, _without_feature_names(transformer)) for name, transformer in stripped.transformer_list + ] + return stripped + + +def clean_feature_names(column_transformer, names, *, rename=None): """Collapse the per-feature name that sklearn's ColumnTransformer duplicates. Each per-column step is named ``f"{kind}_{feature}"`` (see ``compose/factory.py``), @@ -53,8 +109,12 @@ def clean_feature_names(column_transformer, names): input feature name into each output column, so sklearn's default ``f"{step}__{inner}"`` naming doubles it, e.g. ``"num_age__age_bs0"``. This collapses that back to ``"num_age_bs0"``, leaving passthrough/remainder columns - and any name it cannot confidently match unchanged. + and any name it cannot confidently match unchanged. ``rename`` maps fitted + column labels to the input feature names ``names`` were built from (see + :func:`feature_names_out`); a step of a renamed column is then reported under + the new name, e.g. ``num_feature_0`` as ``num_age``. """ + rename = rename or {} step_to_feature = { name: columns[0] for name, _transformer, columns in column_transformer.transformers_ @@ -68,9 +128,13 @@ def clean_feature_names(column_transformer, names): if not sep or feature is None: cleaned.append(raw) continue + kind_prefix = step_name[: -(len(str(feature)) + 1)] if step_name.endswith(f"_{feature}") else "" + feature = rename.get(str(feature), feature) if inner_name == feature or inner_name.startswith(f"{feature}_"): - kind_prefix = step_name[: -(len(feature) + 1)] if step_name.endswith(f"_{feature}") else "" cleaned.append(f"{kind_prefix}_{inner_name}" if kind_prefix else inner_name) + elif kind_prefix: + # Equal to ``raw`` unless the column was renamed. + cleaned.append(f"{kind_prefix}_{feature}__{inner_name}") else: cleaned.append(raw) return cleaned diff --git a/pretab/preprocessor.py b/pretab/preprocessor.py index a8193b8..7d5be5a 100644 --- a/pretab/preprocessor.py +++ b/pretab/preprocessor.py @@ -18,7 +18,7 @@ build_feature_info, build_feature_lineage, build_transformer_summary, - clean_feature_names, + feature_names_out, get_output_slices, ) from .compose.output import ( @@ -308,6 +308,10 @@ class Preprocessor(TransformerMixin, BaseEstimator): Set when the preprocessor is fitted. n_features_in\_ : int Number of input features seen during ``fit``. + feature_names_in\_ : ndarray of str + Names of the input features seen during ``fit``. Defined only when every + column label of ``X`` is a string (a DataFrame or dict); a NumPy array has + no feature names. total_output_dim\_ : int Total number of output columns produced across all input features (equals the width of ``transform(..., return_array=True)``). @@ -514,6 +518,7 @@ def fit(self, X, y=None, embeddings=None): verbose=resolved["verbose"], ) + fitted_on_array = isinstance(X, np.ndarray) X = to_dataframe(X) if self.feature_preprocessing: @@ -582,6 +587,11 @@ def fit(self, X, y=None, embeddings=None): ) self.column_transformer_.fit(X, y) self.n_features_in_ = X.shape[1] + # scikit-learn convention: feature names are recorded only for named input. + if not fitted_on_array and all(isinstance(label, str) for label in X.columns): + self.feature_names_in_ = np.asarray(X.columns, dtype=object) + elif hasattr(self, "feature_names_in_"): + del self.feature_names_in_ self._enforce_output_budget(X.shape[0]) @@ -754,17 +764,40 @@ def get_feature_names_out(self, input_features=None): Parameters ---------- input_features : array-like of str or None, default=None - Input feature names. Passed through to the underlying column transformer. + Input feature names, following scikit-learn's convention. When the + preprocessor was fitted on named (string-labelled) columns they must equal + ``feature_names_in_``. When it was fitted without feature names (a NumPy + array, whose columns are named ``feature_0``, ``feature_1``, ...), any + ``n_features_in_`` names are accepted and used in place of the fitted + ones, so names propagate through a ``Pipeline`` whose upstream steps + output arrays. Returns ------- feature_names_out : numpy.ndarray of str Transformed feature names. + + Raises + ------ + ValueError + If ``input_features`` does not have ``n_features_in_`` entries, or differs + from ``feature_names_in_`` when that is defined. """ check_is_fitted(self) - raw_names = self.column_transformer_.get_feature_names_out(input_features) - return np.array(clean_feature_names(self.column_transformer_, raw_names)) + if input_features is not None: + input_features = np.asarray(input_features, dtype=object) + if len(input_features) != self.n_features_in_: + raise ValueError( + f"input_features should have length equal to number of features " + f"({self.n_features_in_}), got {len(input_features)}" + ) + fitted_names = getattr(self, "feature_names_in_", None) + if fitted_names is not None: + if not np.array_equal(fitted_names, input_features): + raise ValueError("input_features is not equal to feature_names_in_") + input_features = None + return np.array(feature_names_out(self.column_transformer_, input_features)) def get_feature_lineage(self): """Return per-output-column provenance for the fitted preprocessor. diff --git a/tests/integration/test_preprocessor.py b/tests/integration/test_preprocessor.py index df9d224..512df47 100644 --- a/tests/integration/test_preprocessor.py +++ b/tests/integration/test_preprocessor.py @@ -377,3 +377,59 @@ def test_unknown_feature_preprocessing_key_raises(sample_data): X, y = sample_data with pytest.raises(InvalidParamError, match="feature_preprocessing"): Preprocessor(feature_preprocessing={"nnum1": "minmax"}).fit(X, y) + + +# --- get_feature_names_out(input_features) (issue #65) ---------------------------- + + +@pytest.fixture +def named_numeric(): + rng = np.random.default_rng(0) + return pd.DataFrame({"age": rng.normal(40, 10, 60), "income": rng.normal(50, 10, 60)}), rng.normal(size=60) + + +@pytest.mark.parametrize( + "options", + [ + {"numerical_method": "minmax"}, + {"numerical_method": "polynomial"}, + {"numerical_method": "bspline", "output_dim": 5, "missing_policy": "separate_state"}, + {"numerical_method": "ple", "output_dim": 3, "add_missing_indicator": True}, + ], +) +def test_array_fit_uses_the_given_input_features(named_numeric, options): + """Regression guard for issue #65: input_features had to equal the synthetic names.""" + X, y = named_numeric + X = X.copy() + X.iloc[:3, 0] = np.nan + from_array = Preprocessor(**options).fit(X.to_numpy(), y).get_feature_names_out(["age", "income"]) + from_frame = Preprocessor(**options).fit(X, y).get_feature_names_out() + assert from_array.tolist() == from_frame.tolist() + + +def test_array_fit_rejects_input_features_of_the_wrong_length(named_numeric): + X, y = named_numeric + pre = Preprocessor(numerical_method="minmax").fit(X.to_numpy(), y) + with pytest.raises(ValueError, match="length equal to number of features"): + pre.get_feature_names_out(["age"]) + + +def test_named_fit_records_and_validates_feature_names_in(named_numeric): + X, y = named_numeric + pre = Preprocessor(numerical_method="minmax").fit(X, y) + assert pre.feature_names_in_.tolist() == ["age", "income"] + assert pre.get_feature_names_out(["age", "income"]).tolist() == ["num_age", "num_income"] + with pytest.raises(ValueError, match="not equal to feature_names_in_"): + pre.get_feature_names_out(["a", "b"]) + assert not hasattr(pre.fit(X.to_numpy(), y), "feature_names_in_") + + +def test_pipeline_feature_names_propagate_through_an_array_step(named_numeric): + from sklearn.impute import SimpleImputer + from sklearn.pipeline import make_pipeline + + X, y = named_numeric + pipe = make_pipeline(SimpleImputer(), Preprocessor(numerical_method="minmax")).fit(X, y) + names = pipe.get_feature_names_out() + assert names is not None + assert names.tolist() == ["num_age", "num_income"]