diff --git a/pretab/compose/factory.py b/pretab/compose/factory.py index b8bb98b..3b9df5e 100644 --- a/pretab/compose/factory.py +++ b/pretab/compose/factory.py @@ -331,17 +331,21 @@ def build_column_transformer( Numerical features are prefixed ``num_`` and categorical features ``cat_`` to match the transformer names the Preprocessor exposes; untransformed columns - pass through via ``remainder="passthrough"``. + pass through via ``remainder="passthrough"``. Each step selects its column by + the label's string form: scikit-learn reads an integer selector as a + *position*, so the Preprocessor fits and transforms the ColumnTransformer on a + frame whose labels are strings (see + :func:`~pretab.compose.feature_detection.with_string_labels`). """ transformers = [] for feature in numerical_features: method = config.method_for(feature, is_numerical=True) pipeline = create_transformer(method, is_numerical=True, config=config) - transformers.append((f"num_{feature}", pipeline, [feature])) + transformers.append((f"num_{feature}", pipeline, [str(feature)])) for feature in categorical_features: method = config.method_for(feature, is_numerical=False) pipeline = create_transformer(method, is_numerical=False, config=config) - transformers.append((f"cat_{feature}", pipeline, [feature])) + transformers.append((f"cat_{feature}", pipeline, [str(feature)])) return ColumnTransformer( transformers=transformers, remainder="passthrough", diff --git a/pretab/compose/feature_detection.py b/pretab/compose/feature_detection.py index edbdd54..7ee5d04 100644 --- a/pretab/compose/feature_detection.py +++ b/pretab/compose/feature_detection.py @@ -5,12 +5,14 @@ sequence of delegations rather than inlining the classification heuristic. """ +from collections import Counter + import numpy as np import pandas as pd from ..exceptions import PretabDataError, invalid_param_error -__all__ = ["bool_columns_as_object", "detect_column_types", "to_dataframe"] +__all__ = ["bool_columns_as_object", "detect_column_types", "to_dataframe", "with_string_labels"] def to_dataframe(X, *, copy: bool = False) -> pd.DataFrame: @@ -41,6 +43,37 @@ def to_dataframe(X, *, copy: bool = False) -> pd.DataFrame: return X +def with_string_labels(X: pd.DataFrame) -> pd.DataFrame: + """Return ``X`` with every column label converted to its ``str`` form. + + scikit-learn's :class:`~sklearn.compose.ColumnTransformer` treats an integer + column selector as a *position*, so routing a column by an integer label (as + in ``pd.DataFrame(array)`` or ``read_csv(header=None)`` after reordering or + dropping a column) would select the wrong column. The Preprocessor therefore + fits and transforms its ColumnTransformer on string labels. ``X`` itself is + returned when every label is already a string; otherwise a shallow copy with + relabelled columns, so the caller's frame is never modified. + + Raises + ------ + PretabDataError + If two labels share a string form (e.g. ``1`` and ``"1"``), which would + make the columns indistinguishable. + """ + if all(isinstance(label, str) for label in X.columns): + return X + labels = [str(label) for label in X.columns] + collisions = sorted(label for label, count in Counter(labels).items() if count > 1) + if collisions: + raise PretabDataError( + f"Column labels must stay unique when converted to strings; {collisions} would collide.\n" + "Fix: rename the columns so their string forms are unique." + ) + relabelled = X.copy(deep=False) + relabelled.columns = pd.Index(labels) + return relabelled + + def bool_columns_as_object(X: pd.DataFrame) -> pd.DataFrame: """Return ``X`` with its boolean columns cast to ``object``. diff --git a/pretab/preprocessor.py b/pretab/preprocessor.py index 4332544..8680b13 100644 --- a/pretab/preprocessor.py +++ b/pretab/preprocessor.py @@ -14,7 +14,7 @@ from .compose.config import PreprocessorConfig from .compose.factory import build_column_transformer -from .compose.feature_detection import bool_columns_as_object, detect_column_types, to_dataframe +from .compose.feature_detection import bool_columns_as_object, detect_column_types, to_dataframe, with_string_labels from .compose.inspection import ( build_feature_info, build_feature_lineage, @@ -309,7 +309,9 @@ class Preprocessor(TransformerMixin, BaseEstimator): ---------- column_transformer\_ : ColumnTransformer The internal scikit-learn column transformer that handles feature-wise preprocessing. - Set when the preprocessor is fitted. + Set when the preprocessor is fitted. It is fitted on the string form of every column + label (scikit-learn reads an integer column selector as a position), so a column + labelled ``1`` appears there as ``"1"``. n_features_in\_ : int Number of input features seen during ``fit``. total_output_dim\_ : int @@ -491,6 +493,13 @@ def fit(self, X, y=None, embeddings=None): ------- self : Preprocessor Fitted instance of the preprocessor. + + Raises + ------ + PretabDataError + If ``X`` has a duplicate column label, or two column labels share a string + form (e.g. ``1`` and ``"1"``): every column is routed by the string form of + its label, so such columns could not be told apart. """ if self.is_frozen(): @@ -626,7 +635,10 @@ def transform(self, X, embeddings=None, return_array=None): Parameters ---------- X : pandas.DataFrame, numpy.ndarray, or dict - Input features to transform. + Input features to transform. DataFrame and dict columns are matched by + label, integer labels included. A NumPy array is matched by position + after a fit on a frame with non-string labels (such as + ``pd.DataFrame(array)``). embeddings : np.ndarray or list of np.ndarray, optional External embeddings to attach to dictionary output. Required when embeddings were supplied during ``fit`` and unsupported when the @@ -650,7 +662,10 @@ def transform(self, X, embeddings=None, return_array=None): check_is_fitted(self) + is_array = isinstance(X, np.ndarray) X = to_dataframe(X, copy=True) + if is_array: + X = self._label_array_by_position(X) if self.missing_policy == "error": self._reject_missing(X) @@ -769,8 +784,11 @@ def get_feature_names_out(self, input_features=None): Parameters ---------- - input_features : array-like of str or None, default=None + input_features : array-like or None, default=None Input feature names. Passed through to the underlying column transformer. + Each name is compared by its string form, the form the column transformer is + fitted on, so the labels seen at fit are accepted as given (e.g. ``[0, 1]`` + after fitting on ``pd.DataFrame(array)``). Returns ------- @@ -779,6 +797,8 @@ def get_feature_names_out(self, input_features=None): """ check_is_fitted(self) + if input_features is not None: + input_features = [str(feature) for feature in input_features] raw_names = self.column_transformer_.get_feature_names_out(input_features) return np.array(clean_feature_names(self.column_transformer_, raw_names)) @@ -823,6 +843,7 @@ def output_dims_(self) -> dict: check_is_fitted(self) column_transformer = self.column_transformer_ output_indices = column_transformer.output_indices_ + input_labels = self._input_labels() dims: dict = {} for name, _transformer, columns in column_transformer.transformers_: span = output_indices[name] @@ -835,19 +856,45 @@ def output_dims_(self) -> dict: feature = full.split("__", 1)[-1] if "__" in full else full dims[feature] = dims.get(feature, 0) + 1 else: - dims[columns[0]] = width + dims[input_labels.get(columns[0], columns[0])] = width return dims @staticmethod def _column_transformer_input(X): """Adapt a frame to what the internal ColumnTransformer expects. - Boolean columns become ``object`` columns, because the categorical - imputer rejects ``bool`` (see - :func:`~pretab.compose.feature_detection.bool_columns_as_object`). The + Column labels become strings, because scikit-learn reads an integer + column selector as a position (see + :func:`~pretab.compose.feature_detection.with_string_labels`). Boolean + columns become ``object`` columns, because the categorical imputer rejects + ``bool`` (see :func:`~pretab.compose.feature_detection.bool_columns_as_object`). The caller's frame is never modified. """ - return bool_columns_as_object(X) + return bool_columns_as_object(with_string_labels(X)) + + def _input_labels(self) -> dict: + """Map each ColumnTransformer column back to the label it has in the input. + + The ColumnTransformer is fitted on the string form of every label (see + :func:`~pretab.compose.feature_detection.with_string_labels`), so a + non-string label such as ``1`` appears there as ``"1"``. + """ + return {str(label): label for label in (*self.numerical_features_, *self.categorical_features_)} + + def _label_array_by_position(self, X): + """Give the columns of array input the labels seen at fit, by position. + + :func:`~pretab.compose.feature_detection.to_dataframe` names array columns + ``feature_0``, ``feature_1``, .... After a fit on a frame with a non-string + label, such as ``pd.DataFrame(array)``, an array carries no label to route + by, so its columns are matched by position, as scikit-learn does for a + frame without string feature names. Any other input is returned unchanged. + """ + fitted = [str(label) for label in getattr(self.column_transformer_, "feature_names_in_", [])] + input_labels = self._input_labels() + if len(fitted) != X.shape[1] or all(isinstance(input_labels.get(label), str) for label in fitted): + return X + return X.set_axis(fitted, axis=1) def _output_itemsize(self) -> int: """Bytes per element of the dense transformed array. @@ -995,6 +1042,9 @@ def get_feature_info(self, verbose=True): embeddings=self.embeddings_, embedding_dimensions=self.embedding_dimensions_, ) + input_labels = self._input_labels() + numerical_feature_info = {input_labels.get(key, key): info for key, info in numerical_feature_info.items()} + categorical_feature_info = {input_labels.get(key, key): info for key, info in categorical_feature_info.items()} if verbose: # Render at INFO without lowering a more verbose level set earlier. diff --git a/tests/compose/test_feature_detection.py b/tests/compose/test_feature_detection.py index ec98875..6fa46e2 100644 --- a/tests/compose/test_feature_detection.py +++ b/tests/compose/test_feature_detection.py @@ -4,7 +4,12 @@ import pandas as pd import pytest -from pretab.compose.feature_detection import bool_columns_as_object, detect_column_types, to_dataframe +from pretab.compose.feature_detection import ( + bool_columns_as_object, + detect_column_types, + to_dataframe, + with_string_labels, +) from pretab.exceptions import InvalidParamError, PretabDataError @@ -107,6 +112,15 @@ def test_invalid_cat_cutoff_type_raises(): detect_column_types(df, cat_cutoff="bad", treat_all_integers_as_numerical=False) +def test_with_string_labels_relabels_without_touching_the_input(): + frame = pd.DataFrame({1: [1.0], "b": [2.0]}) + relabelled = with_string_labels(frame) + assert list(relabelled.columns) == ["1", "b"] + assert list(frame.columns) == [1, "b"] + string_frame = pd.DataFrame({"a": [1.0]}) + assert with_string_labels(string_frame) is string_frame + + def test_bool_columns_as_object_casts_only_boolean_columns(): frame = pd.DataFrame({"x": [0.5, 1.5], "flag": [True, False], "nullable": pd.array([True, None], dtype="boolean")}) cast = bool_columns_as_object(frame) diff --git a/tests/integration/test_column_labels.py b/tests/integration/test_column_labels.py new file mode 100644 index 0000000..39a0699 --- /dev/null +++ b/tests/integration/test_column_labels.py @@ -0,0 +1,129 @@ +"""Column labels the ColumnTransformer cannot take verbatim. + +Regression guards for issue #55: integer labels were read as *positions* by +scikit-learn's ColumnTransformer, silently swapping columns. +""" + +from typing import cast + +import numpy as np +import pandas as pd +import pytest + +from pretab import Preprocessor +from pretab.exceptions import PretabDataError + + +@pytest.fixture +def int_labelled(): + # Labels are a permutation of the positions: label 1 sits at position 0. + return pd.DataFrame({1: [10.0, 20, 30, 40, 50, 60], 0: [1.0, 2, 3, 4, 5, 6]}) + + +def _passthrough(**kwargs): + return Preprocessor(numerical_method="none", scaling="none", output_structure="blocks", **kwargs) + + +def _blocks(out): + return cast("dict[str, np.ndarray]", out) + + +def test_integer_labels_route_each_column_to_its_own_block(int_labelled): + out = _blocks(_passthrough().fit_transform(int_labelled)) + np.testing.assert_array_equal(out["num_1"].ravel(), int_labelled[1].to_numpy()) + np.testing.assert_array_equal(out["num_0"].ravel(), int_labelled[0].to_numpy()) + # The ColumnTransformer works on a relabelled copy; the caller's frame keeps its labels. + assert list(int_labelled.columns) == [1, 0] + + +def test_feature_preprocessing_with_integer_keys_targets_the_labelled_column(int_labelled): + pre = Preprocessor(feature_preprocessing={1: "minmax", 0: "none"}, scaling="none", output_structure="blocks") + out = _blocks(pre.fit_transform(int_labelled)) + np.testing.assert_allclose(out["num_1"].ravel(), np.linspace(0.0, 1.0, 6)) + np.testing.assert_array_equal(out["num_0"].ravel(), int_labelled[0].to_numpy()) + + +def test_non_contiguous_integer_labels_fit_and_transform(): + frame = pd.DataFrame(np.random.default_rng(0).normal(size=(20, 3))).drop(columns=[0]) + pre = Preprocessor(numerical_method="minmax").fit(frame) + assert pre.get_feature_names_out().tolist() == ["num_1", "num_2"] + out = np.asarray(pre.transform(frame)) + assert out.shape == (20, 2) + np.testing.assert_allclose(out[:, 0], (frame[1] - frame[1].min()) / np.ptp(frame[1])) + + +def test_integer_labels_of_different_column_types_reach_their_own_pipeline(): + # Routed by position, the string column landed in the numerical pipeline and fit + # failed with "Cannot use median strategy with non-numeric data". + frame = pd.DataFrame({1: list("abcabc"), 0: [1.0, 2, 3, 4, 5, 6]}) + pre = Preprocessor(numerical_method="minmax", scaling="none", categorical_method="int").fit(frame) + assert pre.get_feature_names_out().tolist() == ["num_0", "cat_1"] + out = np.asarray(pre.transform(frame), dtype=float) + np.testing.assert_allclose(out[:, 0], np.linspace(0.0, 1.0, 6)) + np.testing.assert_array_equal(out[:, 1], [1.0, 2.0, 3.0, 1.0, 2.0, 3.0]) + default = Preprocessor().fit(frame, np.arange(6.0)) + assert (default.numerical_features_, default.categorical_features_) == ([0], [1]) + np.testing.assert_array_equal(np.asarray(default.transform(frame))[:, -1], [1.0, 2.0, 3.0, 1.0, 2.0, 3.0]) + + +@pytest.mark.parametrize( + ("dtype", "missing_policy", "expected_codes"), + [ + ("bool", "impute", [2.0, 1.0, 2.0, 1.0, 2.0, 2.0]), + ("boolean", "impute", [2.0, 1.0, 2.0, 1.0, 2.0, 2.0]), + ("boolean", "propagate", [2.0, 1.0, 0.0, 1.0, 2.0, 2.0]), + ], +) +def test_boolean_columns_with_integer_labels_keep_routing_and_encoding(dtype, missing_policy, expected_codes): + frame = pd.DataFrame({1: pd.array([True, False, True, False, True, True], dtype=dtype), 0: [1.0, 2, 3, 4, 5, 6]}) + if dtype == "boolean": + frame.loc[2, 1] = pd.NA + original = frame.copy(deep=True) + pre = _passthrough(categorical_method="int", missing_policy=missing_policy).fit(frame) + assert pre.numerical_features_ == [0] + assert pre.categorical_features_ == [1] + assert pre.output_dims_ == {0: 1, 1: 1} + for data in (frame, frame[[0, 1]], frame.to_numpy(na_value=np.nan)): + out = _blocks(pre.transform(data)) + np.testing.assert_array_equal(out["num_0"].ravel(), frame[0].to_numpy()) + np.testing.assert_array_equal(out["cat_1"].ravel(), expected_codes) + pd.testing.assert_frame_equal(frame, original) + + +def test_integer_labels_are_matched_by_label_at_transform(int_labelled): + pre = _passthrough(output_format="dense").fit(int_labelled) + reordered = int_labelled[[0, 1]] + np.testing.assert_array_equal( + _blocks(pre.transform(reordered))["num_1"], _blocks(pre.transform(int_labelled))["num_1"] + ) + + +def test_array_input_after_an_integer_labelled_fit_is_matched_by_position(int_labelled): + pre = _passthrough().fit(int_labelled) + out = _blocks(pre.transform(int_labelled.to_numpy())) + np.testing.assert_array_equal(out["num_1"].ravel(), int_labelled[1].to_numpy()) + np.testing.assert_array_equal(out["num_0"].ravel(), int_labelled[0].to_numpy()) + + +def test_integer_label_metadata_uses_the_input_labels(int_labelled): + pre = Preprocessor(numerical_method="minmax").fit(int_labelled) + assert pre.get_feature_names_out().tolist() == ["num_1", "num_0"] + assert pre.output_dims_ == {1: 1, 0: 1} + numerical, _, _ = pre.get_feature_info(verbose=False) + assert list(numerical) == [1, 0] + lineage = [(record.output_feature, record.source_features) for record in pre.get_feature_lineage()] + assert lineage == [("num_1", ("1",)), ("num_0", ("0",))] + + +def test_labels_colliding_as_strings_are_rejected(): + frame = pd.DataFrame({1: [1.0, 2.0, 3.0], "1": [4.0, 5.0, 6.0]}) + with pytest.raises(PretabDataError, match="unique when converted to strings"): + Preprocessor(numerical_method="minmax").fit(frame) + + +def test_get_feature_names_out_accepts_the_integer_labels_seen_at_fit(): + """The ColumnTransformer is fitted on string labels, so ``input_features`` given as + the original integer labels must still be accepted (it already was before).""" + X = pd.DataFrame(np.random.default_rng(0).normal(size=(40, 2))) + pre = Preprocessor(numerical_method="minmax").fit(X) + np.testing.assert_array_equal(pre.get_feature_names_out([0, 1]), pre.get_feature_names_out())