Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions pretab/compose/factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -331,17 +331,21 @@ def build_column_transformer(

Numerical features are prefixed ``num_`` and categorical features ``cat_`` to
match the transformer names the Preprocessor exposes; untransformed columns
pass through via ``remainder="passthrough"``.
pass through via ``remainder="passthrough"``. Each step selects its column by
the label's string form: scikit-learn reads an integer selector as a
*position*, so the Preprocessor fits and transforms the ColumnTransformer on a
frame whose labels are strings (see
:func:`~pretab.compose.feature_detection.with_string_labels`).
"""
transformers = []
for feature in numerical_features:
method = config.method_for(feature, is_numerical=True)
pipeline = create_transformer(method, is_numerical=True, config=config)
transformers.append((f"num_{feature}", pipeline, [feature]))
transformers.append((f"num_{feature}", pipeline, [str(feature)]))
for feature in categorical_features:
method = config.method_for(feature, is_numerical=False)
pipeline = create_transformer(method, is_numerical=False, config=config)
transformers.append((f"cat_{feature}", pipeline, [feature]))
transformers.append((f"cat_{feature}", pipeline, [str(feature)]))
return ColumnTransformer(
transformers=transformers,
remainder="passthrough",
Expand Down
35 changes: 34 additions & 1 deletion pretab/compose/feature_detection.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,14 @@
sequence of delegations rather than inlining the classification heuristic.
"""

from collections import Counter

import numpy as np
import pandas as pd

from ..exceptions import PretabDataError, invalid_param_error

__all__ = ["bool_columns_as_object", "detect_column_types", "to_dataframe"]
__all__ = ["bool_columns_as_object", "detect_column_types", "to_dataframe", "with_string_labels"]


def to_dataframe(X, *, copy: bool = False) -> pd.DataFrame:
Expand Down Expand Up @@ -41,6 +43,37 @@ def to_dataframe(X, *, copy: bool = False) -> pd.DataFrame:
return X


def with_string_labels(X: pd.DataFrame) -> pd.DataFrame:
"""Return ``X`` with every column label converted to its ``str`` form.

scikit-learn's :class:`~sklearn.compose.ColumnTransformer` treats an integer
column selector as a *position*, so routing a column by an integer label (as
in ``pd.DataFrame(array)`` or ``read_csv(header=None)`` after reordering or
dropping a column) would select the wrong column. The Preprocessor therefore
fits and transforms its ColumnTransformer on string labels. ``X`` itself is
returned when every label is already a string; otherwise a shallow copy with
relabelled columns, so the caller's frame is never modified.

Raises
------
PretabDataError
If two labels share a string form (e.g. ``1`` and ``"1"``), which would
make the columns indistinguishable.
"""
if all(isinstance(label, str) for label in X.columns):
return X
labels = [str(label) for label in X.columns]
collisions = sorted(label for label, count in Counter(labels).items() if count > 1)
if collisions:
raise PretabDataError(
f"Column labels must stay unique when converted to strings; {collisions} would collide.\n"
"Fix: rename the columns so their string forms are unique."
)
relabelled = X.copy(deep=False)
relabelled.columns = pd.Index(labels)
return relabelled


def bool_columns_as_object(X: pd.DataFrame) -> pd.DataFrame:
"""Return ``X`` with its boolean columns cast to ``object``.

Expand Down
68 changes: 59 additions & 9 deletions pretab/preprocessor.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

from .compose.config import PreprocessorConfig
from .compose.factory import build_column_transformer
from .compose.feature_detection import bool_columns_as_object, detect_column_types, to_dataframe
from .compose.feature_detection import bool_columns_as_object, detect_column_types, to_dataframe, with_string_labels
from .compose.inspection import (
build_feature_info,
build_feature_lineage,
Expand Down Expand Up @@ -309,7 +309,9 @@ class Preprocessor(TransformerMixin, BaseEstimator):
----------
column_transformer\_ : ColumnTransformer
The internal scikit-learn column transformer that handles feature-wise preprocessing.
Set when the preprocessor is fitted.
Set when the preprocessor is fitted. It is fitted on the string form of every column
label (scikit-learn reads an integer column selector as a position), so a column
labelled ``1`` appears there as ``"1"``.
n_features_in\_ : int
Number of input features seen during ``fit``.
total_output_dim\_ : int
Expand Down Expand Up @@ -491,6 +493,13 @@ def fit(self, X, y=None, embeddings=None):
-------
self : Preprocessor
Fitted instance of the preprocessor.

Raises
------
PretabDataError
If ``X`` has a duplicate column label, or two column labels share a string
form (e.g. ``1`` and ``"1"``): every column is routed by the string form of
its label, so such columns could not be told apart.
"""

if self.is_frozen():
Expand Down Expand Up @@ -626,7 +635,10 @@ def transform(self, X, embeddings=None, return_array=None):
Parameters
----------
X : pandas.DataFrame, numpy.ndarray, or dict
Input features to transform.
Input features to transform. DataFrame and dict columns are matched by
label, integer labels included. A NumPy array is matched by position
after a fit on a frame with non-string labels (such as
``pd.DataFrame(array)``).
embeddings : np.ndarray or list of np.ndarray, optional
External embeddings to attach to dictionary output. Required when
embeddings were supplied during ``fit`` and unsupported when the
Expand All @@ -650,7 +662,10 @@ def transform(self, X, embeddings=None, return_array=None):

check_is_fitted(self)

is_array = isinstance(X, np.ndarray)
X = to_dataframe(X, copy=True)
if is_array:
X = self._label_array_by_position(X)

if self.missing_policy == "error":
self._reject_missing(X)
Expand Down Expand Up @@ -769,8 +784,11 @@ def get_feature_names_out(self, input_features=None):

Parameters
----------
input_features : array-like of str or None, default=None
input_features : array-like or None, default=None
Input feature names. Passed through to the underlying column transformer.
Each name is compared by its string form, the form the column transformer is
fitted on, so the labels seen at fit are accepted as given (e.g. ``[0, 1]``
after fitting on ``pd.DataFrame(array)``).

Returns
-------
Expand All @@ -779,6 +797,8 @@ def get_feature_names_out(self, input_features=None):
"""

check_is_fitted(self)
if input_features is not None:
input_features = [str(feature) for feature in input_features]
raw_names = self.column_transformer_.get_feature_names_out(input_features)
return np.array(clean_feature_names(self.column_transformer_, raw_names))

Expand Down Expand Up @@ -823,6 +843,7 @@ def output_dims_(self) -> dict:
check_is_fitted(self)
column_transformer = self.column_transformer_
output_indices = column_transformer.output_indices_
input_labels = self._input_labels()
dims: dict = {}
for name, _transformer, columns in column_transformer.transformers_:
span = output_indices[name]
Expand All @@ -835,19 +856,45 @@ def output_dims_(self) -> dict:
feature = full.split("__", 1)[-1] if "__" in full else full
dims[feature] = dims.get(feature, 0) + 1
else:
dims[columns[0]] = width
dims[input_labels.get(columns[0], columns[0])] = width
return dims

@staticmethod
def _column_transformer_input(X):
"""Adapt a frame to what the internal ColumnTransformer expects.

Boolean columns become ``object`` columns, because the categorical
imputer rejects ``bool`` (see
:func:`~pretab.compose.feature_detection.bool_columns_as_object`). The
Column labels become strings, because scikit-learn reads an integer
column selector as a position (see
:func:`~pretab.compose.feature_detection.with_string_labels`). Boolean
columns become ``object`` columns, because the categorical imputer rejects
``bool`` (see :func:`~pretab.compose.feature_detection.bool_columns_as_object`). The
caller's frame is never modified.
"""
return bool_columns_as_object(X)
return bool_columns_as_object(with_string_labels(X))

def _input_labels(self) -> dict:
"""Map each ColumnTransformer column back to the label it has in the input.

The ColumnTransformer is fitted on the string form of every label (see
:func:`~pretab.compose.feature_detection.with_string_labels`), so a
non-string label such as ``1`` appears there as ``"1"``.
"""
return {str(label): label for label in (*self.numerical_features_, *self.categorical_features_)}

def _label_array_by_position(self, X):
"""Give the columns of array input the labels seen at fit, by position.

:func:`~pretab.compose.feature_detection.to_dataframe` names array columns
``feature_0``, ``feature_1``, .... After a fit on a frame with a non-string
label, such as ``pd.DataFrame(array)``, an array carries no label to route
by, so its columns are matched by position, as scikit-learn does for a
frame without string feature names. Any other input is returned unchanged.
"""
fitted = [str(label) for label in getattr(self.column_transformer_, "feature_names_in_", [])]
input_labels = self._input_labels()
if len(fitted) != X.shape[1] or all(isinstance(input_labels.get(label), str) for label in fitted):
return X
return X.set_axis(fitted, axis=1)

def _output_itemsize(self) -> int:
"""Bytes per element of the dense transformed array.
Expand Down Expand Up @@ -995,6 +1042,9 @@ def get_feature_info(self, verbose=True):
embeddings=self.embeddings_,
embedding_dimensions=self.embedding_dimensions_,
)
input_labels = self._input_labels()
numerical_feature_info = {input_labels.get(key, key): info for key, info in numerical_feature_info.items()}
categorical_feature_info = {input_labels.get(key, key): info for key, info in categorical_feature_info.items()}

if verbose:
# Render at INFO without lowering a more verbose level set earlier.
Expand Down
16 changes: 15 additions & 1 deletion tests/compose/test_feature_detection.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,12 @@
import pandas as pd
import pytest

from pretab.compose.feature_detection import bool_columns_as_object, detect_column_types, to_dataframe
from pretab.compose.feature_detection import (
bool_columns_as_object,
detect_column_types,
to_dataframe,
with_string_labels,
)
from pretab.exceptions import InvalidParamError, PretabDataError


Expand Down Expand Up @@ -107,6 +112,15 @@ def test_invalid_cat_cutoff_type_raises():
detect_column_types(df, cat_cutoff="bad", treat_all_integers_as_numerical=False)


def test_with_string_labels_relabels_without_touching_the_input():
frame = pd.DataFrame({1: [1.0], "b": [2.0]})
relabelled = with_string_labels(frame)
assert list(relabelled.columns) == ["1", "b"]
assert list(frame.columns) == [1, "b"]
string_frame = pd.DataFrame({"a": [1.0]})
assert with_string_labels(string_frame) is string_frame


def test_bool_columns_as_object_casts_only_boolean_columns():
frame = pd.DataFrame({"x": [0.5, 1.5], "flag": [True, False], "nullable": pd.array([True, None], dtype="boolean")})
cast = bool_columns_as_object(frame)
Expand Down
129 changes: 129 additions & 0 deletions tests/integration/test_column_labels.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
"""Column labels the ColumnTransformer cannot take verbatim.

Regression guards for issue #55: integer labels were read as *positions* by
scikit-learn's ColumnTransformer, silently swapping columns.
"""

from typing import cast

import numpy as np
import pandas as pd
import pytest

from pretab import Preprocessor
from pretab.exceptions import PretabDataError


@pytest.fixture
def int_labelled():
# Labels are a permutation of the positions: label 1 sits at position 0.
return pd.DataFrame({1: [10.0, 20, 30, 40, 50, 60], 0: [1.0, 2, 3, 4, 5, 6]})


def _passthrough(**kwargs):
return Preprocessor(numerical_method="none", scaling="none", output_structure="blocks", **kwargs)


def _blocks(out):
return cast("dict[str, np.ndarray]", out)


def test_integer_labels_route_each_column_to_its_own_block(int_labelled):
out = _blocks(_passthrough().fit_transform(int_labelled))
np.testing.assert_array_equal(out["num_1"].ravel(), int_labelled[1].to_numpy())
np.testing.assert_array_equal(out["num_0"].ravel(), int_labelled[0].to_numpy())
# The ColumnTransformer works on a relabelled copy; the caller's frame keeps its labels.
assert list(int_labelled.columns) == [1, 0]


def test_feature_preprocessing_with_integer_keys_targets_the_labelled_column(int_labelled):
pre = Preprocessor(feature_preprocessing={1: "minmax", 0: "none"}, scaling="none", output_structure="blocks")
out = _blocks(pre.fit_transform(int_labelled))
np.testing.assert_allclose(out["num_1"].ravel(), np.linspace(0.0, 1.0, 6))
np.testing.assert_array_equal(out["num_0"].ravel(), int_labelled[0].to_numpy())


def test_non_contiguous_integer_labels_fit_and_transform():
frame = pd.DataFrame(np.random.default_rng(0).normal(size=(20, 3))).drop(columns=[0])
pre = Preprocessor(numerical_method="minmax").fit(frame)
assert pre.get_feature_names_out().tolist() == ["num_1", "num_2"]
out = np.asarray(pre.transform(frame))
assert out.shape == (20, 2)
np.testing.assert_allclose(out[:, 0], (frame[1] - frame[1].min()) / np.ptp(frame[1]))


def test_integer_labels_of_different_column_types_reach_their_own_pipeline():
# Routed by position, the string column landed in the numerical pipeline and fit
# failed with "Cannot use median strategy with non-numeric data".
frame = pd.DataFrame({1: list("abcabc"), 0: [1.0, 2, 3, 4, 5, 6]})
pre = Preprocessor(numerical_method="minmax", scaling="none", categorical_method="int").fit(frame)
assert pre.get_feature_names_out().tolist() == ["num_0", "cat_1"]
out = np.asarray(pre.transform(frame), dtype=float)
np.testing.assert_allclose(out[:, 0], np.linspace(0.0, 1.0, 6))
np.testing.assert_array_equal(out[:, 1], [1.0, 2.0, 3.0, 1.0, 2.0, 3.0])
default = Preprocessor().fit(frame, np.arange(6.0))
assert (default.numerical_features_, default.categorical_features_) == ([0], [1])
np.testing.assert_array_equal(np.asarray(default.transform(frame))[:, -1], [1.0, 2.0, 3.0, 1.0, 2.0, 3.0])


@pytest.mark.parametrize(
("dtype", "missing_policy", "expected_codes"),
[
("bool", "impute", [2.0, 1.0, 2.0, 1.0, 2.0, 2.0]),
("boolean", "impute", [2.0, 1.0, 2.0, 1.0, 2.0, 2.0]),
("boolean", "propagate", [2.0, 1.0, 0.0, 1.0, 2.0, 2.0]),
],
)
def test_boolean_columns_with_integer_labels_keep_routing_and_encoding(dtype, missing_policy, expected_codes):
frame = pd.DataFrame({1: pd.array([True, False, True, False, True, True], dtype=dtype), 0: [1.0, 2, 3, 4, 5, 6]})
if dtype == "boolean":
frame.loc[2, 1] = pd.NA
original = frame.copy(deep=True)
pre = _passthrough(categorical_method="int", missing_policy=missing_policy).fit(frame)
assert pre.numerical_features_ == [0]
assert pre.categorical_features_ == [1]
assert pre.output_dims_ == {0: 1, 1: 1}
for data in (frame, frame[[0, 1]], frame.to_numpy(na_value=np.nan)):
out = _blocks(pre.transform(data))
np.testing.assert_array_equal(out["num_0"].ravel(), frame[0].to_numpy())
np.testing.assert_array_equal(out["cat_1"].ravel(), expected_codes)
pd.testing.assert_frame_equal(frame, original)


def test_integer_labels_are_matched_by_label_at_transform(int_labelled):
pre = _passthrough(output_format="dense").fit(int_labelled)
reordered = int_labelled[[0, 1]]
np.testing.assert_array_equal(
_blocks(pre.transform(reordered))["num_1"], _blocks(pre.transform(int_labelled))["num_1"]
)


def test_array_input_after_an_integer_labelled_fit_is_matched_by_position(int_labelled):
pre = _passthrough().fit(int_labelled)
out = _blocks(pre.transform(int_labelled.to_numpy()))
np.testing.assert_array_equal(out["num_1"].ravel(), int_labelled[1].to_numpy())
np.testing.assert_array_equal(out["num_0"].ravel(), int_labelled[0].to_numpy())


def test_integer_label_metadata_uses_the_input_labels(int_labelled):
pre = Preprocessor(numerical_method="minmax").fit(int_labelled)
assert pre.get_feature_names_out().tolist() == ["num_1", "num_0"]
assert pre.output_dims_ == {1: 1, 0: 1}
numerical, _, _ = pre.get_feature_info(verbose=False)
assert list(numerical) == [1, 0]
lineage = [(record.output_feature, record.source_features) for record in pre.get_feature_lineage()]
assert lineage == [("num_1", ("1",)), ("num_0", ("0",))]


def test_labels_colliding_as_strings_are_rejected():
frame = pd.DataFrame({1: [1.0, 2.0, 3.0], "1": [4.0, 5.0, 6.0]})
with pytest.raises(PretabDataError, match="unique when converted to strings"):
Preprocessor(numerical_method="minmax").fit(frame)


def test_get_feature_names_out_accepts_the_integer_labels_seen_at_fit():
"""The ColumnTransformer is fitted on string labels, so ``input_features`` given as
the original integer labels must still be accepted (it already was before)."""
X = pd.DataFrame(np.random.default_rng(0).normal(size=(40, 2)))
pre = Preprocessor(numerical_method="minmax").fit(X)
np.testing.assert_array_equal(pre.get_feature_names_out([0, 1]), pre.get_feature_names_out())
Loading