Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions docs/core_concepts/target_awareness.md
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,12 @@ Any other keyword argument to `fit`, such as `sample_weight`, is passed to the e
`Preprocessor` and the scorer do not see it, so placement and validation scores are
unweighted.

The search takes the estimator type of the estimator it wraps. Around a classifier it is a
classifier itself and delegates `classes_`, `predict_proba`, `predict_log_proba` and
`decision_function` (whichever the estimator provides) to the refit `best_estimator_`. It can
therefore be evaluated as a whole in an outer, stratified cross-validation, for example with
`cross_val_score(search, X, y_class, scoring="roc_auc")`.

```{note}
This is deliberately narrow: it only searches the single `numerical_method` axis with one
global method for every numerical column, not a per-column `feature_preprocessing` search.
Expand Down
91 changes: 87 additions & 4 deletions pretab/compose/search.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,15 @@
"""

from collections.abc import Callable
from copy import deepcopy
from typing import cast

import numpy as np
from sklearn.base import BaseEstimator, clone, is_classifier
from sklearn.metrics import check_scoring
from sklearn.model_selection import BaseCrossValidator, check_cv
from sklearn.utils import get_tags
from sklearn.utils.metaestimators import available_if
from sklearn.utils.validation import _check_method_params, check_is_fitted

from ..core._typing import PredictorLike
Expand All @@ -32,9 +35,30 @@ def _row_subset(data, idx):
return np.asarray(data)[idx]


def _estimator_has(attr):
"""Check whether ``attr`` can be delegated to the downstream estimator.

As in scikit-learn's search estimators, the refit ``best_estimator_`` is
checked after fit and the unfitted ``estimator`` before, so ``hasattr`` on an
unfitted search already reflects the estimator it wraps.
"""

def check(self):
# getattr raises AttributeError when the estimator does not provide attr.
getattr(self.best_estimator_ if hasattr(self, "best_estimator_") else self.estimator, attr)
return True

return check


class RepresentationSearchCV(BaseEstimator):
"""Select the best numerical representation by cross-validation.

The search takes the estimator type of ``estimator``: wrapping a classifier
makes it a classifier (``is_classifier``, stratified outer cross-validation),
and ``predict_proba``, ``predict_log_proba`` and ``decision_function`` are
available whenever ``estimator`` provides them.

Parameters
----------
estimator : estimator
Expand Down Expand Up @@ -68,6 +92,13 @@ class RepresentationSearchCV(BaseEstimator):
Preprocessor for ``best_method_`` refit on all data.
best_estimator_ : estimator
Estimator refit on the best representation of all data.
classes_ : ndarray of shape (n_classes,)
Class labels of ``best_estimator_``. Only available for a classifier.
n_features_in_ : int
Number of features seen during :meth:`fit`.
feature_names_in_ : ndarray of shape (`n_features_in_`,)
Names of the features seen during :meth:`fit`. Only defined when ``X`` has
string column names.
"""

def __init__(self, estimator, methods, *, cv=5, scoring=None, preprocessor_params=None, random_state=None):
Expand Down Expand Up @@ -166,17 +197,69 @@ def fit(self, X, y=None, *, groups=None, **fit_params):
self.best_preprocessor_ = self._make_preprocessor(best_method)
x_all = self.best_preprocessor_.fit_transform(X, y_arr, return_array=True)
self.best_estimator_ = cast(PredictorLike, clone(self.estimator)).fit(x_all, y_arr, **fit_params)
self.n_features_in_ = self.best_preprocessor_.n_features_in_
# scikit-learn convention: feature names are recorded only for input whose
# column labels are all strings.
columns = getattr(X, "columns", None)
if columns is not None and all(isinstance(label, str) for label in columns):
self.feature_names_in_ = np.asarray(columns, dtype=object)
elif hasattr(self, "feature_names_in_"):
del self.feature_names_in_
return self

@property
def classes_(self):
"""Class labels of the refit ``best_estimator_``, for a classifier."""
return self.best_estimator_.classes_

def _best_representation(self, X):
"""Transform ``X`` with the refit ``best_preprocessor_``."""
check_is_fitted(self, "best_estimator_")
return self.best_preprocessor_.transform(X, return_array=True)

def predict(self, X):
"""Predict with the best refit estimator on the best representation."""
check_is_fitted(self, "best_estimator_")
x = self.best_preprocessor_.transform(X, return_array=True)
x = self._best_representation(X)
return self.best_estimator_.predict(x)

@available_if(_estimator_has("predict_proba"))
def predict_proba(self, X):
"""Predict class probabilities with the best refit estimator.

Only available when ``estimator`` implements ``predict_proba``.
"""
x = self._best_representation(X)
return self.best_estimator_.predict_proba(x)

@available_if(_estimator_has("predict_log_proba"))
def predict_log_proba(self, X):
"""Predict class log-probabilities with the best refit estimator.

Only available when ``estimator`` implements ``predict_log_proba``.
"""
x = self._best_representation(X)
return self.best_estimator_.predict_log_proba(x)

@available_if(_estimator_has("decision_function"))
def decision_function(self, X):
"""Compute the decision function of the best refit estimator.

Only available when ``estimator`` implements ``decision_function``.
"""
x = self._best_representation(X)
return self.best_estimator_.decision_function(x)

def score(self, X, y):
"""Score the best refit estimator on ``(X, y)``."""
check_is_fitted(self, "best_estimator_")
x = self.best_preprocessor_.transform(X, return_array=True)
x = self._best_representation(X)
scorer = cast("Callable[..., float]", check_scoring(self.best_estimator_, scoring=self.scoring))
return scorer(self.best_estimator_, x, np.asarray(y).ravel())

def __sklearn_tags__(self):
"""Take the estimator type and classifier / regressor tags of ``estimator``."""
tags = super().__sklearn_tags__() # type: ignore[attr-defined]
estimator_tags = get_tags(self.estimator)
tags.estimator_type = estimator_tags.estimator_type
tags.classifier_tags = deepcopy(estimator_tags.classifier_tags)
tags.regressor_tags = deepcopy(estimator_tags.regressor_tags)
return tags
113 changes: 111 additions & 2 deletions tests/compose/test_search.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,19 @@
import numpy as np
import pandas as pd
import pytest
from sklearn.base import BaseEstimator, RegressorMixin
from sklearn.base import BaseEstimator, RegressorMixin, is_classifier, is_regressor
from sklearn.exceptions import NotFittedError
from sklearn.linear_model import LinearRegression, LogisticRegression, Ridge
from sklearn.model_selection import GroupKFold, GroupShuffleSplit, KFold, LeaveOneGroupOut
from sklearn.model_selection import (
GroupKFold,
GroupShuffleSplit,
KFold,
LeaveOneGroupOut,
StratifiedKFold,
check_cv,
cross_val_score,
)
from sklearn.svm import SVC

from pretab import RepresentationSearchCV
from pretab.exceptions import InvalidParamError
Expand Down Expand Up @@ -215,6 +224,106 @@ def class_data():
return X, np.where(np.sin(X["x"]) > 0, "pos", "neg")


_DELEGATED = ("classes_", "predict_proba", "predict_log_proba", "decision_function")
# Target-aware placement of the "ple" candidate on string class labels.
_CLASSIFICATION = {"task": "classification"}


def test_search_takes_the_estimator_type(class_data):
"""The search was neither a classifier nor a regressor, so nested cross-validation
of a classifier search used KFold instead of StratifiedKFold."""
_, y = class_data
classifier_search = RepresentationSearchCV(LogisticRegression(), methods=["minmax"])
regressor_search = RepresentationSearchCV(Ridge(), methods=["minmax"])

assert is_classifier(classifier_search) and not is_regressor(classifier_search)
assert is_regressor(regressor_search) and not is_classifier(regressor_search)
assert isinstance(check_cv(3, y, classifier=is_classifier(classifier_search)), StratifiedKFold)


def test_classifier_search_delegates_to_the_best_estimator(class_data):
X, y = class_data
search = RepresentationSearchCV(
LogisticRegression(), methods=["minmax", "ple"], cv=3, preprocessor_params=_CLASSIFICATION, random_state=0
).fit(X, y)
x = search.best_preprocessor_.transform(X, return_array=True)

np.testing.assert_array_equal(search.classes_, ["neg", "pos"])
np.testing.assert_allclose(search.predict_proba(X), search.best_estimator_.predict_proba(x))
np.testing.assert_allclose(search.predict_log_proba(X), search.best_estimator_.predict_log_proba(x))
np.testing.assert_allclose(search.decision_function(X), search.best_estimator_.decision_function(x))


@pytest.mark.parametrize("frame_type", ["pandas", "polars"])
def test_grouped_weighted_classifier_preserves_delegation_and_input_metadata(class_data, frame_type):
X, y = class_data
if frame_type == "polars":
pl = pytest.importorskip("polars")
X = pl.DataFrame({"x": X["x"].to_numpy()})
groups = np.repeat(np.arange(30), 10)
weights = np.linspace(0.1, 2.0, len(y))
search = _search(
LogisticRegression(), ["minmax", "ple"], cv=GroupKFold(3), scoring="roc_auc", preprocessor_params=None
).fit(X, y, groups=groups, sample_weight=weights)

assert is_classifier(search)
assert search.best_preprocessor_.task == "classification"
assert search.best_score_ > 0.9
assert search.n_features_in_ == 1
np.testing.assert_array_equal(search.feature_names_in_, ["x"])
np.testing.assert_array_equal(search.classes_, ["neg", "pos"])
x_all = search.best_preprocessor_.transform(X, return_array=True)
expected = LogisticRegression().fit(x_all, y, sample_weight=weights)
np.testing.assert_allclose(search.best_estimator_.coef_, expected.coef_)
np.testing.assert_allclose(search.best_estimator_.intercept_, expected.intercept_)
np.testing.assert_allclose(search.predict_proba(X), expected.predict_proba(x_all))


def test_delegated_methods_follow_the_wrapped_estimator(class_data, nonlinear_data):
X, y = class_data
# SVC without probability=True has a decision_function but no predict_proba.
svc_search = RepresentationSearchCV(SVC(), methods=["minmax"], cv=3)
assert hasattr(svc_search, "decision_function") and not hasattr(svc_search, "predict_proba")
svc_search.fit(X, y)
assert hasattr(svc_search, "decision_function") and not hasattr(svc_search, "predict_proba")

X_reg, y_reg = nonlinear_data
regressor_search = _search(Ridge(), ["minmax"]).fit(X_reg, y_reg)
assert not any(hasattr(regressor_search, attr) for attr in _DELEGATED)


def test_unfitted_classifier_search_raises_not_fitted(class_data):
X, _ = class_data
search = RepresentationSearchCV(LogisticRegression(), methods=["minmax"])
assert not hasattr(search, "classes_")
with pytest.raises(NotFittedError):
search.predict_proba(X)


def test_nested_cross_val_score_with_roc_auc(class_data):
"""roc_auc needs classes_ and predict_proba / decision_function: every outer fold
scored nan."""
X, y = class_data
search = RepresentationSearchCV(
LogisticRegression(), methods=["minmax", "ple"], cv=3, preprocessor_params=_CLASSIFICATION, random_state=0
)
scores = cross_val_score(search, X, y, scoring="roc_auc", cv=3)
assert np.all(np.isfinite(scores))
assert scores.min() > 0.9


def test_search_records_the_input_features(nonlinear_data):
X, y = nonlinear_data
search = _search(LinearRegression(), ["bspline"]).fit(X, y)
assert search.n_features_in_ == 1
np.testing.assert_array_equal(search.feature_names_in_, ["x"])

# Refitting on an array drops the stale names.
search.fit(X.to_numpy(), y)
assert search.n_features_in_ == 1
assert not hasattr(search, "feature_names_in_")


def test_classifier_search_places_against_class_labels(class_data):
"""A classifier made the search keep task="regression": string labels crashed the
target-aware placement and integer labels were treated as a regression target."""
Expand Down
Loading