diff --git a/docs/core_concepts/target_awareness.md b/docs/core_concepts/target_awareness.md index 3b74e94..3ac549e 100644 --- a/docs/core_concepts/target_awareness.md +++ b/docs/core_concepts/target_awareness.md @@ -214,6 +214,12 @@ Any other keyword argument to `fit`, such as `sample_weight`, is passed to the e `Preprocessor` and the scorer do not see it, so placement and validation scores are unweighted. +The search takes the estimator type of the estimator it wraps. Around a classifier it is a +classifier itself and delegates `classes_`, `predict_proba`, `predict_log_proba` and +`decision_function` (whichever the estimator provides) to the refit `best_estimator_`. It can +therefore be evaluated as a whole in an outer, stratified cross-validation, for example with +`cross_val_score(search, X, y_class, scoring="roc_auc")`. + ```{note} This is deliberately narrow: it only searches the single `numerical_method` axis with one global method for every numerical column, not a per-column `feature_preprocessing` search. diff --git a/pretab/compose/search.py b/pretab/compose/search.py index 16680fa..a3c9020 100644 --- a/pretab/compose/search.py +++ b/pretab/compose/search.py @@ -9,12 +9,15 @@ """ from collections.abc import Callable +from copy import deepcopy from typing import cast import numpy as np from sklearn.base import BaseEstimator, clone, is_classifier from sklearn.metrics import check_scoring from sklearn.model_selection import BaseCrossValidator, check_cv +from sklearn.utils import get_tags +from sklearn.utils.metaestimators import available_if from sklearn.utils.validation import _check_method_params, check_is_fitted from ..core._typing import PredictorLike @@ -32,9 +35,30 @@ def _row_subset(data, idx): return np.asarray(data)[idx] +def _estimator_has(attr): + """Check whether ``attr`` can be delegated to the downstream estimator. + + As in scikit-learn's search estimators, the refit ``best_estimator_`` is + checked after fit and the unfitted ``estimator`` before, so ``hasattr`` on an + unfitted search already reflects the estimator it wraps. + """ + + def check(self): + # getattr raises AttributeError when the estimator does not provide attr. + getattr(self.best_estimator_ if hasattr(self, "best_estimator_") else self.estimator, attr) + return True + + return check + + class RepresentationSearchCV(BaseEstimator): """Select the best numerical representation by cross-validation. + The search takes the estimator type of ``estimator``: wrapping a classifier + makes it a classifier (``is_classifier``, stratified outer cross-validation), + and ``predict_proba``, ``predict_log_proba`` and ``decision_function`` are + available whenever ``estimator`` provides them. + Parameters ---------- estimator : estimator @@ -68,6 +92,13 @@ class RepresentationSearchCV(BaseEstimator): Preprocessor for ``best_method_`` refit on all data. best_estimator_ : estimator Estimator refit on the best representation of all data. + classes_ : ndarray of shape (n_classes,) + Class labels of ``best_estimator_``. Only available for a classifier. + n_features_in_ : int + Number of features seen during :meth:`fit`. + feature_names_in_ : ndarray of shape (`n_features_in_`,) + Names of the features seen during :meth:`fit`. Only defined when ``X`` has + string column names. """ def __init__(self, estimator, methods, *, cv=5, scoring=None, preprocessor_params=None, random_state=None): @@ -166,17 +197,69 @@ def fit(self, X, y=None, *, groups=None, **fit_params): self.best_preprocessor_ = self._make_preprocessor(best_method) x_all = self.best_preprocessor_.fit_transform(X, y_arr, return_array=True) self.best_estimator_ = cast(PredictorLike, clone(self.estimator)).fit(x_all, y_arr, **fit_params) + self.n_features_in_ = self.best_preprocessor_.n_features_in_ + # scikit-learn convention: feature names are recorded only for input whose + # column labels are all strings. + columns = getattr(X, "columns", None) + if columns is not None and all(isinstance(label, str) for label in columns): + self.feature_names_in_ = np.asarray(columns, dtype=object) + elif hasattr(self, "feature_names_in_"): + del self.feature_names_in_ return self + @property + def classes_(self): + """Class labels of the refit ``best_estimator_``, for a classifier.""" + return self.best_estimator_.classes_ + + def _best_representation(self, X): + """Transform ``X`` with the refit ``best_preprocessor_``.""" + check_is_fitted(self, "best_estimator_") + return self.best_preprocessor_.transform(X, return_array=True) + def predict(self, X): """Predict with the best refit estimator on the best representation.""" - check_is_fitted(self, "best_estimator_") - x = self.best_preprocessor_.transform(X, return_array=True) + x = self._best_representation(X) return self.best_estimator_.predict(x) + @available_if(_estimator_has("predict_proba")) + def predict_proba(self, X): + """Predict class probabilities with the best refit estimator. + + Only available when ``estimator`` implements ``predict_proba``. + """ + x = self._best_representation(X) + return self.best_estimator_.predict_proba(x) + + @available_if(_estimator_has("predict_log_proba")) + def predict_log_proba(self, X): + """Predict class log-probabilities with the best refit estimator. + + Only available when ``estimator`` implements ``predict_log_proba``. + """ + x = self._best_representation(X) + return self.best_estimator_.predict_log_proba(x) + + @available_if(_estimator_has("decision_function")) + def decision_function(self, X): + """Compute the decision function of the best refit estimator. + + Only available when ``estimator`` implements ``decision_function``. + """ + x = self._best_representation(X) + return self.best_estimator_.decision_function(x) + def score(self, X, y): """Score the best refit estimator on ``(X, y)``.""" - check_is_fitted(self, "best_estimator_") - x = self.best_preprocessor_.transform(X, return_array=True) + x = self._best_representation(X) scorer = cast("Callable[..., float]", check_scoring(self.best_estimator_, scoring=self.scoring)) return scorer(self.best_estimator_, x, np.asarray(y).ravel()) + + def __sklearn_tags__(self): + """Take the estimator type and classifier / regressor tags of ``estimator``.""" + tags = super().__sklearn_tags__() # type: ignore[attr-defined] + estimator_tags = get_tags(self.estimator) + tags.estimator_type = estimator_tags.estimator_type + tags.classifier_tags = deepcopy(estimator_tags.classifier_tags) + tags.regressor_tags = deepcopy(estimator_tags.regressor_tags) + return tags diff --git a/tests/compose/test_search.py b/tests/compose/test_search.py index b64181a..630666f 100644 --- a/tests/compose/test_search.py +++ b/tests/compose/test_search.py @@ -9,10 +9,19 @@ import numpy as np import pandas as pd import pytest -from sklearn.base import BaseEstimator, RegressorMixin +from sklearn.base import BaseEstimator, RegressorMixin, is_classifier, is_regressor from sklearn.exceptions import NotFittedError from sklearn.linear_model import LinearRegression, LogisticRegression, Ridge -from sklearn.model_selection import GroupKFold, GroupShuffleSplit, KFold, LeaveOneGroupOut +from sklearn.model_selection import ( + GroupKFold, + GroupShuffleSplit, + KFold, + LeaveOneGroupOut, + StratifiedKFold, + check_cv, + cross_val_score, +) +from sklearn.svm import SVC from pretab import RepresentationSearchCV from pretab.exceptions import InvalidParamError @@ -215,6 +224,106 @@ def class_data(): return X, np.where(np.sin(X["x"]) > 0, "pos", "neg") +_DELEGATED = ("classes_", "predict_proba", "predict_log_proba", "decision_function") +# Target-aware placement of the "ple" candidate on string class labels. +_CLASSIFICATION = {"task": "classification"} + + +def test_search_takes_the_estimator_type(class_data): + """The search was neither a classifier nor a regressor, so nested cross-validation + of a classifier search used KFold instead of StratifiedKFold.""" + _, y = class_data + classifier_search = RepresentationSearchCV(LogisticRegression(), methods=["minmax"]) + regressor_search = RepresentationSearchCV(Ridge(), methods=["minmax"]) + + assert is_classifier(classifier_search) and not is_regressor(classifier_search) + assert is_regressor(regressor_search) and not is_classifier(regressor_search) + assert isinstance(check_cv(3, y, classifier=is_classifier(classifier_search)), StratifiedKFold) + + +def test_classifier_search_delegates_to_the_best_estimator(class_data): + X, y = class_data + search = RepresentationSearchCV( + LogisticRegression(), methods=["minmax", "ple"], cv=3, preprocessor_params=_CLASSIFICATION, random_state=0 + ).fit(X, y) + x = search.best_preprocessor_.transform(X, return_array=True) + + np.testing.assert_array_equal(search.classes_, ["neg", "pos"]) + np.testing.assert_allclose(search.predict_proba(X), search.best_estimator_.predict_proba(x)) + np.testing.assert_allclose(search.predict_log_proba(X), search.best_estimator_.predict_log_proba(x)) + np.testing.assert_allclose(search.decision_function(X), search.best_estimator_.decision_function(x)) + + +@pytest.mark.parametrize("frame_type", ["pandas", "polars"]) +def test_grouped_weighted_classifier_preserves_delegation_and_input_metadata(class_data, frame_type): + X, y = class_data + if frame_type == "polars": + pl = pytest.importorskip("polars") + X = pl.DataFrame({"x": X["x"].to_numpy()}) + groups = np.repeat(np.arange(30), 10) + weights = np.linspace(0.1, 2.0, len(y)) + search = _search( + LogisticRegression(), ["minmax", "ple"], cv=GroupKFold(3), scoring="roc_auc", preprocessor_params=None + ).fit(X, y, groups=groups, sample_weight=weights) + + assert is_classifier(search) + assert search.best_preprocessor_.task == "classification" + assert search.best_score_ > 0.9 + assert search.n_features_in_ == 1 + np.testing.assert_array_equal(search.feature_names_in_, ["x"]) + np.testing.assert_array_equal(search.classes_, ["neg", "pos"]) + x_all = search.best_preprocessor_.transform(X, return_array=True) + expected = LogisticRegression().fit(x_all, y, sample_weight=weights) + np.testing.assert_allclose(search.best_estimator_.coef_, expected.coef_) + np.testing.assert_allclose(search.best_estimator_.intercept_, expected.intercept_) + np.testing.assert_allclose(search.predict_proba(X), expected.predict_proba(x_all)) + + +def test_delegated_methods_follow_the_wrapped_estimator(class_data, nonlinear_data): + X, y = class_data + # SVC without probability=True has a decision_function but no predict_proba. + svc_search = RepresentationSearchCV(SVC(), methods=["minmax"], cv=3) + assert hasattr(svc_search, "decision_function") and not hasattr(svc_search, "predict_proba") + svc_search.fit(X, y) + assert hasattr(svc_search, "decision_function") and not hasattr(svc_search, "predict_proba") + + X_reg, y_reg = nonlinear_data + regressor_search = _search(Ridge(), ["minmax"]).fit(X_reg, y_reg) + assert not any(hasattr(regressor_search, attr) for attr in _DELEGATED) + + +def test_unfitted_classifier_search_raises_not_fitted(class_data): + X, _ = class_data + search = RepresentationSearchCV(LogisticRegression(), methods=["minmax"]) + assert not hasattr(search, "classes_") + with pytest.raises(NotFittedError): + search.predict_proba(X) + + +def test_nested_cross_val_score_with_roc_auc(class_data): + """roc_auc needs classes_ and predict_proba / decision_function: every outer fold + scored nan.""" + X, y = class_data + search = RepresentationSearchCV( + LogisticRegression(), methods=["minmax", "ple"], cv=3, preprocessor_params=_CLASSIFICATION, random_state=0 + ) + scores = cross_val_score(search, X, y, scoring="roc_auc", cv=3) + assert np.all(np.isfinite(scores)) + assert scores.min() > 0.9 + + +def test_search_records_the_input_features(nonlinear_data): + X, y = nonlinear_data + search = _search(LinearRegression(), ["bspline"]).fit(X, y) + assert search.n_features_in_ == 1 + np.testing.assert_array_equal(search.feature_names_in_, ["x"]) + + # Refitting on an array drops the stale names. + search.fit(X.to_numpy(), y) + assert search.n_features_in_ == 1 + assert not hasattr(search, "feature_names_in_") + + def test_classifier_search_places_against_class_labels(class_data): """A classifier made the search keep task="regression": string labels crashed the target-aware placement and integer labels were treated as a regression target."""