diff --git a/docs/core_concepts/target_awareness.md b/docs/core_concepts/target_awareness.md index 3ac549e..7c0dbc9 100644 --- a/docs/core_concepts/target_awareness.md +++ b/docs/core_concepts/target_awareness.md @@ -162,7 +162,9 @@ to fit it) leaks information the same way an unguarded supervised transformer do `numerical_method` values, refits the best one on all the data, and keeps every candidate's scoring honest by fitting a fresh `Preprocessor` per fold. With a classifier as the estimator, every `Preprocessor` uses `task="classification"`, so target-aware methods place against the -class labels; a `task` set in `preprocessor_params` takes precedence. +class labels; a `task` set in `preprocessor_params` takes precedence. The candidate names are +checked when `fit` starts, so a misspelled method raises before any candidate is +cross-validated. ```python import numpy as np diff --git a/pretab/compose/search.py b/pretab/compose/search.py index a3c9020..6d82f6b 100644 --- a/pretab/compose/search.py +++ b/pretab/compose/search.py @@ -22,8 +22,9 @@ from ..core._typing import PredictorLike from ..core.validation import is_polars_frame, polars_to_pandas -from ..exceptions import InvalidParamError +from ..exceptions import InvalidParamError, invalid_param_error from ..preprocessor import Preprocessor +from .registry import NUMERICAL_ALIASES, NUMERICAL_METHODS, resolve_method __all__ = ["RepresentationSearchCV"] @@ -64,7 +65,12 @@ class RepresentationSearchCV(BaseEstimator): estimator : estimator Downstream supervised estimator fit on the transformed features. methods : sequence of str - Candidate ``numerical_method`` values to search over. + Candidate ``numerical_method`` values to search over, e.g. + ``["minmax", "bspline"]``. Names are resolved like + ``Preprocessor(numerical_method=...)`` (aliases and ``None`` included) and + checked when :meth:`fit` starts, before any candidate is cross-validated. + A single string is rejected instead of being read as a sequence of + one-letter names. cv : int or cross-validation generator, default=5 Cross-validation splitting strategy passed to :func:`sklearn.model_selection.check_cv`. A group splitter such as @@ -76,7 +82,8 @@ class RepresentationSearchCV(BaseEstimator): preprocessor_params : dict or None, default=None Extra keyword arguments forwarded to every :class:`Preprocessor`. When ``estimator`` is a classifier, ``task`` defaults to ``"classification"`` so - target-aware placement treats ``y`` as class labels. + target-aware placement treats ``y`` as class labels. ``numerical_method`` + cannot be set here: the search sets it to each entry of ``methods``. random_state : int or None, default=None Seed forwarded to each :class:`Preprocessor`. @@ -121,6 +128,48 @@ def _make_preprocessor(self, method): params.setdefault("task", "classification") return Preprocessor(numerical_method=method, **params) + def _validated_methods(self) -> list: + """Return ``methods`` as a list, checked before any candidate is fitted. + + A typo would otherwise only surface inside the first fold of that + candidate, after every candidate listed before it has been + cross-validated, and a single string would be searched letter by letter. + """ + name = type(self).__name__ + if isinstance(self.methods, str): + raise invalid_param_error( + name, + "methods", + self.methods, + f"must be a sequence of numerical method names, not a single string; pass [{self.methods!r}] " + "to search one method", + ) + methods = list(self.methods) + if not methods: + raise InvalidParamError("methods must be a non-empty sequence of numerical methods.") + for method in methods: + # None is Preprocessor's spelling of "none" (leave numerical columns unchanged). + known = method is None or ( + isinstance(method, str) + and resolve_method(method, NUMERICAL_METHODS, NUMERICAL_ALIASES) in NUMERICAL_METHODS + ) + if not known: + raise invalid_param_error( + name, + "methods", + method, + "every entry must name a numerical preprocessing method", + valid=set(NUMERICAL_METHODS), + ) + if "numerical_method" in (self.preprocessor_params or {}): + raise invalid_param_error( + name, + "preprocessor_params", + self.preprocessor_params, + "must not set 'numerical_method': the search sets it to each entry of methods", + ) + return methods + def fit(self, X, y=None, *, groups=None, **fit_params): """Search over ``methods`` and refit the best representation on all data. @@ -151,10 +200,16 @@ def fit(self, X, y=None, *, groups=None, **fit_params): ------- self : RepresentationSearchCV Fitted search. + + Raises + ------ + InvalidParamError + If ``methods`` is empty, a single string, or has an entry that does not + name a numerical method; if ``preprocessor_params`` sets + ``numerical_method``; or if ``y`` is None. Raised before any candidate + is fitted. """ - methods = list(self.methods) - if not methods: - raise InvalidParamError("methods must be a non-empty sequence of numerical methods.") + methods = self._validated_methods() if y is None: raise InvalidParamError("RepresentationSearchCV requires y at fit time; got y=None.") y_arr = np.asarray(y).ravel() diff --git a/tests/compose/test_search.py b/tests/compose/test_search.py index 630666f..4d48c2d 100644 --- a/tests/compose/test_search.py +++ b/tests/compose/test_search.py @@ -113,6 +113,54 @@ def test_empty_methods_raises(nonlinear_data): RepresentationSearchCV(LinearRegression(), methods=[]).fit(X, y) +def test_methods_as_a_single_string_is_rejected(nonlinear_data): + """A string was iterated letter by letter: "bspline" failed as the unknown method 'b'.""" + X, y = nonlinear_data + with pytest.raises(InvalidParamError, match=r"not a single string; pass \['bspline'\]"): + RepresentationSearchCV(LinearRegression(), methods="bspline").fit(X, y) + + +class _CountingRegressor(RegressorMixin, BaseEstimator): + """Counts fits on the class, so clones made by the search are counted too.""" + + n_fits = 0 + + def fit(self, X, y): + type(self).n_fits += 1 + return self + + def predict(self, X): + return np.zeros(len(X)) + + +@pytest.mark.parametrize("unknown", ["no-such-method", "one-hot", "tensorspline", 3]) +def test_unknown_method_is_rejected_before_any_candidate_is_fitted(nonlinear_data, unknown): + """An unknown name used to fail only after the candidates before it were cross-validated.""" + X, y = nonlinear_data + _CountingRegressor.n_fits = 0 + search = _search(_CountingRegressor(), ["minmax", unknown]) + with pytest.raises(InvalidParamError, match=r"RepresentationSearchCV\.methods = .* is invalid"): + search.fit(X, y) + assert _CountingRegressor.n_fits == 0 + assert not hasattr(search, "cv_results_") + + +def test_method_aliases_and_none_stay_valid(nonlinear_data): + X, y = nonlinear_data + search = _search(LinearRegression(), ["std", "B-Spline", None]).fit(X, y) + assert list(search.cv_results_) == ["std", "B-Spline", None] + + +def test_numerical_method_in_preprocessor_params_is_rejected(nonlinear_data): + """It collided with the searched method in a bare TypeError from Preprocessor().""" + X, y = nonlinear_data + search = RepresentationSearchCV( + LinearRegression(), methods=["minmax"], preprocessor_params={"numerical_method": "ple"} + ) + with pytest.raises(InvalidParamError, match="must not set 'numerical_method'"): + search.fit(X, y) + + def test_requires_y_at_fit(nonlinear_data): X, _ = nonlinear_data with pytest.raises(InvalidParamError):