Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions docs/core_concepts/missing_values.md
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,15 @@ unchanged at `transform`. PreTab never drops a row for missing values: every inp
produces an output row.
```

```{note}
A column with no observed value at `fit` (for example an optional field that is empty in a
training window or a cross-validation fold) still keeps its block, so the output layout does not
depend on which rows were seen. The imputers fill it with `0` (or `fill_value` for
`strategy="constant"`), which makes it a constant column, and `fit` emits a `DataWarning`
naming it. See [Edge-case behaviour](../representations/choosing_a_method.md) for the methods
that cannot be fitted on a constant column.
```

## The `missing_policy` control

For finer control, `missing_policy` selects one of five behaviours for the whole
Expand Down
9 changes: 9 additions & 0 deletions docs/representations/choosing_a_method.md
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,15 @@ PreTab is explicit about degenerate inputs rather than failing silently.
row for it.
- **NaN into a finite-only method**: raises a typed error unless imputation is configured. See
[Missing values](../core_concepts/missing_values.md).
- **Column with no observed value at fit** (missing on every row, e.g. an optional field that is
empty in a training window or a cross-validation fold): `Preprocessor` keeps the column's
block and emits a `DataWarning` naming it. The imputers fill it with `0` (or `fill_value`
for `strategy="constant"`), so it is fitted as a constant column; a method that cannot be
fitted on a constant column (the splines above, or Box-Cox) raises a typed error naming the
column. With imputation disabled the column stays missing: the scalers, `"quantile"` and
`"none"` keep its block as missing values, and every other method raises the same typed
error. `policy={"constant": "error"}` rejects an empty numerical column like any other
constant one.

## Non-goals

Expand Down
21 changes: 14 additions & 7 deletions pretab/compose/factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,18 @@ def _filter_kwargs(allowed, kwargs):
return {key: kwargs[key] for key in allowed if key in kwargs}


def _imputer(strategy, add_indicator, imputer_kwargs):
"""Build the leading :class:`~sklearn.impute.SimpleImputer` of a per-column pipeline.

A column with no observed value at fit is kept (``keep_empty_features=True``)
and filled with ``0``, or with ``fill_value`` for ``strategy="constant"``.
scikit-learn's default drops it, which leaves the rest of the pipeline an
input without columns: the block then fails to fit or vanishes from the output.
"""
imputer_kwargs = {"keep_empty_features": True, **(imputer_kwargs or {})}
return SimpleImputer(strategy=strategy, add_indicator=add_indicator, **imputer_kwargs)


def _clamp_spline_basis(output_dim):
"""Clamp a requested output dimension into the supported B/M/I spline range.

Expand Down Expand Up @@ -131,10 +143,7 @@ def get_numerical_transformer_steps(
steps = []

if add_imputer:
imputer_kwargs = imputer_kwargs or {}
steps.append(
("imputer", SimpleImputer(strategy=imputer_strategy, add_indicator=add_missing_indicator, **imputer_kwargs))
)
steps.append(("imputer", _imputer(imputer_strategy, add_missing_indicator, imputer_kwargs)))

# Optional scaling step, added only when it is not already the chosen method.
scalers = {
Expand Down Expand Up @@ -211,9 +220,7 @@ def get_categorical_transformer_steps(
imputer_kwargs = imputer_kwargs or {}
if "missing_values" not in imputer_kwargs:
steps.append(("normalize_missing", _CategoricalMissingNormalizer()))
steps.append(
("imputer", SimpleImputer(strategy=imputer_strategy, add_indicator=add_missing_indicator, **imputer_kwargs))
)
steps.append(("imputer", _imputer(imputer_strategy, add_missing_indicator, imputer_kwargs)))

if method not in CATEGORICAL_METHODS:
raise invalid_param_error(
Expand Down
7 changes: 5 additions & 2 deletions pretab/compose/inspection.py
Original file line number Diff line number Diff line change
Expand Up @@ -266,8 +266,11 @@ def build_feature_info(column_transformer, *, embeddings, embedding_dimensions):

elif "continuous_ordinal" in steps:
step = representation_pipeline.named_steps["continuous_ordinal"]
categories = len(step.mapping_[columns.index(feature_name)])
dimension = separate_state_dimension if separate_state_dimension is not None else 1
position = columns.index(feature_name)
# An encoder that received no column (a 0-width block) has no mapping.
has_mapping = position < len(step.mapping_)
categories = len(step.mapping_[position]) if has_mapping else None
dimension = separate_state_dimension if separate_state_dimension is not None else int(has_mapping)
categorical_feature_info[feature_name] = {
"preprocessing": preprocessing_type,
"dimension": dimension,
Expand Down
90 changes: 86 additions & 4 deletions pretab/preprocessor.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import os
import time
import warnings
from typing import cast
from typing import Any, cast

import numpy as np
from scipy import sparse as sp
Expand Down Expand Up @@ -38,8 +38,10 @@
from .core.policy import RepresentationPolicy, apply_constant_policy
from .exceptions import (
ConfigWarning,
DataWarning,
FrozenRepresentationError,
OutputBudgetError,
PretabConfigError,
PretabDataError,
PretabSerializationError,
invalid_param_error,
Expand Down Expand Up @@ -371,6 +373,11 @@ class Preprocessor(TransformerMixin, BaseEstimator):
``"dummy"`` -> ``"one-hot"``, ``"ordinal"`` / ``"label"`` -> ``"int"``, ``"poly"`` ->
``"polynomial"``, and ``"passthrough"`` -> ``"none"``.

A column with no observed value at ``fit`` keeps its block: the imputers fill it with ``0``
(or ``fill_value`` for ``strategy="constant"``), so it is fitted as a constant column, and
``fit`` emits a :class:`~pretab.exceptions.DataWarning` naming it. A method that cannot be
fitted on such a column raises a :class:`~pretab.exceptions.PretabDataError` naming it.

Boolean columns (``bool`` and pandas' nullable ``boolean``) are categorical and are encoded
like any other binary categorical column, with the categories ``False`` and ``True``; a
missing value of the nullable ``boolean`` dtype is treated as missing (``NaN``).
Expand Down Expand Up @@ -516,9 +523,18 @@ def fit(self, X, y=None, embeddings=None):
Raises
------
PretabDataError
If ``X`` has a duplicate column label, or two column labels share a string
If a column has no observed (non-missing) value and its method cannot be
fitted on such a column (e.g. the splines or Box-Cox on the imputed constant).
The message names the column.
Also if ``X`` has a duplicate column label, or two column labels share a string
form (e.g. ``1`` and ``"1"``): every column is routed by the string form of
its label, so such columns could not be told apart.

Warns
-----
DataWarning
If a column has no observed (non-missing) value. Its block is kept and fitted
on the entirely missing column, a single constant once imputed.
"""

if self.is_frozen():
Expand Down Expand Up @@ -588,8 +604,24 @@ def fit(self, X, y=None, embeddings=None):
self.policy_ = RepresentationPolicy.resolve(self.policy)
self.numerical_features_ = list(numerical_features)
self.categorical_features_ = list(categorical_features)
# A feature without any observed value (e.g. an optional field that is empty
# in this training window or CV fold) still gets its block, fitted on a
# column that is missing throughout (a single constant once imputed).
empty_features = list(X.columns[X.isna().to_numpy().all(axis=0)]) if len(X) else []
if empty_features:
warnings.warn(
f"{type(self).__name__} received column(s) {empty_features} with no observed (non-missing) "
"value at fit. Their blocks are kept but fitted on an entirely missing column (a single "
"constant value once imputed), so they carry no information until the preprocessor is "
"refit on data where those columns are observed.",
DataWarning,
stacklevel=2,
)
if numerical_features and self.policy_.constant != "allow":
numeric_values = X[numerical_features].to_numpy(dtype=np.float64, na_value=np.nan)
numeric_values = X[numerical_features].to_numpy(dtype=np.float64, na_value=np.nan, copy=True)
# An empty column is represented by one constant value, so the policy
# treats it like any other constant column.
numeric_values[:, np.isnan(numeric_values).all(axis=0)] = 0.0
apply_constant_policy(numeric_values, self.policy_, estimator=self)

valid_formats = ("auto", "dense", "sparse")
Expand Down Expand Up @@ -629,7 +661,13 @@ def fit(self, X, y=None, embeddings=None):
self.column_transformer_.set_output(transform="default")
# ColumnTransformer.fit runs fit_transform internally; keep the training
# output to resolve output_format="auto" once, from the training density.
training_output = self.column_transformer_.fit_transform(self._column_transformer_input(X), y)
X_ct = self._column_transformer_input(X)
try:
training_output = self.column_transformer_.fit_transform(X_ct, y)
except ValueError as exc:
if empty_features and not isinstance(exc, PretabConfigError):
self._raise_for_empty_features(X_ct, y, empty_features, config)
raise
if not sp.issparse(training_output):
training_output = np.asarray(training_output)
self.output_format_ = resolve_output_format(training_output, self.output_format)
Expand Down Expand Up @@ -1058,6 +1096,50 @@ def _reject_missing(self, X) -> None:
"('propagate', 'impute', 'impute_with_indicator', 'separate_state')."
)

def _raise_for_empty_features(self, X, y, empty_features, config) -> None:
"""Name the empty feature behind a failed ``fit``, if one is to blame.

Called when fitting the ColumnTransformer failed. The block of each feature
without an observed value is refitted on its own, and the first that fails
raises a :class:`~pretab.exceptions.PretabDataError` naming that feature,
chained to the block's own error: methods that cannot be fitted on a
constant column (e.g. the B/M/I and cubic splines, Box-Cox) otherwise only
report "the feature at index 0". A block is only blamed when it fits a
stand-in column of distinct observed values with the same ``y``, so a
failure with another cause (e.g. a ``y`` of the wrong length) is not
attributed to the empty column. Returns without raising when no empty
column is to blame, so the caller re-raises the original error.
"""
empty = {str(feature): feature for feature in empty_features}
for _name, transformer, columns in self.column_transformer_.transformers:
if columns[0] not in empty:
continue
feature = empty[columns[0]]
is_numerical = feature in self.numerical_features_
try:
cast(Any, clone(transformer)).fit(X[columns], y)
except PretabConfigError:
continue # a configuration problem, not one of the empty column
except ValueError as exc:
stand_in = X[columns].copy()
stand_in[columns[0]] = (
np.linspace(1.0, 2.0, len(X))
if is_numerical
else np.resize(np.array(["a", "b"], dtype=object), len(X))
)
try:
cast(Any, clone(transformer)).fit(stand_in, y)
except ValueError:
continue # the block fails with observed values too: not the emptiness
method = config.method_for(feature, is_numerical=is_numerical)
raise PretabDataError(
f"Column {feature!r} has no observed (non-missing) value at fit, and its {method!r} "
"representation cannot be fitted on such a column (entirely missing, or a single "
f"constant once imputed): {exc}\n"
"Fix: drop the column, fit on data where it is observed, or select a method that "
"accepts a constant column for it via feature_preprocessing."
) from exc

def _enforce_output_budget(self, n_rows: int) -> None:
"""Check the fitted output width against the configured output budget.

Expand Down
9 changes: 9 additions & 0 deletions tests/compose/test_factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,15 @@ def test_imputer_omitted_when_disabled():
assert "imputer" not in _names(get_numerical_transformer_steps("standardization", add_imputer=False))


def test_imputer_keeps_a_column_without_observed_values():
empty = np.full((4, 1), np.nan)
assert Pipeline(get_numerical_transformer_steps("minmax")).fit_transform(empty).shape == (4, 1)
assert Pipeline(get_categorical_transformer_steps("one-hot")).fit_transform(empty.astype(object)).shape == (4, 1)
# imputer_kwargs can still restore scikit-learn's default of dropping it.
steps = dict(get_numerical_transformer_steps("minmax", imputer_kwargs={"keep_empty_features": False}))
assert steps["imputer"].keep_empty_features is False


def test_none_method_uses_noop_step():
assert _names(get_numerical_transformer_steps("none", add_imputer=False)) == ["noop"]

Expand Down
20 changes: 19 additions & 1 deletion tests/compose/test_inspection.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,14 @@
"""Unit tests for :mod:`pretab.compose.inspection`."""

import warnings

import numpy as np
import pandas as pd
import pytest
from sklearn.compose import ColumnTransformer
from sklearn.pipeline import Pipeline

from pretab.compose.factory import build_column_transformer
from pretab.compose.factory import build_column_transformer, get_categorical_transformer_steps
from pretab.compose.inspection import (
build_feature_info,
build_transformer_summary,
Expand Down Expand Up @@ -56,6 +60,20 @@ def test_build_feature_info_does_not_infer_kind_from_cat_in_feature_name(make_co
assert feature not in categorical


def test_build_feature_info_reports_a_zero_width_block():
# scikit-learn's imputer default drops a column without observed values, so
# the encoder after it receives no column at all.
steps = get_categorical_transformer_steps("int", imputer_kwargs={"keep_empty_features": False})
ct = ColumnTransformer([("cat_city", Pipeline(steps), ["city"])])
with warnings.catch_warnings():
warnings.filterwarnings("ignore", "Skipping features without any observed values", UserWarning)
ct.fit(pd.DataFrame({"city": pd.Series([np.nan] * 4, dtype=object)}))

_, categorical, _ = build_feature_info(ct, embeddings=False, embedding_dimensions={})

assert categorical["city"]["dimension"] == 0


def test_build_transformer_summary_has_header_and_rows():
numerical = {"age": {"preprocessing": "imputer -> standardization", "dimension": 1, "categories": None}}
categorical = {"city": {"preprocessing": "imputer -> continuous_ordinal", "dimension": 1, "categories": 3}}
Expand Down
Loading
Loading