Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions docs/core_concepts/missing_values.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,13 @@ pre = Preprocessor(
)
```

With imputation enabled, the indicator (also used by `missing_policy="impute_with_indicator"`)
is a raw `0`/`1` column computed on the input before imputation and placed next to the
feature's representation. It is never scaled or expanded by the numerical or categorical
method, and it is added only for features that had missing values at `fit`.
For categorical columns, the indicator recognizes `None`, `pd.NA` and `NaN`,
consistently with imputation.

```{note}
Disabling imputation lets `NaN` reach the transformer directly. Scalers, splines, and the
feature maps (`rbf`, `relu`, `sigmoid`, `tanh`) tolerate it, so an affected row's output is
Expand Down
5 changes: 3 additions & 2 deletions pretab/compose/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -175,10 +175,11 @@ def imputation_plan(self, *, is_numerical: bool) -> dict:
booleans and the imputer ``strategy``. When ``missing_policy`` is ``None``
the explicit ``*_imputation`` / ``add_missing_indicator`` parameters stay
authoritative (historical behaviour); otherwise ``missing_policy`` decides.
``add_indicator`` appends a missing indicator built on the raw input next
to the representation (see :func:`~pretab.compose.factory.create_transformer`).
``add_missing_indicator=True`` with imputation disabled for this column
kind routes through the standalone ``MissingStateIndicator`` (via
``separate_state``) instead of the imputer's own indicator, since
``SimpleImputer.add_indicator`` only takes effect when the imputer runs.
``separate_state``) instead, preserving its one-column-per-feature output.
"""
strategy = (
(self.numerical_imputation or "median")
Expand Down
20 changes: 17 additions & 3 deletions pretab/compose/factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@

import numpy as np
from sklearn.compose import ColumnTransformer
from sklearn.impute import SimpleImputer
from sklearn.impute import MissingIndicator, SimpleImputer
from sklearn.pipeline import FeatureUnion, Pipeline
from sklearn.preprocessing import MinMaxScaler, StandardScaler

Expand Down Expand Up @@ -291,7 +291,6 @@ def create_transformer(method: str, *, is_numerical: bool, config: PreprocessorC
target_aware=config.target_aware,
add_imputer=plan["add_imputer"],
imputer_strategy=plan["strategy"],
add_missing_indicator=plan["add_indicator"],
output_dim=config.output_dim,
adaptive=config.adaptive,
min_output_dim=config.min_output_dim if config.adaptive else None,
Expand Down Expand Up @@ -321,7 +320,6 @@ def create_transformer(method: str, *, is_numerical: bool, config: PreprocessorC
method,
add_imputer=plan["add_imputer"],
imputer_strategy=plan["strategy"],
add_missing_indicator=plan["add_indicator"],
**constructor_kwargs,
)

Expand All @@ -330,6 +328,22 @@ def create_transformer(method: str, *, is_numerical: bool, config: PreprocessorC
# Emit a dedicated ``__missing`` column (built on the raw input) alongside
# the imputed representation, so the indicator never enters the basis.
return FeatureUnion([("representation", pipeline), ("missing", MissingStateIndicator())])
if plan["add_indicator"]:
# The imputer's missing indicator, built on the raw input next to the
# representation instead of being fed through the scaler and basis as a
# second feature. It keeps SimpleImputer(add_indicator=True)'s semantics
# (a column only for features with missing values at fit) and names.
indicator = MissingIndicator(error_on_new=False)
if is_numerical:
missing_branch = indicator
else:
missing_branch = Pipeline(
[("normalize_missing", _CategoricalMissingNormalizer()), ("indicator", indicator)]
)
return FeatureUnion(
[("representation", pipeline), ("missing", missing_branch)],
verbose_feature_names_out=False,
)
return pipeline


Expand Down
26 changes: 20 additions & 6 deletions pretab/compose/inspection.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
"clean_feature_names",
"feature_names_out",
"get_output_slices",
"representation_leaf",
]


Expand Down Expand Up @@ -178,6 +179,19 @@ def _separate_state_branches(transformer):
return branches["representation"], branches["missing"]


def representation_leaf(transformer):
"""Return the last step of a fitted per-column block's representation.

For a separate-state / missing-indicator union this is the last step of its
``"representation"`` pipeline (the raw missing indicator beside it is not
part of the representation); for a pipeline it is the last step, and any
other block is returned unchanged.
"""
separate_state = _separate_state_branches(transformer)
representation = transformer if separate_state is None else separate_state[0]
return representation.steps[-1][1] if hasattr(representation, "steps") else representation


def build_feature_info(column_transformer, *, embeddings, embedding_dimensions):
"""Collect per-feature metadata (preprocessing, dimension, categories).

Expand Down Expand Up @@ -410,12 +424,12 @@ def build_feature_lineage(column_transformer):

separate_state = _separate_state_branches(transformer)
if separate_state is not None:
representation_pipeline, _missing_indicator = separate_state
union_names = [str(value) for value in transformer.get_feature_names_out(list(columns))]
representation_width = sum(value.startswith("representation__") for value in union_names)
missing_width = sum(value.startswith("missing__") for value in union_names)
if representation_width + missing_width != width:
representation_width = width - missing_width
representation_pipeline, missing_indicator = separate_state
# The missing branch comes last in the union; read its width from the
# fitted indicator, which may emit no column (MissingIndicator only
# marks features that had missing values at fit).
missing_width = len(missing_indicator.get_feature_names_out([str(column) for column in columns]))
representation_width = width - missing_width

family, component, uses_target, is_interaction = _resolve_block_representation(
representation_pipeline, columns
Expand Down
11 changes: 9 additions & 2 deletions pretab/compose/serialize.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
from ..core.representation import FeatureLineage, RepresentationSpec
from ..exceptions import PretabError, PretabSerializationError
from ..placement.base import PlacementResult
from .inspection import representation_leaf
from .registry import TransformerSpec

SCHEMA_VERSION = 1
Expand Down Expand Up @@ -227,15 +228,21 @@ def _library_versions() -> dict:


def _representation_summary(preprocessor) -> list:
"""Best-effort declarative per-representation summary (family/columns/locations)."""
"""Best-effort declarative per-representation summary (family/columns/locations).

One entry per block whose representation ends in a PreTab transformer. For a
missing-indicator / separate-state block that is the last step of its
representation branch (see :func:`~pretab.compose.inspection.representation_leaf`),
not the union itself.
"""
summary: list = []
column_transformer = getattr(preprocessor, "column_transformer_", None)
if column_transformer is None:
return summary
for name, transformer, columns in column_transformer.transformers_:
if name == "remainder":
continue
leaf = transformer.steps[-1][1] if hasattr(transformer, "steps") else transformer
leaf = representation_leaf(transformer)
spec_fn = getattr(leaf, "get_representation_spec", None)
if spec_fn is None:
continue
Expand Down
16 changes: 10 additions & 6 deletions pretab/preprocessor.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
build_transformer_summary,
feature_names_out,
get_output_slices,
representation_leaf,
)
from .compose.output import (
compute_output_report,
Expand Down Expand Up @@ -218,10 +219,12 @@ class Preprocessor(TransformerMixin, BaseEstimator):
Strategy for the ``SimpleImputer`` that runs *before* every categorical method. ``None``
disables imputation for categorical columns.
add_missing_indicator : bool, default=False
If True, append a binary missing-value indicator column for each imputed feature (via the
imputer's ``add_indicator``; a standalone :class:`~pretab.transformers.MissingStateIndicator`
is used instead when imputation is disabled for that column kind). Applies to both
numerical and categorical pipelines.
If True, append a binary (0/1) missing-value indicator column for each imputed feature
that has missing values at ``fit`` (scikit-learn's ``MissingIndicator``, computed on the raw
input next to the representation, so it never passes through the scaler or basis; a
standalone :class:`~pretab.transformers.MissingStateIndicator` is used instead when
imputation is disabled for that column kind). Applies to both numerical and categorical
pipelines.
missing_policy : {"error", "propagate", "impute", "impute_with_indicator", "separate_state"} or None, default=None
High-level missing-value strategy. ``None`` (default) keeps the explicit
``numerical_imputation`` / ``categorical_imputation`` / ``add_missing_indicator``
Expand All @@ -232,7 +235,8 @@ class Preprocessor(TransformerMixin, BaseEstimator):
- ``"propagate"`` -- disable imputation so NaNs reach the transformers unchanged
(each family applies its own missing-value contract).
- ``"impute"`` -- impute with the configured strategy (no indicator).
- ``"impute_with_indicator"`` -- impute and append a missing indicator column.
- ``"impute_with_indicator"`` -- impute and append a raw 0/1 missing indicator column,
as ``add_missing_indicator=True`` does.
- ``"separate_state"`` -- impute for the representation *and* emit a dedicated
``__missing`` column per feature that stays outside the ordinary basis, so a
downstream model can learn a separate response to missingness.
Expand Down Expand Up @@ -1161,7 +1165,7 @@ def _log_internal_decisions(self):
"""Log fitted internal decisions (bins / knots / centers) at DEBUG."""
for step_name, transformer, columns in self.column_transformer_.transformers_:
name = block_name(step_name, columns)
last_step = transformer.steps[-1][1] if hasattr(transformer, "steps") else transformer
last_step = representation_leaf(transformer)
for attr in (
"thresholds_",
"knots_",
Expand Down
107 changes: 107 additions & 0 deletions tests/integration/test_missing_policy.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
"""Tests for the high-level ``missing_policy`` orchestration knob (P8.5)."""

import json

import numpy as np
import pandas as pd
import pytest
Expand Down Expand Up @@ -262,3 +264,108 @@ def test_add_missing_indicator_no_longer_requires_imputation_enabled(frame_with_
assert out.size > 0
assert np.isnan(out).any()
assert p.total_output_dim_ > 0


# --- imputer indicator stays outside the representation (issue #62) -----------


def _indicator_values(pre, frame, name):
names = list(pre.get_feature_names_out())
return np.asarray(pre.transform(frame, return_array=True), dtype=float)[:, names.index(name)]


@pytest.mark.parametrize("options", [{"add_missing_indicator": True}, {"missing_policy": "impute_with_indicator"}])
def test_imputer_indicator_is_a_raw_binary_column(frame_with_nan, y, options):
"""Regression guard for issue #62: the indicator was standardized / expanded
by the representation like a second input feature."""
pre = Preprocessor(numerical_method="standardization", **options).fit(frame_with_nan, y)
values = _indicator_values(pre, frame_with_nan, "num_a__missingindicator_a")
np.testing.assert_array_equal(values, frame_with_nan["a"].isna().astype(float))


@pytest.mark.parametrize("method", ["bspline", "rbf", "pspline", "fourier", "cubicspline", "polynomial"])
def test_imputer_indicator_adds_exactly_one_column_per_feature_with_missing(frame_with_nan, y, method):
plain = Preprocessor(numerical_method=method, output_dim=5, missing_policy="impute").fit(frame_with_nan, y)
pre = Preprocessor(numerical_method=method, output_dim=5, missing_policy="impute_with_indicator").fit(
frame_with_nan, y
)
names = list(pre.get_feature_names_out())
indicators = [name for name in names if "missingindicator" in name]
assert indicators == ["num_a__missingindicator_a", "num_b__missingindicator_b"]
assert [name for name in names if name not in indicators] == list(plain.get_feature_names_out())
assert pre.output_dims_ == {key: width + 1 for key, width in plain.output_dims_.items()}


@pytest.mark.parametrize("method", ["bspline", "rbf", "pspline", "fourier", "cubicspline"])
def test_imputer_indicator_lineage_and_feature_info(frame_with_nan, y, method):
pre = Preprocessor(numerical_method=method, output_dim=5, add_missing_indicator=True).fit(frame_with_nan, y)
lineage = pre.get_feature_lineage()
assert [record.output_feature for record in lineage] == list(pre.get_feature_names_out())
missing = [record for record in lineage if record.family == "missing_state"]
assert [record.output_feature for record in missing] == ["num_a__missingindicator_a", "num_b__missingindicator_b"]
numerical, _, _ = pre.get_feature_info(verbose=False)
assert {feature: info["dimension"] for feature, info in numerical.items()} == pre.output_dims_


def test_imputer_indicator_on_categorical_integer_codes(y):
frame = pd.DataFrame({"c": ["x", "y", np.nan, "x", "y", "x"]})
pre = Preprocessor(categorical_method="int", add_missing_indicator=True).fit(frame, y)
assert list(pre.get_feature_names_out()) == ["cat_c", "cat_c__missingindicator_c"]
np.testing.assert_array_equal(_indicator_values(pre, frame, "cat_c__missingindicator_c"), frame["c"].isna())


def test_imputer_indicator_only_marks_features_with_missing_values_at_fit(clean_frame, frame_with_nan, y):
pre = Preprocessor(numerical_method="minmax", add_missing_indicator=True).fit(clean_frame, y)
assert not any("missing" in name for name in pre.get_feature_names_out())
assert np.asarray(pre.transform(frame_with_nan, return_array=True)).shape == (6, 2)


# --- representation summary of missing-indicator / separate-state blocks --------


@pytest.mark.parametrize(
"options",
[
{"add_missing_indicator": True},
{"missing_policy": "impute_with_indicator"},
{"missing_policy": "separate_state"},
],
)
def test_missing_state_blocks_report_their_representation(frame_with_nan, y, options):
"""The spec summary read each block's last step, which for these blocks is the
FeatureUnion of the representation and the missing indicator, so to_spec() and
reproducibility_report() listed no representation for them."""
plain = _bspline(missing_policy="impute").fit(frame_with_nan, y)
pre = _bspline(**options).fit(frame_with_nan, y)

assert pre.reproducibility_report()["representations"] == {"a": "bspline", "b": "bspline"}
assert pre.to_spec()["representations"] == plain.to_spec()["representations"]


def test_block_with_the_imputer_built_in_indicator_still_loads_and_reports(frame_with_nan, y):
"""A preprocessor fitted by PreTab 1.0.0 with add_missing_indicator has the
imputer's built-in indicator in front of the representation (a pipeline, not
a union). Such an object still transforms, round-trips through its spec with
the same fingerprint, and lists its representations."""
from sklearn.pipeline import Pipeline

pre = _bspline(add_missing_indicator=True).fit(frame_with_nan, y)

def legacy_block(transformer, columns):
if not isinstance(transformer, Pipeline):
transformer = dict(transformer.transformer_list)["representation"]
block = clone(transformer)
assert isinstance(block, Pipeline)
return block.set_params(imputer__add_indicator=True).fit(frame_with_nan[columns], y)

pre.column_transformer_.transformers_ = [
(name, legacy_block(transformer, columns), columns)
for name, transformer, columns in pre.column_transformer_.transformers_
]
out = np.asarray(pre.transform(frame_with_nan, return_array=True))
restored = Preprocessor.from_spec(json.loads(json.dumps(pre.to_spec())))

assert out.shape == (6, 2 * 2 * 8) # the old layout expands the indicator too
np.testing.assert_array_equal(np.asarray(restored.transform(frame_with_nan, return_array=True)), out)
assert restored.fingerprint_ == pre.fingerprint_
assert restored.reproducibility_report()["representations"] == {"a": "bspline", "b": "bspline"}
Loading
Loading