diff --git a/docs/core_concepts/missing_values.md b/docs/core_concepts/missing_values.md index 0eba1f5..799c477 100644 --- a/docs/core_concepts/missing_values.md +++ b/docs/core_concepts/missing_values.md @@ -25,6 +25,13 @@ pre = Preprocessor( ) ``` +With imputation enabled, the indicator (also used by `missing_policy="impute_with_indicator"`) +is a raw `0`/`1` column computed on the input before imputation and placed next to the +feature's representation. It is never scaled or expanded by the numerical or categorical +method, and it is added only for features that had missing values at `fit`. +For categorical columns, the indicator recognizes `None`, `pd.NA` and `NaN`, +consistently with imputation. + ```{note} Disabling imputation lets `NaN` reach the transformer directly. Scalers, splines, and the feature maps (`rbf`, `relu`, `sigmoid`, `tanh`) tolerate it, so an affected row's output is diff --git a/pretab/compose/config.py b/pretab/compose/config.py index b5c7dc7..a9c5ff2 100644 --- a/pretab/compose/config.py +++ b/pretab/compose/config.py @@ -175,10 +175,11 @@ def imputation_plan(self, *, is_numerical: bool) -> dict: booleans and the imputer ``strategy``. When ``missing_policy`` is ``None`` the explicit ``*_imputation`` / ``add_missing_indicator`` parameters stay authoritative (historical behaviour); otherwise ``missing_policy`` decides. + ``add_indicator`` appends a missing indicator built on the raw input next + to the representation (see :func:`~pretab.compose.factory.create_transformer`). ``add_missing_indicator=True`` with imputation disabled for this column kind routes through the standalone ``MissingStateIndicator`` (via - ``separate_state``) instead of the imputer's own indicator, since - ``SimpleImputer.add_indicator`` only takes effect when the imputer runs. + ``separate_state``) instead, preserving its one-column-per-feature output. """ strategy = ( (self.numerical_imputation or "median") diff --git a/pretab/compose/factory.py b/pretab/compose/factory.py index 0465244..fa601df 100644 --- a/pretab/compose/factory.py +++ b/pretab/compose/factory.py @@ -12,7 +12,7 @@ import numpy as np from sklearn.compose import ColumnTransformer -from sklearn.impute import SimpleImputer +from sklearn.impute import MissingIndicator, SimpleImputer from sklearn.pipeline import FeatureUnion, Pipeline from sklearn.preprocessing import MinMaxScaler, StandardScaler @@ -291,7 +291,6 @@ def create_transformer(method: str, *, is_numerical: bool, config: PreprocessorC target_aware=config.target_aware, add_imputer=plan["add_imputer"], imputer_strategy=plan["strategy"], - add_missing_indicator=plan["add_indicator"], output_dim=config.output_dim, adaptive=config.adaptive, min_output_dim=config.min_output_dim if config.adaptive else None, @@ -321,7 +320,6 @@ def create_transformer(method: str, *, is_numerical: bool, config: PreprocessorC method, add_imputer=plan["add_imputer"], imputer_strategy=plan["strategy"], - add_missing_indicator=plan["add_indicator"], **constructor_kwargs, ) @@ -330,6 +328,22 @@ def create_transformer(method: str, *, is_numerical: bool, config: PreprocessorC # Emit a dedicated ``__missing`` column (built on the raw input) alongside # the imputed representation, so the indicator never enters the basis. return FeatureUnion([("representation", pipeline), ("missing", MissingStateIndicator())]) + if plan["add_indicator"]: + # The imputer's missing indicator, built on the raw input next to the + # representation instead of being fed through the scaler and basis as a + # second feature. It keeps SimpleImputer(add_indicator=True)'s semantics + # (a column only for features with missing values at fit) and names. + indicator = MissingIndicator(error_on_new=False) + if is_numerical: + missing_branch = indicator + else: + missing_branch = Pipeline( + [("normalize_missing", _CategoricalMissingNormalizer()), ("indicator", indicator)] + ) + return FeatureUnion( + [("representation", pipeline), ("missing", missing_branch)], + verbose_feature_names_out=False, + ) return pipeline diff --git a/pretab/compose/inspection.py b/pretab/compose/inspection.py index 9ceecdf..ae90fea 100644 --- a/pretab/compose/inspection.py +++ b/pretab/compose/inspection.py @@ -26,6 +26,7 @@ "clean_feature_names", "feature_names_out", "get_output_slices", + "representation_leaf", ] @@ -178,6 +179,19 @@ def _separate_state_branches(transformer): return branches["representation"], branches["missing"] +def representation_leaf(transformer): + """Return the last step of a fitted per-column block's representation. + + For a separate-state / missing-indicator union this is the last step of its + ``"representation"`` pipeline (the raw missing indicator beside it is not + part of the representation); for a pipeline it is the last step, and any + other block is returned unchanged. + """ + separate_state = _separate_state_branches(transformer) + representation = transformer if separate_state is None else separate_state[0] + return representation.steps[-1][1] if hasattr(representation, "steps") else representation + + def build_feature_info(column_transformer, *, embeddings, embedding_dimensions): """Collect per-feature metadata (preprocessing, dimension, categories). @@ -410,12 +424,12 @@ def build_feature_lineage(column_transformer): separate_state = _separate_state_branches(transformer) if separate_state is not None: - representation_pipeline, _missing_indicator = separate_state - union_names = [str(value) for value in transformer.get_feature_names_out(list(columns))] - representation_width = sum(value.startswith("representation__") for value in union_names) - missing_width = sum(value.startswith("missing__") for value in union_names) - if representation_width + missing_width != width: - representation_width = width - missing_width + representation_pipeline, missing_indicator = separate_state + # The missing branch comes last in the union; read its width from the + # fitted indicator, which may emit no column (MissingIndicator only + # marks features that had missing values at fit). + missing_width = len(missing_indicator.get_feature_names_out([str(column) for column in columns])) + representation_width = width - missing_width family, component, uses_target, is_interaction = _resolve_block_representation( representation_pipeline, columns diff --git a/pretab/compose/serialize.py b/pretab/compose/serialize.py index c4b213f..7418224 100644 --- a/pretab/compose/serialize.py +++ b/pretab/compose/serialize.py @@ -29,6 +29,7 @@ from ..core.representation import FeatureLineage, RepresentationSpec from ..exceptions import PretabError, PretabSerializationError from ..placement.base import PlacementResult +from .inspection import representation_leaf from .registry import TransformerSpec SCHEMA_VERSION = 1 @@ -227,7 +228,13 @@ def _library_versions() -> dict: def _representation_summary(preprocessor) -> list: - """Best-effort declarative per-representation summary (family/columns/locations).""" + """Best-effort declarative per-representation summary (family/columns/locations). + + One entry per block whose representation ends in a PreTab transformer. For a + missing-indicator / separate-state block that is the last step of its + representation branch (see :func:`~pretab.compose.inspection.representation_leaf`), + not the union itself. + """ summary: list = [] column_transformer = getattr(preprocessor, "column_transformer_", None) if column_transformer is None: @@ -235,7 +242,7 @@ def _representation_summary(preprocessor) -> list: for name, transformer, columns in column_transformer.transformers_: if name == "remainder": continue - leaf = transformer.steps[-1][1] if hasattr(transformer, "steps") else transformer + leaf = representation_leaf(transformer) spec_fn = getattr(leaf, "get_representation_spec", None) if spec_fn is None: continue diff --git a/pretab/preprocessor.py b/pretab/preprocessor.py index 9a9d991..cfffa67 100644 --- a/pretab/preprocessor.py +++ b/pretab/preprocessor.py @@ -22,6 +22,7 @@ build_transformer_summary, feature_names_out, get_output_slices, + representation_leaf, ) from .compose.output import ( compute_output_report, @@ -218,10 +219,12 @@ class Preprocessor(TransformerMixin, BaseEstimator): Strategy for the ``SimpleImputer`` that runs *before* every categorical method. ``None`` disables imputation for categorical columns. add_missing_indicator : bool, default=False - If True, append a binary missing-value indicator column for each imputed feature (via the - imputer's ``add_indicator``; a standalone :class:`~pretab.transformers.MissingStateIndicator` - is used instead when imputation is disabled for that column kind). Applies to both - numerical and categorical pipelines. + If True, append a binary (0/1) missing-value indicator column for each imputed feature + that has missing values at ``fit`` (scikit-learn's ``MissingIndicator``, computed on the raw + input next to the representation, so it never passes through the scaler or basis; a + standalone :class:`~pretab.transformers.MissingStateIndicator` is used instead when + imputation is disabled for that column kind). Applies to both numerical and categorical + pipelines. missing_policy : {"error", "propagate", "impute", "impute_with_indicator", "separate_state"} or None, default=None High-level missing-value strategy. ``None`` (default) keeps the explicit ``numerical_imputation`` / ``categorical_imputation`` / ``add_missing_indicator`` @@ -232,7 +235,8 @@ class Preprocessor(TransformerMixin, BaseEstimator): - ``"propagate"`` -- disable imputation so NaNs reach the transformers unchanged (each family applies its own missing-value contract). - ``"impute"`` -- impute with the configured strategy (no indicator). - - ``"impute_with_indicator"`` -- impute and append a missing indicator column. + - ``"impute_with_indicator"`` -- impute and append a raw 0/1 missing indicator column, + as ``add_missing_indicator=True`` does. - ``"separate_state"`` -- impute for the representation *and* emit a dedicated ``__missing`` column per feature that stays outside the ordinary basis, so a downstream model can learn a separate response to missingness. @@ -1161,7 +1165,7 @@ def _log_internal_decisions(self): """Log fitted internal decisions (bins / knots / centers) at DEBUG.""" for step_name, transformer, columns in self.column_transformer_.transformers_: name = block_name(step_name, columns) - last_step = transformer.steps[-1][1] if hasattr(transformer, "steps") else transformer + last_step = representation_leaf(transformer) for attr in ( "thresholds_", "knots_", diff --git a/tests/integration/test_missing_policy.py b/tests/integration/test_missing_policy.py index c7ec70e..a85e1ef 100644 --- a/tests/integration/test_missing_policy.py +++ b/tests/integration/test_missing_policy.py @@ -1,5 +1,7 @@ """Tests for the high-level ``missing_policy`` orchestration knob (P8.5).""" +import json + import numpy as np import pandas as pd import pytest @@ -262,3 +264,108 @@ def test_add_missing_indicator_no_longer_requires_imputation_enabled(frame_with_ assert out.size > 0 assert np.isnan(out).any() assert p.total_output_dim_ > 0 + + +# --- imputer indicator stays outside the representation (issue #62) ----------- + + +def _indicator_values(pre, frame, name): + names = list(pre.get_feature_names_out()) + return np.asarray(pre.transform(frame, return_array=True), dtype=float)[:, names.index(name)] + + +@pytest.mark.parametrize("options", [{"add_missing_indicator": True}, {"missing_policy": "impute_with_indicator"}]) +def test_imputer_indicator_is_a_raw_binary_column(frame_with_nan, y, options): + """Regression guard for issue #62: the indicator was standardized / expanded + by the representation like a second input feature.""" + pre = Preprocessor(numerical_method="standardization", **options).fit(frame_with_nan, y) + values = _indicator_values(pre, frame_with_nan, "num_a__missingindicator_a") + np.testing.assert_array_equal(values, frame_with_nan["a"].isna().astype(float)) + + +@pytest.mark.parametrize("method", ["bspline", "rbf", "pspline", "fourier", "cubicspline", "polynomial"]) +def test_imputer_indicator_adds_exactly_one_column_per_feature_with_missing(frame_with_nan, y, method): + plain = Preprocessor(numerical_method=method, output_dim=5, missing_policy="impute").fit(frame_with_nan, y) + pre = Preprocessor(numerical_method=method, output_dim=5, missing_policy="impute_with_indicator").fit( + frame_with_nan, y + ) + names = list(pre.get_feature_names_out()) + indicators = [name for name in names if "missingindicator" in name] + assert indicators == ["num_a__missingindicator_a", "num_b__missingindicator_b"] + assert [name for name in names if name not in indicators] == list(plain.get_feature_names_out()) + assert pre.output_dims_ == {key: width + 1 for key, width in plain.output_dims_.items()} + + +@pytest.mark.parametrize("method", ["bspline", "rbf", "pspline", "fourier", "cubicspline"]) +def test_imputer_indicator_lineage_and_feature_info(frame_with_nan, y, method): + pre = Preprocessor(numerical_method=method, output_dim=5, add_missing_indicator=True).fit(frame_with_nan, y) + lineage = pre.get_feature_lineage() + assert [record.output_feature for record in lineage] == list(pre.get_feature_names_out()) + missing = [record for record in lineage if record.family == "missing_state"] + assert [record.output_feature for record in missing] == ["num_a__missingindicator_a", "num_b__missingindicator_b"] + numerical, _, _ = pre.get_feature_info(verbose=False) + assert {feature: info["dimension"] for feature, info in numerical.items()} == pre.output_dims_ + + +def test_imputer_indicator_on_categorical_integer_codes(y): + frame = pd.DataFrame({"c": ["x", "y", np.nan, "x", "y", "x"]}) + pre = Preprocessor(categorical_method="int", add_missing_indicator=True).fit(frame, y) + assert list(pre.get_feature_names_out()) == ["cat_c", "cat_c__missingindicator_c"] + np.testing.assert_array_equal(_indicator_values(pre, frame, "cat_c__missingindicator_c"), frame["c"].isna()) + + +def test_imputer_indicator_only_marks_features_with_missing_values_at_fit(clean_frame, frame_with_nan, y): + pre = Preprocessor(numerical_method="minmax", add_missing_indicator=True).fit(clean_frame, y) + assert not any("missing" in name for name in pre.get_feature_names_out()) + assert np.asarray(pre.transform(frame_with_nan, return_array=True)).shape == (6, 2) + + +# --- representation summary of missing-indicator / separate-state blocks -------- + + +@pytest.mark.parametrize( + "options", + [ + {"add_missing_indicator": True}, + {"missing_policy": "impute_with_indicator"}, + {"missing_policy": "separate_state"}, + ], +) +def test_missing_state_blocks_report_their_representation(frame_with_nan, y, options): + """The spec summary read each block's last step, which for these blocks is the + FeatureUnion of the representation and the missing indicator, so to_spec() and + reproducibility_report() listed no representation for them.""" + plain = _bspline(missing_policy="impute").fit(frame_with_nan, y) + pre = _bspline(**options).fit(frame_with_nan, y) + + assert pre.reproducibility_report()["representations"] == {"a": "bspline", "b": "bspline"} + assert pre.to_spec()["representations"] == plain.to_spec()["representations"] + + +def test_block_with_the_imputer_built_in_indicator_still_loads_and_reports(frame_with_nan, y): + """A preprocessor fitted by PreTab 1.0.0 with add_missing_indicator has the + imputer's built-in indicator in front of the representation (a pipeline, not + a union). Such an object still transforms, round-trips through its spec with + the same fingerprint, and lists its representations.""" + from sklearn.pipeline import Pipeline + + pre = _bspline(add_missing_indicator=True).fit(frame_with_nan, y) + + def legacy_block(transformer, columns): + if not isinstance(transformer, Pipeline): + transformer = dict(transformer.transformer_list)["representation"] + block = clone(transformer) + assert isinstance(block, Pipeline) + return block.set_params(imputer__add_indicator=True).fit(frame_with_nan[columns], y) + + pre.column_transformer_.transformers_ = [ + (name, legacy_block(transformer, columns), columns) + for name, transformer, columns in pre.column_transformer_.transformers_ + ] + out = np.asarray(pre.transform(frame_with_nan, return_array=True)) + restored = Preprocessor.from_spec(json.loads(json.dumps(pre.to_spec()))) + + assert out.shape == (6, 2 * 2 * 8) # the old layout expands the indicator too + np.testing.assert_array_equal(np.asarray(restored.transform(frame_with_nan, return_array=True)), out) + assert restored.fingerprint_ == pre.fingerprint_ + assert restored.reproducibility_report()["representations"] == {"a": "bspline", "b": "bspline"} diff --git a/tests/integration/test_pandas_missing.py b/tests/integration/test_pandas_missing.py index cfe9750..0cf2411 100644 --- a/tests/integration/test_pandas_missing.py +++ b/tests/integration/test_pandas_missing.py @@ -16,9 +16,14 @@ def _frame(): @pytest.mark.parametrize("categorical_method", ["int", "one-hot"]) @pytest.mark.parametrize("impute", [True, False]) -def test_nullable_categorical_fit_matches_nan(categorical_method, impute): +@pytest.mark.parametrize("add_missing_indicator", [True, False]) +def test_nullable_categorical_fit_matches_nan(categorical_method, impute, add_missing_indicator): frame = _frame() - options: dict[str, Any] = {"numerical_method": "minmax", "categorical_method": categorical_method} + options: dict[str, Any] = { + "numerical_method": "minmax", + "categorical_method": categorical_method, + "add_missing_indicator": add_missing_indicator, + } if not impute: options.update(numerical_imputation=None, categorical_imputation=None) expected = Preprocessor(**options).fit(frame) @@ -31,7 +36,15 @@ def test_nullable_categorical_fit_matches_nan(categorical_method, impute): ) -@pytest.mark.parametrize("missing_options", [{}, {"add_missing_indicator": True}, {"missing_policy": "separate_state"}]) +@pytest.mark.parametrize( + "missing_options", + [ + {}, + {"add_missing_indicator": True}, + {"missing_policy": "impute_with_indicator"}, + {"missing_policy": "separate_state"}, + ], +) def test_object_none_imputation_matches_nan(missing_options): reference = _frame() frame = reference.copy() @@ -47,7 +60,7 @@ def test_object_none_imputation_matches_nan(missing_options): @pytest.mark.parametrize("categorical_method", ["int", "one-hot"]) -@pytest.mark.parametrize("missing_policy", [None, "propagate", "separate_state"]) +@pytest.mark.parametrize("missing_policy", [None, "propagate", "impute_with_indicator", "separate_state"]) def test_mixed_categorical_missing_matches_nan(categorical_method, missing_policy): frame = pd.DataFrame({"c": ["b", None, "a", pd.NA, np.nan, "a"]}, index=pd.Index([7, 2, 19, 4, 30, 1])) original = frame.copy(deep=True) @@ -64,6 +77,28 @@ def test_mixed_categorical_missing_matches_nan(categorical_method, missing_polic pd.testing.assert_frame_equal(frame, original) +@pytest.mark.parametrize("categorical_method", ["int", "one-hot"]) +@pytest.mark.parametrize( + "missing_options", [{"add_missing_indicator": True}, {"missing_policy": "impute_with_indicator"}] +) +def test_categorical_indicator_matches_raw_missing_at_transform(categorical_method, missing_options): + training = pd.DataFrame({"c": ["a", None, "b", pd.NA, np.nan]}, dtype=object) + frame = pd.DataFrame({"c": [pd.NA, None, "b", "unseen", np.nan]}, dtype=object) + original = frame.copy(deep=True) + fitted = Preprocessor(categorical_method=categorical_method, **missing_options).fit(training) + names = fitted.get_feature_names_out().tolist() + indicator_name = "cat_c__missingindicator_c" + assert names.count(indicator_name) == 1 + + transformed = np.asarray(fitted.transform(frame), dtype=float) + + np.testing.assert_array_equal(transformed[:, names.index(indicator_name)], frame["c"].isna().astype(float)) + assert transformed.shape == (len(frame), len(names)) + missing = [record.output_feature for record in fitted.get_feature_lineage() if record.family == "missing_state"] + assert missing == [indicator_name] + pd.testing.assert_frame_equal(frame, original) + + @pytest.mark.parametrize("dtype", ["Float64", "Int64"]) def test_nullable_numerical_imputation_matches_nan(dtype): reference = pd.DataFrame({"x": [1.0, np.nan, 3.0, 5.0], "c": ["a", "b", "a", "b"]}, index=pd.Index([8, 2, 6, 1])) diff --git a/tests/integration/test_serialization.py b/tests/integration/test_serialization.py index fb9a745..eb9e46b 100644 --- a/tests/integration/test_serialization.py +++ b/tests/integration/test_serialization.py @@ -169,12 +169,13 @@ def test_round_trip_reproduces_unseen_category_encoding(frame, target): @pytest.mark.parametrize("categorical_method", ["int", "one-hot"]) @pytest.mark.parametrize("impute", [True, False]) -def test_round_trip_with_nullable_categorical_missing(categorical_method, impute): +@pytest.mark.parametrize("add_missing_indicator", [True, False]) +def test_round_trip_with_nullable_categorical_missing(categorical_method, impute, add_missing_indicator): frame = pd.DataFrame({"c": ["a", None, "b", "a"]}).convert_dtypes() options = {"categorical_method": categorical_method} if not impute: options["categorical_imputation"] = None - fitted = Preprocessor(**options).fit(frame) + fitted = Preprocessor(add_missing_indicator=add_missing_indicator, **options).fit(frame) original_fingerprint = fitted.fingerprint_ spec = json.loads(json.dumps(fitted.to_spec())) restored = Preprocessor.from_spec(spec) diff --git a/tests/integration/test_verbosity.py b/tests/integration/test_verbosity.py index eaba720..8783435 100644 --- a/tests/integration/test_verbosity.py +++ b/tests/integration/test_verbosity.py @@ -120,12 +120,33 @@ def test_verbose_3_logs_internal_decisions(sample_data, caplog): assert "thresholds_" in debug_text or "total_output_dim_" in debug_text -def test_verbose_3_logs_internal_decisions_under_the_block_name(caplog): +@pytest.mark.parametrize("options", [{"add_missing_indicator": True}, {"missing_policy": "separate_state"}]) +def test_verbose_3_logs_internal_decisions_of_missing_state_blocks(sample_data, caplog, options): + """These blocks are a FeatureUnion of the representation and the missing + indicator; the union itself has no fitted thresholds, so none were logged.""" + X, y = sample_data + caplog.set_level(logging.DEBUG, logger="pretab") + Preprocessor(numerical_method="ple", verbose=3, **options).fit(X, y) + debug_text = "\n".join(r.getMessage() for r in caplog.records if r.levelno == logging.DEBUG) + assert "num_num1.thresholds_" in debug_text + + +@pytest.mark.parametrize( + "options", + [ + {}, + {"add_missing_indicator": True}, + {"missing_policy": "impute_with_indicator"}, + {"missing_policy": "separate_state"}, + ], +) +def test_verbose_3_logs_internal_decisions_under_the_block_name(caplog, options): """Issue #71: a label that cannot be part of a step name is logged by its block name.""" rng = np.random.default_rng(0) X = pd.DataFrame({"_score": rng.normal(size=60)}) + X.iloc[0, 0] = np.nan caplog.set_level(logging.DEBUG, logger="pretab") - Preprocessor(numerical_method="ple", verbose=3).fit(X, rng.normal(size=60)) + Preprocessor(numerical_method="ple", verbose=3, **options).fit(X, rng.normal(size=60)) debug_text = "\n".join(r.getMessage() for r in caplog.records if r.levelno == logging.DEBUG) assert "num__score.total_output_dim_" in debug_text assert "num_col0" not in debug_text