diff --git a/CHANGELOG.md b/CHANGELOG.md index d1f61a8..ff77cfa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,10 @@ adheres to [Semantic Versioning](https://semver.org/). - A dependency-optional Great Expectations recipe demonstrating the repair-then-validate workflow with an in-memory checkpoint. +### Changed +- `explain_clean()` now profiles only post-clean columns that can contribute a + decision narrative, avoiding a redundant full-width context pass. + ## [2.0.0] - 2026-07-20 Remediation of the July 2026 v1.2.0 production-readiness audit: the unsafe diff --git a/src/freshdata/explain.py b/src/freshdata/explain.py index 72ed52b..d705057 100644 --- a/src/freshdata/explain.py +++ b/src/freshdata/explain.py @@ -115,6 +115,24 @@ def _narratives( return lines +def _narrative_contexts( + df: pd.DataFrame, + config: CleanConfig, + actions: list[Action], +) -> dict: + """Profile only columns that can contribute an explanation narrative.""" + needed = { + action.column + for action in actions + if action.column is not None and action.rationale + } + needed.update( + str(col) for col, has_missing in df.isna().any().items() if has_missing + ) + columns = [col for col in df.columns if str(col) in needed] + return build_contexts(df, config, columns=columns) if columns else {} + + @dataclass class ExplainReport(HtmlReprMixin): """Structured explanation of a clean() run.""" @@ -199,9 +217,10 @@ def explain_clean( cleaned, report = run_pipeline(df, cfg) roles_df = infer_roles(df, config=cfg) + actions = list(report) actions_by_step: dict[str, list[dict[str, Any]]] = defaultdict(list) - for action in report: + for action in actions: actions_by_step[action.step].append({ "column": action.column, "description": action.description, @@ -212,7 +231,7 @@ def explain_clean( "model_id": action.model_id, }) - post_contexts = build_contexts(cleaned, cfg) + post_contexts = _narrative_contexts(cleaned, cfg, actions) return ExplainReport( strategy=cfg.strategy, rows_before=len(df), @@ -223,7 +242,7 @@ def explain_clean( after_stats=_column_stats(cleaned), cell_changes=_cell_changes(df, cleaned), actions_by_step=dict(actions_by_step), - narratives=_narratives(post_contexts, list(report), strategy=cfg.strategy), + narratives=_narratives(post_contexts, actions, strategy=cfg.strategy), report=report, roles=roles_df, ) diff --git a/tests/test_explain.py b/tests/test_explain.py index 714cc6f..114ab30 100644 --- a/tests/test_explain.py +++ b/tests/test_explain.py @@ -2,11 +2,14 @@ from __future__ import annotations +from unittest.mock import patch + import pandas as pd import pytest import freshdata as fd from expectations import ALL_ONLINE_TIER1, load_online_fixture +from freshdata.engine.context import build_context from freshdata.explain import _cell_changes @@ -34,6 +37,21 @@ def test_explain_clean_narratives_on_missing(): assert explanation.narratives or explanation.report.actions +def test_explain_clean_profiles_wide_noop_frame_once(): + """The post-clean narrative pass must not re-profile irrelevant columns.""" + n_columns = 120 + df = pd.DataFrame({ + f"feature_{col}": [f"value-{row}-{col}" for row in range(40)] + for col in range(n_columns) + }) + + with patch("freshdata.engine.context.build_context", wraps=build_context) as mock: + explanation = fd.explain_clean(df, strategy="balanced", verbose=False) + + assert explanation.narratives == [] + assert mock.call_count == n_columns + + @pytest.mark.parametrize("name", ALL_ONLINE_TIER1[:3]) def test_explain_clean_on_online_fixtures(name): df = load_online_fixture(name)