Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,10 @@ adheres to [Semantic Versioning](https://semver.org/).
- A dependency-optional Great Expectations recipe demonstrating the
repair-then-validate workflow with an in-memory checkpoint.

### Changed
- `explain_clean()` now profiles only post-clean columns that can contribute a
decision narrative, avoiding a redundant full-width context pass.

## [2.0.0] - 2026-07-20

Remediation of the July 2026 v1.2.0 production-readiness audit: the unsafe
Expand Down
25 changes: 22 additions & 3 deletions src/freshdata/explain.py
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,24 @@ def _narratives(
return lines


def _narrative_contexts(
df: pd.DataFrame,
config: CleanConfig,
actions: list[Action],
) -> dict:
"""Profile only columns that can contribute an explanation narrative."""
needed = {
action.column
for action in actions
if action.column is not None and action.rationale
}
needed.update(
str(col) for col, has_missing in df.isna().any().items() if has_missing
)
columns = [col for col in df.columns if str(col) in needed]
return build_contexts(df, config, columns=columns) if columns else {}


@dataclass
class ExplainReport(HtmlReprMixin):
"""Structured explanation of a clean() run."""
Expand Down Expand Up @@ -199,9 +217,10 @@ def explain_clean(
cleaned, report = run_pipeline(df, cfg)

roles_df = infer_roles(df, config=cfg)
actions = list(report)

actions_by_step: dict[str, list[dict[str, Any]]] = defaultdict(list)
for action in report:
for action in actions:
actions_by_step[action.step].append({
"column": action.column,
"description": action.description,
Expand All @@ -212,7 +231,7 @@ def explain_clean(
"model_id": action.model_id,
})

post_contexts = build_contexts(cleaned, cfg)
post_contexts = _narrative_contexts(cleaned, cfg, actions)
return ExplainReport(
strategy=cfg.strategy,
rows_before=len(df),
Expand All @@ -223,7 +242,7 @@ def explain_clean(
after_stats=_column_stats(cleaned),
cell_changes=_cell_changes(df, cleaned),
actions_by_step=dict(actions_by_step),
narratives=_narratives(post_contexts, list(report), strategy=cfg.strategy),
narratives=_narratives(post_contexts, actions, strategy=cfg.strategy),
report=report,
roles=roles_df,
)
18 changes: 18 additions & 0 deletions tests/test_explain.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,14 @@

from __future__ import annotations

from unittest.mock import patch

import pandas as pd
import pytest

import freshdata as fd
from expectations import ALL_ONLINE_TIER1, load_online_fixture
from freshdata.engine.context import build_context
from freshdata.explain import _cell_changes


Expand Down Expand Up @@ -34,6 +37,21 @@ def test_explain_clean_narratives_on_missing():
assert explanation.narratives or explanation.report.actions


def test_explain_clean_profiles_wide_noop_frame_once():
"""The post-clean narrative pass must not re-profile irrelevant columns."""
n_columns = 120
df = pd.DataFrame({
f"feature_{col}": [f"value-{row}-{col}" for row in range(40)]
for col in range(n_columns)
})

with patch("freshdata.engine.context.build_context", wraps=build_context) as mock:
explanation = fd.explain_clean(df, strategy="balanced", verbose=False)

assert explanation.narratives == []
assert mock.call_count == n_columns


@pytest.mark.parametrize("name", ALL_ONLINE_TIER1[:3])
def test_explain_clean_on_online_fixtures(name):
df = load_online_fixture(name)
Expand Down
Loading