Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 5 additions & 13 deletions diffly/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,18 +12,10 @@

from ._compat import typer
from ._utils import ABS_TOL_DEFAULT, ABS_TOL_TEMPORAL_DEFAULT, REL_TOL_DEFAULT
from .metrics import Metric, MetricFn
from .metrics.change import DEFAULT_CHANGE_METRICS
from .metrics.data import DEFAULT_DATA_METRICS
from .metrics import ALL_METRICS

app = typer.Typer()

#: All metric presets selectable via ``--metric``, combining the change and data sets.
AVAILABLE_METRICS: dict[str, MetricFn | Metric] = {
**DEFAULT_CHANGE_METRICS,
**DEFAULT_DATA_METRICS,
}


@app.command()
def main(
Expand Down Expand Up @@ -148,7 +140,7 @@ def main(
typer.Option(
help=(
"Metric presets to display per column. Repeatable. "
f"Available: {', '.join(AVAILABLE_METRICS)}."
f"Available: {', '.join(ALL_METRICS)}."
)
),
Comment thread
MoritzPotthoffQC marked this conversation as resolved.
] = [],
Expand All @@ -162,11 +154,11 @@ def main(
hidden_column = [*hidden_column, *hidden_columns]

for name in metric:
if name not in AVAILABLE_METRICS:
if name not in ALL_METRICS:
raise typer.BadParameter(
f"Unknown metric: {name!r}. Available: {', '.join(AVAILABLE_METRICS)}."
f"Unknown metric: {name!r}. Available: {', '.join(ALL_METRICS)}."
)
metrics = {name: AVAILABLE_METRICS[name] for name in metric}
metrics = {name: ALL_METRICS[name] for name in metric}

comparison = compare_frames(
pl.scan_parquet(left),
Expand Down
42 changes: 25 additions & 17 deletions diffly/comparison.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,11 @@
from __future__ import annotations

import datetime as dt
import inspect
import warnings
from collections.abc import Iterable, Mapping, Sequence
from functools import cached_property
from typing import TYPE_CHECKING, Literal, Self, overload
from typing import TYPE_CHECKING, Literal, Self, cast, overload

import polars as pl
from polars.schema import Schema as PolarsSchema
Expand All @@ -25,7 +26,9 @@
lazy_len,
make_and_validate_mapping,
)
from .metrics import Metric, MetricFn, _make_numeric_metric
from .metrics._common import Metric
from .metrics.change import ChangeMetric, ChangeMetricFn
from .metrics.data import DataMetric, DataMetricFn

if TYPE_CHECKING: # pragma: no cover
# NOTE: We cannot import at runtime as we're otherwise running into circular
Expand Down Expand Up @@ -920,7 +923,7 @@ def summary(
right_name: str = Side.RIGHT,
slim: bool = False,
hidden_columns: list[str] | None = None,
metrics: Mapping[str, MetricFn | Metric] | None = None,
metrics: Mapping[str, ChangeMetricFn | DataMetricFn | Metric] | None = None,
) -> Summary:
"""Generate a summary of all aspects of the comparison.

Expand Down Expand Up @@ -951,16 +954,14 @@ def summary(
hidden_columns: Columns for which no values are printed, e.g. because they
contain sensitive information.
metrics: Optional mapping from display label to a metric. A value may be a
callable ``(left_expr, right_expr) -> pl.Expr`` or a
:class:`~diffly.metrics.Metric`. Each callable receives two
:class:`polars.Expr` referring to the left and right values of a single
column across all joined rows, and must return a scalar aggregation
expression. Bare callables are only computed for numerical columns; wrap
one in a :class:`~diffly.metrics.Metric` with a column selector to target
other column types (e.g. ``Metric(fn, selector=cs.all())``).
See :doc:`/api/metrics` for the full list of presets and the
:data:`~diffly.metrics.MetricFn` type. When ``None`` (default), no metrics
are computed; presets are not applied automatically. Prefer short labels —
:class:`~diffly.metrics.change.ChangeMetric`, a
:class:`~diffly.metrics.data.DataMetric`, or a bare callable resolved
by its arity (two arguments → change metric on numerical columns, one
argument → data metric on all columns). To target other column types,
construct the metric explicitly with a column selector
(e.g. ``ChangeMetric(fn, selector=cs.numeric())``). See :doc:`/api/metrics`
for the full list of presets. When ``None`` (default), no metrics are
computed; presets are not applied automatically. Prefer short labels —
the summary has a fixed width and many or long labels degrade rendering.

Returns:
Expand All @@ -978,10 +979,7 @@ def summary(
from .summary import Summary

resolved_metrics = (
{
label: v if isinstance(v, Metric) else _make_numeric_metric(v)
for label, v in metrics.items()
}
{label: _resolve_metric(v) for label, v in metrics.items()}
if metrics is not None
else None
)
Expand Down Expand Up @@ -1239,3 +1237,13 @@ def _list_length_exprs(
for e in _list_length_exprs(expr.struct[field.name], field.dtype)
]
return []


def _resolve_metric(v: ChangeMetricFn | DataMetricFn | Metric) -> Metric:
if isinstance(v, (ChangeMetric, DataMetric)):

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

?

Suggested change
if isinstance(v, (ChangeMetric, DataMetric)):
if isinstance(v, Metric):

return v
# Infer the metric family from the callable's arity: a single-argument
# callable describes one side (data), two arguments describe a change.
if len(inspect.signature(v).parameters) >= 2:

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why not:

Suggested change
if len(inspect.signature(v).parameters) >= 2:
if isinstance(v, ChangeMetricFn):

Also, how about being a bit defensive here and do:

	elif isinstance(v, DataMetricFn):
		...
	else:
		raise

so that the signature issue is easier to understand for the user?

return ChangeMetric(fn=cast(ChangeMetricFn, v))
return DataMetric(fn=cast(DataMetricFn, v))
Comment thread
MoritzPotthoffQC marked this conversation as resolved.
44 changes: 6 additions & 38 deletions diffly/metrics/__init__.py
Original file line number Diff line number Diff line change
@@ -1,50 +1,18 @@
# Copyright (c) QuantCo 2025-2026
# SPDX-License-Identifier: BSD-3-Clause

"""Metrics computed per column when generating a summary.

Two families are provided:

- Metrics in :mod:`~diffly.metrics.change` describe the change between numeric
columns itself by aggregating over ``right - left``.
- Metrics in :mod:`~diffly.metrics.data` describe the left and right datasets
individually, explaining how a change affects the data.
- :class:`~diffly.metrics.change.ChangeMetric`s in :mod:`~diffly.metrics.change` describe the change between
numeric columns itself by aggregating over ``right - left``.
- :class:`~diffly.metrics.data.DataMetric`s in :mod:`~diffly.metrics.data` describe the left and right
datasets individually, explaining how a change affects the data.
"""

from __future__ import annotations

from . import change, data
from ._common import Metric, MetricFn
from .change import (
_make_numeric_metric,
max,
mean,
mean_absolute_deviation,
mean_relative_deviation,
median,
min,
quantile,
std,
)

DEFAULT_METRICS: dict[str, MetricFn | Metric] = {
**change.DEFAULT_CHANGE_METRICS,
}
"""The default preset metrics, consisting of the change default set."""
from ._common import ALL_METRICS

__all__ = [
"DEFAULT_METRICS",
"Metric",
"MetricFn",
"change",
"data",
"max",
"mean",
"mean_absolute_deviation",
"mean_relative_deviation",
"median",
"min",
"quantile",
"std",
"_make_numeric_metric",
]
__all__ = ["ALL_METRICS", "change", "data"]
31 changes: 9 additions & 22 deletions diffly/metrics/_common.py
Original file line number Diff line number Diff line change
@@ -1,27 +1,14 @@
# Copyright (c) QuantCo 2025-2026
# SPDX-License-Identifier: BSD-3-Clause

from __future__ import annotations
from .change import DEFAULT_CHANGE_METRICS, ChangeMetric
from .data import DEFAULT_DATA_METRICS, DataMetric

from collections.abc import Callable
from dataclasses import dataclass
Metric = ChangeMetric | DataMetric

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We use ChangeMetricFn | DataMetricFn in a few places. For comfort, how about also defining

MetricFn = ChangeMetricFn | DataMetricFn

?

"""A change or data metric paired with a column-applicability selector."""

import polars as pl
import polars.selectors as cs


@dataclass(frozen=True)
class Metric:
"""A metric function paired with a column-applicability selector."""

fn: MetricFn
selector: cs.Selector


MetricFn = Callable[[pl.Expr, pl.Expr], pl.Expr]
"""A metric function maps ``(left_expr, right_expr)`` to a scalar aggregation
expression.

The expressions refer to the left-side and right-side values of a single column across
all joined rows.
"""
ALL_METRICS: dict[str, Metric] = {

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

?

Suggested change
ALL_METRICS: dict[str, Metric] = {
DEFAULT_METRICS: dict[str, Metric] = {

**DEFAULT_CHANGE_METRICS,
**DEFAULT_DATA_METRICS,
}
"""All preset metrics, combining the change and data default sets."""
50 changes: 33 additions & 17 deletions diffly/metrics/change.py
Original file line number Diff line number Diff line change
@@ -1,21 +1,37 @@
# Copyright (c) QuantCo 2025-2026
# SPDX-License-Identifier: BSD-3-Clause

"""Metrics describing the change between numeric columns.

These aggregate over ``right - left`` to characterize the change itself.
"""

from __future__ import annotations

from collections.abc import Callable
from dataclasses import dataclass, field

import polars as pl
import polars.selectors as cs

from ._common import Metric, MetricFn
ChangeMetricFn = Callable[[pl.Expr, pl.Expr], pl.Expr]
"""A `ChangeMetricFn` maps a pair of column expressions to a scalar aggregation
expression."""


@dataclass(frozen=True)
class ChangeMetric:
"""A metric quantifying the *change* in a column between the two sides of a
comparison.

Change metrics are rendered as extra columns in the "Columns" table, alongside the
match rate.
"""

fn: ChangeMetricFn
"""Aggregates over ``right - left`` (e.g. the mean delta) to describe the change
itself."""

selector: cs.Selector = field(default_factory=cs.numeric)
"""Selects the columns the metric applies to; defaults to numeric columns."""


def _make_numeric_metric(fn: MetricFn) -> Metric:
return Metric(fn=fn, selector=cs.numeric())
# ---------------------------------- CHANGE METRICS ---------------------------------- #


def mean(left: pl.Expr, right: pl.Expr) -> pl.Expr:
Expand Down Expand Up @@ -54,7 +70,7 @@ def mean_relative_deviation(left: pl.Expr, right: pl.Expr) -> pl.Expr:
return ((right - left) / left).abs().mean()


def quantile(q: float) -> MetricFn:
def quantile(q: float) -> ChangeMetricFn:
"""Factory returning a metric that computes the ``q``-quantile of
``right - left``."""
if not 0 <= q <= 1:
Expand All @@ -66,13 +82,13 @@ def _quantile(left: pl.Expr, right: pl.Expr) -> pl.Expr:
return _quantile


DEFAULT_CHANGE_METRICS: dict[str, MetricFn] = {
"Mean": mean,
"Median": median,
"Min": min,
"Max": max,
"Std": std,
"Mean absolute deviation": mean_absolute_deviation,
"Mean relative deviation": mean_relative_deviation,
DEFAULT_CHANGE_METRICS: dict[str, ChangeMetric] = {
"Mean": ChangeMetric(fn=mean),
"Median": ChangeMetric(fn=median),
"Min": ChangeMetric(fn=min),
"Max": ChangeMetric(fn=max),
"Std": ChangeMetric(fn=std),
"Mean absolute deviation": ChangeMetric(fn=mean_absolute_deviation),
"Mean relative deviation": ChangeMetric(fn=mean_relative_deviation),
}
"""Preset metrics describing the change between numeric columns."""
Loading