Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions RELEASES.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@

#### New features

- Lazy empirical Sinkhorn computes Euclidean cost blocks on the input backend, avoiding host transfers during GPU solves with matching sample and weight types.


- `ot.dr.wda` gains `solver='torch'`, a PyTorch autodiff solver with Riemannian gradient descent, usable on installations without autograd or pymanopt, and accepting torch tensors directly (PR #858, Issue #806)
- `ot.dr` dependencies (autograd, pymanopt, scikit-learn, torch) are now imported optionally, so importing `ot.dr` no longer requires all of them; each function raises an explicit `ImportError` naming what it needs (PR #858)
Expand Down
47 changes: 34 additions & 13 deletions ot/bregman/_empirical.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
import warnings

from ..utils import dist, list_to_array, unif, LazyTensor
from ..backend import get_backend
from ..backend import get_backend, torch

from ._sinkhorn import sinkhorn, sinkhorn2
from ..lowrank import (
Expand Down Expand Up @@ -71,6 +71,31 @@ def func(i, j, X_a, X_b, f, g, metric, reg):
return T


def _lazy_cost_blocks(X_s, X_t, metric, a, nx):
# Keep the dual iteration detached from the samples as in the NumPy path.
if (
metric in ("sqeuclidean", "euclidean")
and nx.dtype_device(X_s) == nx.dtype_device(X_t) == nx.dtype_device(a)
):
X_s, X_t = nx.detach(X_s, X_t)

def cost(rows, cols):
return dist(X_s[rows], X_t[cols], metric=metric, nx=nx)

if nx.__name__ == "torch" and X_s.device.type in ("cpu", "cuda"):
# Autocast can overflow the dot product before the squared norms cancel.
cost = torch.autocast(X_s.device.type, enabled=False)(cost)

else:
# Other metrics can require scipy, and the cost must match the weights.
X_s, X_t = nx.to_numpy(X_s, X_t)

def cost(rows, cols):
return nx.from_numpy(dist(X_s[rows], X_t[cols], metric=metric), type_as=a)

return cost


def empirical_sinkhorn(
X_s,
X_t,
Expand Down Expand Up @@ -132,6 +157,8 @@ def empirical_sinkhorn(
If True, then only calculate the cost matrix by block and return
the dual potentials only (to save memory). If False, calculate full
cost matrix and return outputs of sinkhorn function.
Euclidean and squared Euclidean cost blocks are computed on the input
backend when samples and source weights share a dtype and device.
batchSize: int or tuple of 2 int, optional
Size of the batches used to compute the sinkhorn update without memory overhead.
When a tuple is provided it sets the size of the left/right batches.
Expand Down Expand Up @@ -212,31 +239,27 @@ def empirical_sinkhorn(
lse_f = nx.zeros((ns,), type_as=a)
lse_g = nx.zeros((nt,), type_as=a)

X_s_np = nx.to_numpy(X_s)
X_t_np = nx.to_numpy(X_t)
cost = _lazy_cost_blocks(X_s, X_t, metric, a, nx)

for i_ot in range(numIterMax):
lse_f_cols = []
for i in range_s:
M = dist(X_s_np[i : i + bs, :], X_t_np, metric=metric)
M = nx.from_numpy(M, type_as=a)
M = cost(slice(i, i + bs), slice(None))
lse_f_cols.append(nx.logsumexp(g[None, :] - M / reg, axis=1))
lse_f = nx.concatenate(lse_f_cols, axis=0)
f = log_a - lse_f

lse_g_cols = []
for j in range_t:
M = dist(X_s_np, X_t_np[j : j + bt, :], metric=metric)
M = nx.from_numpy(M, type_as=a)
M = cost(slice(None), slice(j, j + bt))
lse_g_cols.append(nx.logsumexp(f[:, None] - M / reg, axis=0))
lse_g = nx.concatenate(lse_g_cols, axis=0)
g = log_b - lse_g

if (i_ot + 1) % 10 == 0:
m1_cols = []
for i in range_s:
M = dist(X_s_np[i : i + bs, :], X_t_np, metric=metric)
M = nx.from_numpy(M, type_as=a)
M = cost(slice(i, i + bs), slice(None))
m1_cols.append(
nx.sum(
nx.exp(f[i : i + bs, None] + g[None, :] - M / reg), axis=1
Expand Down Expand Up @@ -468,12 +491,10 @@ def empirical_sinkhorn2(

loss = 0

X_s_np = nx.to_numpy(X_s)
X_t_np = nx.to_numpy(X_t)
cost = _lazy_cost_blocks(X_s, X_t, metric, a, nx)

for i in range_s:
M_block = dist(X_s_np[i : i + bs, :], X_t_np, metric=metric)
M_block = nx.from_numpy(M_block, type_as=a)
M_block = cost(slice(i, i + bs), slice(None))
pi_block = nx.exp(f[i : i + bs, None] + g[None, :] - M_block / reg)
loss += nx.sum(M_block * pi_block)

Expand Down
45 changes: 45 additions & 0 deletions test/test_bregman.py
Original file line number Diff line number Diff line change
Expand Up @@ -1194,6 +1194,51 @@ def test_geomloss_solver(nx, metric):
ot.bregman.empirical_sinkhorn2_geomloss(X_sb, X_tb, 1, metric="wrong_metric")


@pytest.mark.skipif(not torch, reason="requires torch")
@pytest.mark.parametrize("metric", ["sqeuclidean", "euclidean"])
@pytest.mark.parametrize("device", ["cpu", "cuda"])
def test_lazy_euclidean_cost_on_device(monkeypatch, metric, device):
if device == "cuda" and not torch.cuda.is_available():
pytest.skip("CUDA unavailable")

rng = np.random.RandomState(42)
xs, xt = rng.randn(7, 2), rng.randn(5, 2)
expected = ot.bregman.empirical_sinkhorn2(xs, xt, 2, metric=metric)
xs, xt = (torch.tensor(x, device=device) for x in (xs, xt))
nx = ot.backend.get_backend(xs)

def no_host_copy(*args):
pytest.fail("Euclidean cost blocks should stay on the input backend")

monkeypatch.setattr(nx, "to_numpy", no_host_copy)
result = ot.solve_sample(xs, xt, reg=2, metric=metric, lazy=True, batch_size=3)
torch.testing.assert_close(result.value_linear, xs.new_tensor(expected))


@pytest.mark.skipif(not torch, reason="requires torch")
@pytest.mark.parametrize("device", ["cpu", "cuda"])
def test_lazy_empirical_sinkhorn_autocast(device):
if device == "cuda" and not torch.cuda.is_available():
pytest.skip("CUDA unavailable")
xs = 200 + torch.arange(14, dtype=torch.float32, device=device).reshape(7, 2) / 8
xt = xs[:5] - 0.25
kwargs = dict(reg=2, isLazy=True, batchSize=3, stopThr=1e-5, numIterMax=100)
expected = ot.bregman.empirical_sinkhorn2(xs, xt, **kwargs)
with torch.autocast(device):
actual = ot.bregman.empirical_sinkhorn2(xs, xt, **kwargs)
torch.testing.assert_close(actual, expected)


@pytest.mark.skipif(not torch, reason="requires torch")
def test_lazy_empirical_sinkhorn_scipy_metric():
rng = np.random.RandomState(42)
xs, xt = rng.randn(7, 2), rng.randn(5, 2)
kwargs = dict(reg=2, metric="chebyshev", isLazy=True, batchSize=3)
expected = ot.bregman.empirical_sinkhorn2(xs, xt, **kwargs)
loss = ot.bregman.empirical_sinkhorn2(torch.tensor(xs), torch.tensor(xt), **kwargs)
torch.testing.assert_close(loss, torch.tensor(expected))


def test_lazy_empirical_sinkhorn(nx):
# test sinkhorn
n = 10
Expand Down
Loading