From 1d1b9999e8bb062b5664682a1a7c78c8e57b33dc Mon Sep 17 00:00:00 2001 From: cuteNBD <206630735+bingdongni@users.noreply.github.com> Date: Mon, 16 Nov 2026 08:47:11 +0800 Subject: [PATCH 1/3] Compute lazy Euclidean Sinkhorn costs on the input backend --- RELEASES.md | 2 ++ ot/bregman/_empirical.py | 41 ++++++++++++++++++++++++++++------------ test/test_bregman.py | 31 ++++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 12 deletions(-) diff --git a/RELEASES.md b/RELEASES.md index 20a45aef2..27876acca 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -4,6 +4,8 @@ #### New features +- Lazy empirical Sinkhorn computes Euclidean cost blocks on the input backend, avoiding host transfers during GPU solves with matching sample and weight types. + - `ot.dr.wda` gains `solver='torch'`, a PyTorch autodiff solver with Riemannian gradient descent, usable on installations without autograd or pymanopt, and accepting torch tensors directly (PR #858, Issue #806) - `ot.dr` dependencies (autograd, pymanopt, scikit-learn, torch) are now imported optionally, so importing `ot.dr` no longer requires all of them; each function raises an explicit `ImportError` naming what it needs (PR #858) diff --git a/ot/bregman/_empirical.py b/ot/bregman/_empirical.py index ccc5ac25e..46e5120a3 100644 --- a/ot/bregman/_empirical.py +++ b/ot/bregman/_empirical.py @@ -71,6 +71,27 @@ def func(i, j, X_a, X_b, f, g, metric, reg): return T +def _lazy_cost_blocks(X_s, X_t, metric, a, nx): + # Keep the dual iteration detached from the samples as in the NumPy path. + if ( + metric in ("sqeuclidean", "euclidean") + and nx.dtype_device(X_s) == nx.dtype_device(X_t) == nx.dtype_device(a) + ): + X_s, X_t = nx.detach(X_s, X_t) + + def cost(rows, cols): + return dist(X_s[rows], X_t[cols], metric=metric, nx=nx) + + else: + # Other metrics can require scipy, and the cost must match the weights. + X_s, X_t = nx.to_numpy(X_s, X_t) + + def cost(rows, cols): + return nx.from_numpy(dist(X_s[rows], X_t[cols], metric=metric), type_as=a) + + return cost + + def empirical_sinkhorn( X_s, X_t, @@ -132,6 +153,8 @@ def empirical_sinkhorn( If True, then only calculate the cost matrix by block and return the dual potentials only (to save memory). If False, calculate full cost matrix and return outputs of sinkhorn function. + Euclidean and squared Euclidean cost blocks are computed on the input + backend when samples and source weights share a dtype and device. batchSize: int or tuple of 2 int, optional Size of the batches used to compute the sinkhorn update without memory overhead. When a tuple is provided it sets the size of the left/right batches. @@ -212,22 +235,19 @@ def empirical_sinkhorn( lse_f = nx.zeros((ns,), type_as=a) lse_g = nx.zeros((nt,), type_as=a) - X_s_np = nx.to_numpy(X_s) - X_t_np = nx.to_numpy(X_t) + cost = _lazy_cost_blocks(X_s, X_t, metric, a, nx) for i_ot in range(numIterMax): lse_f_cols = [] for i in range_s: - M = dist(X_s_np[i : i + bs, :], X_t_np, metric=metric) - M = nx.from_numpy(M, type_as=a) + M = cost(slice(i, i + bs), slice(None)) lse_f_cols.append(nx.logsumexp(g[None, :] - M / reg, axis=1)) lse_f = nx.concatenate(lse_f_cols, axis=0) f = log_a - lse_f lse_g_cols = [] for j in range_t: - M = dist(X_s_np, X_t_np[j : j + bt, :], metric=metric) - M = nx.from_numpy(M, type_as=a) + M = cost(slice(None), slice(j, j + bt)) lse_g_cols.append(nx.logsumexp(f[:, None] - M / reg, axis=0)) lse_g = nx.concatenate(lse_g_cols, axis=0) g = log_b - lse_g @@ -235,8 +255,7 @@ def empirical_sinkhorn( if (i_ot + 1) % 10 == 0: m1_cols = [] for i in range_s: - M = dist(X_s_np[i : i + bs, :], X_t_np, metric=metric) - M = nx.from_numpy(M, type_as=a) + M = cost(slice(i, i + bs), slice(None)) m1_cols.append( nx.sum( nx.exp(f[i : i + bs, None] + g[None, :] - M / reg), axis=1 @@ -468,12 +487,10 @@ def empirical_sinkhorn2( loss = 0 - X_s_np = nx.to_numpy(X_s) - X_t_np = nx.to_numpy(X_t) + cost = _lazy_cost_blocks(X_s, X_t, metric, a, nx) for i in range_s: - M_block = dist(X_s_np[i : i + bs, :], X_t_np, metric=metric) - M_block = nx.from_numpy(M_block, type_as=a) + M_block = cost(slice(i, i + bs), slice(None)) pi_block = nx.exp(f[i : i + bs, None] + g[None, :] - M_block / reg) loss += nx.sum(M_block * pi_block) diff --git a/test/test_bregman.py b/test/test_bregman.py index 17b400306..892d5593d 100644 --- a/test/test_bregman.py +++ b/test/test_bregman.py @@ -1194,6 +1194,37 @@ def test_geomloss_solver(nx, metric): ot.bregman.empirical_sinkhorn2_geomloss(X_sb, X_tb, 1, metric="wrong_metric") +@pytest.mark.skipif(not torch, reason="requires torch") +@pytest.mark.parametrize("metric", ["sqeuclidean", "euclidean"]) +@pytest.mark.parametrize("device", ["cpu", "cuda"]) +def test_lazy_euclidean_cost_on_device(monkeypatch, metric, device): + if device == "cuda" and not torch.cuda.is_available(): + pytest.skip("CUDA unavailable") + + rng = np.random.RandomState(42) + xs, xt = rng.randn(7, 2), rng.randn(5, 2) + expected = ot.bregman.empirical_sinkhorn2(xs, xt, 2, metric=metric) + xs, xt = (torch.tensor(x, device=device) for x in (xs, xt)) + nx = ot.backend.get_backend(xs) + + def no_host_copy(*args): + pytest.fail("Euclidean cost blocks should stay on the input backend") + + monkeypatch.setattr(nx, "to_numpy", no_host_copy) + result = ot.solve_sample(xs, xt, reg=2, metric=metric, lazy=True, batch_size=3) + torch.testing.assert_close(result.value_linear, xs.new_tensor(expected)) + + +@pytest.mark.skipif(not torch, reason="requires torch") +def test_lazy_empirical_sinkhorn_scipy_metric(): + rng = np.random.RandomState(42) + xs, xt = rng.randn(7, 2), rng.randn(5, 2) + kwargs = dict(reg=2, metric="chebyshev", isLazy=True, batchSize=3) + expected = ot.bregman.empirical_sinkhorn2(xs, xt, **kwargs) + loss = ot.bregman.empirical_sinkhorn2(torch.tensor(xs), torch.tensor(xt), **kwargs) + torch.testing.assert_close(loss, torch.tensor(expected)) + + def test_lazy_empirical_sinkhorn(nx): # test sinkhorn n = 10 From 11e278f822e27314310d8fe00ef8c736af63cdfd Mon Sep 17 00:00:00 2001 From: cuteNBD <206630735+bingdongni@users.noreply.github.com> Date: Mon, 16 Nov 2026 09:01:49 +0800 Subject: [PATCH 2/3] Preserve the cost dtype inside PyTorch autocast --- ot/bregman/_empirical.py | 6 +++++- test/test_bregman.py | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/ot/bregman/_empirical.py b/ot/bregman/_empirical.py index 46e5120a3..ea68d8890 100644 --- a/ot/bregman/_empirical.py +++ b/ot/bregman/_empirical.py @@ -13,7 +13,7 @@ import warnings from ..utils import dist, list_to_array, unif, LazyTensor -from ..backend import get_backend +from ..backend import get_backend, torch from ._sinkhorn import sinkhorn, sinkhorn2 from ..lowrank import ( @@ -82,6 +82,10 @@ def _lazy_cost_blocks(X_s, X_t, metric, a, nx): def cost(rows, cols): return dist(X_s[rows], X_t[cols], metric=metric, nx=nx) + if nx.__name__ == "torch" and X_s.device.type in ("cpu", "cuda"): + # Autocast can overflow the dot product before the squared norms cancel. + cost = torch.autocast(X_s.device.type, enabled=False)(cost) + else: # Other metrics can require scipy, and the cost must match the weights. X_s, X_t = nx.to_numpy(X_s, X_t) diff --git a/test/test_bregman.py b/test/test_bregman.py index 892d5593d..e0474fad0 100644 --- a/test/test_bregman.py +++ b/test/test_bregman.py @@ -1215,6 +1215,20 @@ def no_host_copy(*args): torch.testing.assert_close(result.value_linear, xs.new_tensor(expected)) +@pytest.mark.skipif(not torch, reason="requires torch") +@pytest.mark.parametrize("device", ["cpu", "cuda"]) +def test_lazy_empirical_sinkhorn_autocast(device): + if device == "cuda" and not torch.cuda.is_available(): + pytest.skip("CUDA unavailable") + xs = 200 + torch.arange(14, dtype=torch.float32, device=device).reshape(7, 2) / 8 + xt = xs[:5] - 0.25 + kwargs = dict(reg=2, isLazy=True, batchSize=3) + expected = ot.bregman.empirical_sinkhorn2(xs, xt, **kwargs) + with torch.autocast(device): + actual = ot.bregman.empirical_sinkhorn2(xs, xt, **kwargs) + torch.testing.assert_close(actual, expected) + + @pytest.mark.skipif(not torch, reason="requires torch") def test_lazy_empirical_sinkhorn_scipy_metric(): rng = np.random.RandomState(42) From be3345eaad3236cf098ee99f67f6003501d7b54b Mon Sep 17 00:00:00 2001 From: cuteNBD <206630735+bingdongni@users.noreply.github.com> Date: Mon, 16 Nov 2026 09:07:42 +0800 Subject: [PATCH 3/3] Use a float32 stopping tolerance in the autocast regression --- test/test_bregman.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_bregman.py b/test/test_bregman.py index e0474fad0..7c9f6c8e6 100644 --- a/test/test_bregman.py +++ b/test/test_bregman.py @@ -1222,7 +1222,7 @@ def test_lazy_empirical_sinkhorn_autocast(device): pytest.skip("CUDA unavailable") xs = 200 + torch.arange(14, dtype=torch.float32, device=device).reshape(7, 2) / 8 xt = xs[:5] - 0.25 - kwargs = dict(reg=2, isLazy=True, batchSize=3) + kwargs = dict(reg=2, isLazy=True, batchSize=3, stopThr=1e-5, numIterMax=100) expected = ot.bregman.empirical_sinkhorn2(xs, xt, **kwargs) with torch.autocast(device): actual = ot.bregman.empirical_sinkhorn2(xs, xt, **kwargs)