From 842f1f1ba76c612b12c55f9f084b4505fae093fc Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Sat, 23 May 2026 19:58:21 +0200 Subject: [PATCH 1/5] perf(load): skip Variable.load dispatch for numpy data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Variable.load() and Variable.load_async() always end with ``self._data = to_duck_array(self._data)`` which, for an in-memory ``numpy.ndarray``, walks the dispatch chain only to return ``self._data`` unchanged. The whole call is pure overhead in that case — the same no-op pattern that ``IndexVariable.load`` already short-circuits. Add an ``isinstance(self._data, np.ndarray)`` guard at the top of both methods. Behavior is unchanged on chunked, ExplicitlyIndexed, or non-numpy duck-array inputs. Measured on ``isel(...).load()`` of synthetic scalar-var datasets against upstream/main (best of 5, GC off): 400 scalar vars: 0.524 ms -> 0.324 ms ~1.62x 2000 scalar vars: 2.484 ms -> 1.490 ms ~1.67x Speedup scales with the number of variables (1.44x at 50 vars -> 1.67x at 2000 vars) and is flat across per-variable data size (~1.56x from size=0 to size=10,000), confirming the saving is pure dispatch overhead removal. Refs #11352. Co-authored-by: Claude --- doc/whats-new.rst | 7 +++++++ xarray/core/variable.py | 8 ++++++++ 2 files changed, 15 insertions(+) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 0425452de8d..6fa8d9086a3 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -42,6 +42,13 @@ Documentation Internal Changes ~~~~~~~~~~~~~~~~ +- Skip the ``to_duck_array`` dispatch in :py:meth:`Variable.load` and + :py:meth:`Variable.load_async` when the underlying data is already a + ``numpy.ndarray``. The dispatch was a no-op for that case but added + noticeable per-variable overhead to :py:meth:`Dataset.load` / + :py:meth:`DataArray.load` on datasets with many in-memory variables + (:issue:`11352`). + .. _whats-new.2026.04.0: diff --git a/xarray/core/variable.py b/xarray/core/variable.py index b4cdf5cf6ca..f0724919cf5 100644 --- a/xarray/core/variable.py +++ b/xarray/core/variable.py @@ -1019,6 +1019,12 @@ def load(self, **kwargs) -> Self: DataArray.load Dataset.load """ + # Fast path: an in-memory numpy array has nothing to load. The full + # to_duck_array dispatch otherwise walks is_chunked_array, the + # ExplicitlyIndexed isinstance check, and is_duck_array only to return + # self._data unchanged. + if isinstance(self._data, np.ndarray): + return self self._data = to_duck_array(self._data, **kwargs) return self @@ -1052,6 +1058,8 @@ async def load_async(self, **kwargs) -> Self: DataArray.load_async Dataset.load_async """ + if isinstance(self._data, np.ndarray): + return self self._data = await async_to_duck_array(self._data, **kwargs) return self From b279a3a44aac673bff196c0eb8b114f8216d4e55 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Tue, 26 May 2026 09:11:52 +0200 Subject: [PATCH 2/5] fix(load): preserve fast path on ndarray subclasses without chunks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous `isinstance(self._data, np.ndarray)` short-circuit incorrectly returned `self` (skipping the load) for ndarray subclasses with a `chunks` attribute — test fakes like DummyChunkedArray, or any third-party chunked array implementation that subclasses ndarray. Narrow to `isinstance + not hasattr("chunks")` so plain ndarrays and non-chunked subclasses (MaskedArray, np.matrix) still skip the to_duck_array dispatch, while subclasses that advertise chunks fall through to the full path. Co-authored-by: Claude --- xarray/core/variable.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/xarray/core/variable.py b/xarray/core/variable.py index f0724919cf5..ef7ed5ebe17 100644 --- a/xarray/core/variable.py +++ b/xarray/core/variable.py @@ -1019,11 +1019,10 @@ def load(self, **kwargs) -> Self: DataArray.load Dataset.load """ - # Fast path: an in-memory numpy array has nothing to load. The full - # to_duck_array dispatch otherwise walks is_chunked_array, the - # ExplicitlyIndexed isinstance check, and is_duck_array only to return - # self._data unchanged. - if isinstance(self._data, np.ndarray): + # Fast path: an in-memory numpy ndarray has nothing to load. Subclasses + # that advertise a `chunks` attribute (test fakes, third-party chunked + # ndarray subclasses) must still go through to_duck_array. + if isinstance(self._data, np.ndarray) and not hasattr(self._data, "chunks"): return self self._data = to_duck_array(self._data, **kwargs) return self @@ -1058,7 +1057,7 @@ async def load_async(self, **kwargs) -> Self: DataArray.load_async Dataset.load_async """ - if isinstance(self._data, np.ndarray): + if isinstance(self._data, np.ndarray) and not hasattr(self._data, "chunks"): return self self._data = await async_to_duck_array(self._data, **kwargs) return self From e57a45f64bfff0502e9f08b0af684c2c2d278a21 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Wed, 27 May 2026 15:20:29 +0200 Subject: [PATCH 3/5] refactor: move numpy fast-path into to_duck_array / to_numpy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per review feedback on #11355, centralize the in-memory numpy short-circuit inside the dispatch helpers themselves rather than guarding Variable.load. The fast-path now fires for every caller of to_duck_array, async_to_duck_array, and to_numpy, not just Variable.load. Behavior change: the guard is now type(data) is np.ndarray instead of isinstance + not hasattr("chunks") — exact-type match is faster and the fallback path correctly handles ndarray subclasses, so the hasattr guard is no longer needed. Measured impact: - Dataset.load (200 scalar vars): 1.39x - Dataset.load (2000 scalar vars): 1.44x - Variable.to_numpy() x 1000: 5.64x - DataArray.to_numpy() x 1000: 4.59x - repr / to_pandas / to_dataframe: unchanged (dispatch was not the bottleneck) Co-authored-by: Claude [This is Claude Code on behalf of Felix Bumann] --- doc/whats-new.rst | 12 ++++++------ xarray/core/variable.py | 7 ------- xarray/namedarray/pycompat.py | 15 +++++++++++++++ 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 6fa8d9086a3..48219f69837 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -42,12 +42,12 @@ Documentation Internal Changes ~~~~~~~~~~~~~~~~ -- Skip the ``to_duck_array`` dispatch in :py:meth:`Variable.load` and - :py:meth:`Variable.load_async` when the underlying data is already a - ``numpy.ndarray``. The dispatch was a no-op for that case but added - noticeable per-variable overhead to :py:meth:`Dataset.load` / - :py:meth:`DataArray.load` on datasets with many in-memory variables - (:issue:`11352`). +- Short-circuit ``to_duck_array``, ``async_to_duck_array``, and ``to_numpy`` + when the input is already a plain ``numpy.ndarray``. The dispatch was a no-op + for that case but added noticeable per-call overhead in :py:meth:`Dataset.load`, + :py:meth:`DataArray.load`, :py:meth:`DataArray.to_numpy`, + :py:meth:`Variable.to_numpy`, and the in-memory paths of ``to_pandas`` / + ``to_dataframe`` / plotting (:issue:`11352`). .. _whats-new.2026.04.0: diff --git a/xarray/core/variable.py b/xarray/core/variable.py index ef7ed5ebe17..b4cdf5cf6ca 100644 --- a/xarray/core/variable.py +++ b/xarray/core/variable.py @@ -1019,11 +1019,6 @@ def load(self, **kwargs) -> Self: DataArray.load Dataset.load """ - # Fast path: an in-memory numpy ndarray has nothing to load. Subclasses - # that advertise a `chunks` attribute (test fakes, third-party chunked - # ndarray subclasses) must still go through to_duck_array. - if isinstance(self._data, np.ndarray) and not hasattr(self._data, "chunks"): - return self self._data = to_duck_array(self._data, **kwargs) return self @@ -1057,8 +1052,6 @@ async def load_async(self, **kwargs) -> Self: DataArray.load_async Dataset.load_async """ - if isinstance(self._data, np.ndarray) and not hasattr(self._data, "chunks"): - return self self._data = await async_to_duck_array(self._data, **kwargs) return self diff --git a/xarray/namedarray/pycompat.py b/xarray/namedarray/pycompat.py index 5832f7cc9e7..75bc8174393 100644 --- a/xarray/namedarray/pycompat.py +++ b/xarray/namedarray/pycompat.py @@ -102,6 +102,12 @@ def to_numpy( from xarray.core.indexing import ExplicitlyIndexed from xarray.namedarray.parallelcompat import get_chunked_array_type + # Fast path: a plain numpy ndarray is already the target type, so skip + # the .to_numpy() / ExplicitlyIndexed / chunked / cupy / pint / sparse + # dispatch. ndarray subclasses fall through to keep their behavior. + if type(data) is np.ndarray: + return data + try: # for tests only at the moment return data.to_numpy() # type: ignore[no-any-return,union-attr] @@ -134,6 +140,12 @@ def to_duck_array(data: Any, **kwargs: dict[str, Any]) -> duckarray[_ShapeType, ) from xarray.namedarray.parallelcompat import get_chunked_array_type + # Fast path: a plain numpy ndarray is already an in-memory duck array, + # so skip the chunked/ExplicitlyIndexed/duck-array dispatch. ndarray + # subclasses fall through to the normal path so they keep their behavior. + if type(data) is np.ndarray: + return data # type: ignore[return-value] + if is_chunked_array(data): chunkmanager = get_chunked_array_type(data) loaded_data, *_ = chunkmanager.compute(data, **kwargs) # type: ignore[var-annotated] @@ -155,6 +167,9 @@ async def async_to_duck_array( ImplicitToExplicitIndexingAdapter, ) + if type(data) is np.ndarray: + return data # type: ignore[return-value] + if isinstance(data, ExplicitlyIndexed | ImplicitToExplicitIndexingAdapter): return await data.async_get_duck_array() # type: ignore[union-attr, no-any-return] else: From 8bd3979055efc8253b456ef963f345c281bac7d2 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Wed, 27 May 2026 16:06:17 +0200 Subject: [PATCH 4/5] refactor: simplify non-numpy paths in to_duck_array / to_numpy Keep the type(data) is np.ndarray fast-paths (which short-circuit the common case in one pointer compare), but tidy the slow paths so they match the same DRY principle Illviljan raised on #11354: - to_numpy: replace `try: data.to_numpy() except AttributeError` with `if hasattr(data, "to_numpy")`. Identical semantics, no exception machinery for non-ndarray inputs that lack the method. - to_duck_array: restructure so is_duck_array is called once instead of twice (previously via is_chunked_array AND in the duck-array branch). Pull the ExplicitlyIndexed check up so the duck-array dispatch is expressed as a single is_duck_array + dask check. Measured impact on plain ndarrays (vs main): isel(...).load() with 200 scalar vars 1.37x isel(...).load() with 2000 scalar vars 1.40x DataArray.to_numpy() x 1000 4.57x Variable.to_numpy() x 1000 5.42x Co-authored-by: Claude [This is Claude Code on behalf of Felix Bumann] --- xarray/namedarray/pycompat.py | 39 +++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/xarray/namedarray/pycompat.py b/xarray/namedarray/pycompat.py index 75bc8174393..e2a9cc7b065 100644 --- a/xarray/namedarray/pycompat.py +++ b/xarray/namedarray/pycompat.py @@ -8,7 +8,7 @@ from packaging.version import Version from xarray.core.utils import is_scalar -from xarray.namedarray.utils import is_duck_array, is_duck_dask_array +from xarray.namedarray.utils import is_dask_collection, is_duck_array, is_duck_dask_array integer_types = (int, np.integer) @@ -102,17 +102,17 @@ def to_numpy( from xarray.core.indexing import ExplicitlyIndexed from xarray.namedarray.parallelcompat import get_chunked_array_type - # Fast path: a plain numpy ndarray is already the target type, so skip - # the .to_numpy() / ExplicitlyIndexed / chunked / cupy / pint / sparse - # dispatch. ndarray subclasses fall through to keep their behavior. + # Fast path: a plain numpy ndarray is the dispatch target, so skip the + # whole ExplicitlyIndexed / chunked / cupy / pint / sparse / np.asarray + # chain in one pointer compare. ndarray subclasses fall through. if type(data) is np.ndarray: return data - try: - # for tests only at the moment + # `hasattr` avoids the AttributeError that `try: data.to_numpy()` raises + # on every non-ndarray that lacks `to_numpy` — exception machinery is + # measurably more expensive than the attribute lookup. + if hasattr(data, "to_numpy"): return data.to_numpy() # type: ignore[no-any-return,union-attr] - except AttributeError: - pass if isinstance(data, ExplicitlyIndexed): data = data.get_duck_array() # type: ignore[no-untyped-call] @@ -140,23 +140,26 @@ def to_duck_array(data: Any, **kwargs: dict[str, Any]) -> duckarray[_ShapeType, ) from xarray.namedarray.parallelcompat import get_chunked_array_type - # Fast path: a plain numpy ndarray is already an in-memory duck array, - # so skip the chunked/ExplicitlyIndexed/duck-array dispatch. ndarray - # subclasses fall through to the normal path so they keep their behavior. + # Fast path: a plain numpy ndarray is already an in-memory duck array, so + # skip the chunked / ExplicitlyIndexed / duck-array dispatch entirely. + # ndarray subclasses fall through to the normal path. if type(data) is np.ndarray: return data # type: ignore[return-value] - if is_chunked_array(data): + if isinstance(data, ExplicitlyIndexed | ImplicitToExplicitIndexingAdapter): + return data.get_duck_array() # type: ignore[no-untyped-call, no-any-return] + + # Single `is_duck_array` call: the previous form invoked it once via + # `is_chunked_array` and once again in the duck-array branch. + if not is_duck_array(data): + return np.asarray(data) # type: ignore[return-value] + + if hasattr(data, "chunks") or is_dask_collection(data): chunkmanager = get_chunked_array_type(data) loaded_data, *_ = chunkmanager.compute(data, **kwargs) # type: ignore[var-annotated] return loaded_data - if isinstance(data, ExplicitlyIndexed | ImplicitToExplicitIndexingAdapter): - return data.get_duck_array() # type: ignore[no-untyped-call, no-any-return] - elif is_duck_array(data): - return data - else: - return np.asarray(data) # type: ignore[return-value] + return data async def async_to_duck_array( From 8fd6badee79fa4abe23b0c801749ab7b929a45dc Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 27 May 2026 14:06:59 +0000 Subject: [PATCH 5/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- xarray/namedarray/pycompat.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/xarray/namedarray/pycompat.py b/xarray/namedarray/pycompat.py index e2a9cc7b065..eaedc720973 100644 --- a/xarray/namedarray/pycompat.py +++ b/xarray/namedarray/pycompat.py @@ -8,7 +8,11 @@ from packaging.version import Version from xarray.core.utils import is_scalar -from xarray.namedarray.utils import is_dask_collection, is_duck_array, is_duck_dask_array +from xarray.namedarray.utils import ( + is_dask_collection, + is_duck_array, + is_duck_dask_array, +) integer_types = (int, np.integer)