From 50fd64864857d8b0460ec6bee8ddb688b7b0eb6b Mon Sep 17 00:00:00 2001 From: Tony Y Date: Thu, 19 Nov 2026 20:59:02 +0000 Subject: [PATCH] Do not count ignore_nan_inequality pairs against max_diffs With ignore_nan_inequality=True, NaN-vs-NaN pairs are forgiven when the values are compared, but they still consumed the max_diffs budget: _diff counted every level on entry and the iterable/dict loops counted every pair before descending. With enough NaN pairs (e.g. long numpy arrays), the traversal hit the limit, logged the max-diffs warning and stopped before reaching the real diffs, returning an empty result. Skip the counting for pairs that ignore_nan_inequality forgives, via a shared _is_ignored_nan_inequality helper (also used for the existing forgiveness check in _diff). Real diffs keep counting as before. Fixes #433 --- deepdiff/diff.py | 28 +++++++++++++++++++++------- tests/test_diff_numpy.py | 15 +++++++++++++++ tests/test_diff_text.py | 22 ++++++++++++++++++++++ 3 files changed, 58 insertions(+), 7 deletions(-) diff --git a/deepdiff/diff.py b/deepdiff/diff.py index c044383a..68df7782 100755 --- a/deepdiff/diff.py +++ b/deepdiff/diff.py @@ -810,11 +810,11 @@ def _diff_dict( self._report_result(item_removed_key, change_level, local_tree=local_tree) for key in t_keys_intersect: # key present in both dicts - need to compare values - if self._count_diff() is StopIteration: - return # pragma: no cover. This is already covered for addition. - key1 = t1_clean_to_keys[key] if t1_clean_to_keys else key key2 = t2_clean_to_keys[key] if t2_clean_to_keys else key + if not self._is_ignored_nan_inequality(t1[key1], t2[key2]) and self._count_diff() is StopIteration: + return # pragma: no cover. This is already covered for addition. + item_id = id(t1[key1]) if parents_ids and item_id in parents_ids: continue @@ -1079,7 +1079,7 @@ def _diff_by_forming_pairs_and_comparing_one_by_one( t2_from_index=t2_from_index, t2_to_index=t2_to_index, match_key_spec=match_key_spec, ): - if self._count_diff() is StopIteration: + if not self._is_ignored_nan_inequality(x, y) and self._count_diff() is StopIteration: return # pragma: no cover. This is already covered for addition. reference_param1 = i @@ -2122,6 +2122,15 @@ def _diff_types(self, level, local_tree=None): level.report_type = 'type_changes' self._report_result('type_changes', level, local_tree=local_tree) + def _is_ignored_nan_inequality(self, t1, t2): + """Whether comparing t1 and t2 is a NaN-vs-NaN comparison that + ignore_nan_inequality forgives, i.e. the pair is not a diff.""" + return bool( + self.ignore_nan_inequality + and isinstance(t1, (float, np_floating)) + and str(t1) == str(t2) == 'nan' + ) + def _count_diff(self): if (self.max_diffs is not None and self._stats[DIFF_COUNT] > self.max_diffs): if not self._stats[MAX_DIFF_LIMIT_REACHED]: @@ -2186,8 +2195,13 @@ def _diff(self, level, parents_ids=frozenset(), _original_type=None, local_tree= parents_ids: the ids of all the parent objects in the tree from the current node. _original_type: If the objects had an original type that was different than what currently exists in the level.t1 and t2 """ - if self._count_diff() is StopIteration: - return + # Pairs forgiven by ignore_nan_inequality are not diffs, so they must + # not consume the max_diffs budget: otherwise enough NaN pairs make + # the traversal stop (with the max-diffs warning) before the real + # diffs are reached, and those diffs are silently dropped. + if not self._is_ignored_nan_inequality(level.t1, level.t2): + if self._count_diff() is StopIteration: + return if self._use_custom_operator(level): return @@ -2218,7 +2232,7 @@ def _diff(self, level, parents_ids=frozenset(), _original_type=None, local_tree= self._report_result('values_changed', level, local_tree=local_tree) return - if self.ignore_nan_inequality and isinstance(level.t1, (float, np_floating)) and str(level.t1) == str(level.t2) == 'nan': + if self._is_ignored_nan_inequality(level.t1, level.t2): return if isinstance(level.t1, booleans): diff --git a/tests/test_diff_numpy.py b/tests/test_diff_numpy.py index 129500fb..acd49c9f 100644 --- a/tests/test_diff_numpy.py +++ b/tests/test_diff_numpy.py @@ -174,3 +174,18 @@ class TestNumpy: def test_numpy(self, test_name, t1, t2, deepdiff_kwargs, expected_result): diff = DeepDiff(t1, t2, **deepdiff_kwargs) assert expected_result == diff, f"test_numpy {test_name} failed." + + +def test_numpy_ignore_nan_inequality_does_not_consume_max_diffs(): + # With ignore_nan_inequality=True, NaN pairs are not diffs and must not + # consume the max_diffs budget; otherwise the traversal stops (with the + # max-diffs warning) before reaching the real diff at the end. + t1 = np.array([np.nan, np.nan, np.nan, np.nan, 1]) + t2 = np.array([np.nan, np.nan, np.nan, np.nan, 2]) + diff = DeepDiff(t1, t2, ignore_nan_inequality=True, max_diffs=2) + expected = { + 'values_changed': { + 'root[4]': {'new_value': np.float64(2.0), 'old_value': np.float64(1.0)} + } + } + assert expected == diff diff --git a/tests/test_diff_text.py b/tests/test_diff_text.py index cfeda768..ff34898b 100755 --- a/tests/test_diff_text.py +++ b/tests/test_diff_text.py @@ -1938,6 +1938,28 @@ def gen2(): def test_ignore_nan_inequality(self, t1, t2, params, expected_result): assert expected_result == list(DeepDiff(t1, t2, **params).keys()) + @pytest.mark.skipif(pypy3, reason="some versions of pypy3 have nan==nan") + def test_ignore_nan_inequality_does_not_consume_max_diffs(self): + # NaN pairs forgiven by ignore_nan_inequality are not diffs, so they + # must not eat into the max_diffs budget: the real diff after them + # still has to be found, with no max-diffs warning. + nan = float('nan') + t1 = [nan, nan, nan, nan, 1] + t2 = [nan, nan, nan, nan, 2] + expected = {'values_changed': {'root[4]': {'new_value': 2, 'old_value': 1}}} + assert expected == DeepDiff(t1, t2, ignore_nan_inequality=True, max_diffs=2) + + t1 = {'a': nan, 'b': nan, 'c': nan, 'd': 1} + t2 = {'a': nan, 'b': nan, 'c': nan, 'd': 2} + expected = {'values_changed': {"root['d']": {'new_value': 2, 'old_value': 1}}} + assert expected == DeepDiff(t1, t2, ignore_nan_inequality=True, max_diffs=2) + + def test_max_diffs_still_applies_to_real_diffs(self): + # Sanity check the other direction: real diffs keep counting, so the + # traversal still stops early once the budget is exceeded. + ddiff = DeepDiff([1, 2, 3, 4], [5, 6, 7, 8], max_diffs=2) + assert {'values_changed': {'root[0]': {'new_value': 5, 'old_value': 1}}} == ddiff + @pytest.mark.parametrize('ignore_order, ignore_private_variables, expected', [ (True, True, {}), (False, True, {}),