test(reconcile): assert the move cutoff's behaviour, not only its value (#673)

tests/test_canonical_baseline.py

@@ -235,3 +235,56 @@ def test_canonical_output_matches_baseline(key: str, old_path: Path, new_path: P
             f"evidence in the same pull request. {REGENERATE}"
         )
     assert actual == expected, f"digest held but a recorded count moved for {key}. {REGENERATE}"
+
+
+# --- One audited pair's classification split, pinned in SOURCE (#673) -----------
+# The gate above already asserts the summary counts, and already prints the delta on
+# failure, so "assert the counts as well as the digest" is not what is missing. What is
+# missing is that every expected value it compares against lives in a REGENERABLE file,
+# and its own remedy line says to regenerate. That is correct for a change detector and
+# is the wrong last word on a decision rule: #673 shifted the effective move cutoff
+# without touching MOVE_THRESHOLD, the whole fast tier stayed green, and this gate's
+# report was "canonical output changed ... Regenerate with UPDATE_BASELINE=1".
+#
+# The numbers below are literals in the test file. `UPDATE_BASELINE=1` cannot move them;
+# only a person editing this source can, which is a deliberate act that appears in a
+# diff and can be argued with in review. That is the whole contribution -- detection is
+# shared with the gate above, attribution and remedy are not.
+#
+# The FULL split is pinned rather than `moved` alone, because a lost move is conserved
+# rather than destroyed: it becomes one removal plus one addition. #673's mutation moved
+# 163 -> 161 while removed went 136 -> 138 and added 50 -> 52. Keying on `moved` alone
+# would catch that one, but not a change that reclassified in the other direction while
+# leaving the move count level.
+AUDITED_PAIR = "119-hr-1/1_reported-in-house->2_engrossed-in-house"
+
+#: Measured on this pair, and the same figures #673 quotes. Not regenerable.
+AUDITED_SUMMARY = {"added": 50, "modified": 148, "moved": 163, "removed": 136, "unchanged": 0}
+
+
+def test_the_audited_pair_keeps_its_classification_split():
+    """How many provisions this pair recognizes as moved, said in the test's own source.
+
+    Answers a different question from the digest gate above. That one asks whether any
+    byte moved and offers regeneration as the fix; this asks whether the matcher still
+    reaches the same verdict about the same bill, and regeneration is not available.
+
+    One pair, deliberately. This is an attribution aid rather than a second corpus sweep,
+    and pinning every pair here would duplicate the digest gate's coverage at its
+    maintenance cost while making a legitimate corpus change 27 edits instead of one.
+    119-hr-1 is chosen because it is the pair #673 measured, so the numbers below can be
+    checked against the issue rather than taken on trust.
+    """
+    old_path, new_path = next((old, new) for key, old, new in _PAIRS if key == AUDITED_PAIR)
+    summary = baseline_record(old_path, new_path)["summary"]
+
+    assert summary == AUDITED_SUMMARY, (
+        f"the classification split for {AUDITED_PAIR} moved:\n"
+        f"  expected: {AUDITED_SUMMARY}\n"
+        f"  actual:   {summary}\n"
+        f"moved {AUDITED_SUMMARY['moved']} -> {summary.get('moved')} means that many provisions "
+        f"changed status between 'the same text that moved' and 'one deleted, one added'.\n"
+        f"This is NOT fixed by regenerating the baseline -- these numbers live in this file. "
+        f"If the change is intended, edit them here and bring the precision and recall evidence "
+        f"ADR 0020 asks for."
+    )

tests/test_reconcile.py

@@ -24,7 +24,7 @@
     settle_correspondences,
     unmatched_population,
 )
-from deltatrack.similarity import MOVE_THRESHOLD
+from deltatrack.similarity import MOVE_THRESHOLD, text_similarity
 from tests.corpus_paths import fixture_path
 
 
@@ -133,6 +133,103 @@ def test_low_similarity_stays_separate(self):
         assert result[0].change_type == "removed"
         assert result[1].change_type == "added"
 
+    # --- At the cutoff (#673) ---------------------------------------------------
+    # The cases above bracket MOVE_THRESHOLD from a distance. The nearest sits at 0.6667,
+    # 11.1% clear on the high side, and both low-side cases score 0.0000 -- zero word
+    # overlap, the furthest possible distance -- so each would pass for any cutoff in
+    # (0, 1]. The constant is pinned by test_the_move_cutoff_is_pinned_for_phase_1; the
+    # BEHAVIOUR the constant exists to produce was not, and #673 measured the consequence:
+    # moving MOVE_THRESHOLD from 0.6 to 0.45, a 25% shift in what counts as a move, left
+    # all ten tests in this file green.
+    #
+    # The two fixtures below are REAL corpus text, not composed to hit a number. They were
+    # found by scoring the unmatched population of all 27 committed manifest pairs through
+    # production's own round-1 -> round-2 handoff: 12 pairs land on exactly 0.6, 38 in
+    # [0.58, 0.60) and 36 in [0.60, 0.62). That matters because a fixture written to reach
+    # a similarity encodes the author's belief about the scorer, and then tests the belief.
+    #
+    # These read as deliberate, which near-boundary fixtures have to: if the rule
+    # legitimately changes, these are the first tests that should be revisited, and their
+    # texts are quoted from named bills so a reader can re-measure rather than guess.
+
+    def test_a_pair_at_exactly_the_cutoff_is_moved(self):
+        """Similarity of exactly 0.6 is a move, because the rule is `>=` and not `>`.
+
+        MOVE_THRESHOLD's own comment says "At or above this ratio", and `move_candidates`
+        spells it `sim >= threshold`. Nothing tested the difference. Every other case in
+        this file clears the cutoff by a margin, so flipping that one character reclassifies
+        real provisions while the whole file stays green.
+
+        Both texts are short-title provisions from 118-hr-4366, the Senate engrossed
+        amendment against the House engrossed amendment. Their measured word-level ratio is
+        exactly 0.6: `>= MOVE_THRESHOLD` is True and `> MOVE_THRESHOLD` is False, which is
+        what makes this the one fixture that can see the direction.
+        """
+        old_text = (
+            "This division may be cited as the Military Construction, Veterans Affairs, "
+            "and Related Agencies Appropriations Act, 2024."
+        )
+        new_text = "This title may be cited as the Department of Commerce Appropriations Act, 2024."
+
+        assert text_similarity(old_text, new_text) == MOVE_THRESHOLD, (
+            "this fixture is only meaningful while it sits exactly ON the cutoff; re-measure "
+            "it before adjusting either the text or the constant"
+        )
+
+        result = reconciled(
+            [_node("o1", ("division j", "sec. 1"), old_text)],
+            [_node("n1", ("title i", "sec. 1"), new_text)],
+        )
+
+        moved = [c for c in result if c.change_type == "moved"]
+        assert len(moved) == 1, (
+            "a pair at exactly MOVE_THRESHOLD was not reconciled as a move, so the "
+            f"comparison is excluding its own boundary: {[c.change_type for c in result]}"
+        )
+
+    def test_a_pair_just_below_the_cutoff_stays_separate(self):
+        """The tightest real miss in the corpus stays a removal plus an addition.
+
+        0.5957, which is 0.0043 below the cutoff. The existing low-side cases sit at
+        0.0000, so this is the first test in the file that would notice the cutoff being
+        loosened rather than merely deleted -- and it is the half that keeps the
+        at-the-cutoff case above honest, since a rule that classified everything as a move
+        would satisfy that one alone.
+
+        Both texts are effective-date provisions from 119-hr-1, reported-in-house against
+        engrossed-in-house.
+        """
+        old_text = (
+            "(b)Effective date The amendment made by this section shall apply to designations made "
+            "after the date of the enactment of this Act in taxable years ending after such date."
+        )
+        new_text = (
+            "(b)Effective date The amendment made by this section shall apply to taxable years "
+            "beginning after December 31, 2025."
+        )
+
+        # The measured similarity of these two texts, which is a fact about the TEXT and
+        # not about the cutoff. Asserted as a literal so an edit to either string, or a
+        # change in how the scorer tokenizes, is caught here and says so -- rather than
+        # surfacing as the classification assertion below, where it would read as a
+        # threshold regression. Deliberately NOT compared against MOVE_THRESHOLD: leaving
+        # the cutoff out of the precondition is what lets a change to the cutoff fail on
+        # the behaviour instead, naming what actually went wrong.
+        assert text_similarity(old_text, new_text) == 0.5957446808510638, (
+            "this fixture no longer measures what its docstring says; re-measure it before "
+            "reading anything into the assertion below"
+        )
+
+        result = reconciled(
+            [_node("o1", ("sec. 70101",), old_text)],
+            [_node("n1", ("sec. 70101",), new_text)],
+        )
+
+        assert [c.change_type for c in result] == ["removed", "added"], (
+            "a pair below MOVE_THRESHOLD was reconciled as a move, so the effective cutoff "
+            f"has dropped below its constant: {[c.change_type for c in result]}"
+        )
+
     def test_moved_with_text_changes(self):
         old_text = "For acquisition and construction, $1,876,875,000, to remain available until September 30, 2025."
         new_text = "For acquisition and construction, $2,022,775,000, to remain available until expended."