The leading-particle claim takes its group, not the rest of the name (#390, #365)

docs/design/rules.md

@@ -108,8 +108,8 @@ P1. Rationale: a never-given particle standing alone cannot be
       "de la Vega"                →  family="de la Vega"
       "Sir de Mesnil"             →  family="de Mesnil"
       "Mesnil de"  family-first   →  family="Mesnil de"
-      "de Mesnil Juan"            →  family="de Mesnil"  deviates: #364 (today: family="de Mesnil Juan")
-      "de Mesnil Juan"            →  given="Juan"  deviates: #364 (today: given="")
+      "de Mesnil Juan"            →  family="de Mesnil"
+      "de Mesnil Juan"            →  given="Juan"
       "van Gogh"                  →  given="van"  · boundary
     Accepted: a bare "de" stays the given name — there is nothing to
     fold into, and inventing a surname would be worse.
@@ -121,7 +121,7 @@ P1. Rationale: a never-given particle standing alone cannot be
     the bare particle reading as a given name, not any name part
     that begins with one.
       "Juan de la Vega"  family-first  →  family="Juan"
-    history: decisions.md#P1 · interacts: P2, P4, P6 · implemented: nameparser/_pipeline/_post_rules.py
+    history: decisions.md#P1 · interacts: P2, P4, P6 · implemented: nameparser/_pipeline/_assign.py, nameparser/_pipeline/_post_rules.py
 
 P2. Rationale: a particle is written as part of the surname it
     precedes, and a title stands outside the name entirely.

nameparser/_pipeline/_assign.py

@@ -268,7 +268,37 @@ def _assign_main(seg_idx: int, state: ParseState,
     order = _effective_order(state.policy,
                              [pieces[i] for i in name_pieces], tokens,
                              dot_divided=bool(state.interpunct_offsets))
-    roles = _name_positions(order, len(name_pieces))
+    # rules.md#P1: "a never-given particle standing alone where the
+    # given name would go — or opening the name — marks the name as
+    # surname-only: the particle run and the one name word it
+    # attaches to are the family, and any name words beyond that read
+    # by position." (history: decisions.md#P1)
+    # The claim happens HERE, before positions are handed out, because
+    # the particle is evidence about the name and name_order governs
+    # only what no vocabulary has claimed (decisions.md#O4). Doing it
+    # after assignment is what made the fold's reach depend on which
+    # role the order happened to give the particle (#364, #365).
+    claimed: list[int] = []
+    if len(name_pieces) >= 2:
+        head = pieces[name_pieces[0]]
+        if (len(head) == 1
+                and "particle" in tokens[head[0]].tags
+                and "vocab:particle-ambiguous" not in tokens[head[0]].tags):
+            claimed, name_pieces = name_pieces[:2], name_pieces[2:]
+    if claimed:
+        # The family slot is taken, so the remainder positions itself
+        # as if it did not exist: ask for one more slot than there are
+        # pieces and drop FAMILY. Reusing _name_positions rather than
+        # re-deriving the order keeps one definition of "by position"
+        # -- under FAMILY_FIRST a plain count would hand the leftover
+        # the family a second time.
+        roles: list[Role] = [
+            r for r in _name_positions(order, len(name_pieces) + 1)
+            if r is not Role.FAMILY]
+        for piece_idx in claimed:
+            _set_roles(tokens, pieces[piece_idx], Role.FAMILY)
+    else:
+        roles = list(_name_positions(order, len(name_pieces)))
     for pos, piece_idx in enumerate(name_pieces):
         _set_roles(tokens, pieces[piece_idx], roles[pos])
     for piece_idx in suffix_pieces:

nameparser/_pipeline/_post_rules.py

@@ -1,16 +1,23 @@
 """Stage: post_rules.
 
-Consumes: tokens (roles assigned), plus pieces and structure -- the
-particle fold reads the opening piece of segment 0, or of segment 1
-under a family comma (#359). structure was always read here, for the
-rotation gate.
+Consumes: tokens (roles assigned), plus structure for the rotation
+gate.
 Produces: tokens with roles adjusted by the post rules.
 Reads: Policy.patronymic_rules, Policy.middle_as_family;
 Lexicon.given_name_titles.
 
 Implements rules H1, P1, O1, O2 and O3 of docs/design/rules.md; each
 is cited at its code below, and P1/O1/O2's history lives in
 docs/design/decisions.md.
+
+P1 is SPLIT across two stages as of #390, and the split is the rule's
+own shape rather than an accident: a leading particle CLAIMS the
+family before positions exist, so that half lives in assign; a lone
+particle that positioning has already dropped into the GIVEN role can
+only be seen afterwards, so that half stays here. The pieces scan the
+leading half used (`_leading_name_piece`, and with it the reading of
+state.pieces) went with it -- assign reaches the same piece by peeling
+titles before it counts name pieces.
 """
 from __future__ import annotations
 
@@ -37,34 +44,10 @@
     r"^(оглу|оглы|оғлу|ўғли|угли|кызы|гызы|қызы|қизи|улы|ұлы|уулу)$", re.I)
 
 
-_NAME_ROLES = (Role.GIVEN, Role.MIDDLE, Role.FAMILY)
-
-
 def _idx(tokens: list[WorkToken], role: Role) -> list[int]:
     return [i for i, t in enumerate(tokens) if t.role is role]
 
 
-def _leading_name_piece(state: ParseState,
-                        tokens: list[WorkToken]) -> tuple[int, ...]:
-    """The piece that OPENS the name, whatever role name_order gave it:
-    the first piece holding a GIVEN, MIDDLE or FAMILY token, in the
-    segment the positional read governs. Every piece holding none of
-    those is walked past -- title and suffix pieces, but NICKNAME and
-    MAIDEN as well, and anything assign left unroled -- and any number
-    of them, not only a single leading title. The segment is 0, except
-    under a family comma, where segment 0 is already fixed as the
-    surname and the name continues in segment 1. Empty on either of
-    two exits: that segment does not exist, or none of its pieces
-    holds a name token."""
-    seg = 1 if state.structure is Structure.FAMILY_COMMA else 0
-    if seg >= len(state.pieces):
-        return ()
-    for piece in state.pieces[seg]:
-        if any(tokens[i].role in _NAME_ROLES for i in piece):
-            return piece
-    return ()
-
-
 def _retag(tokens: list[WorkToken], i: int, role: Role) -> None:
     tokens[i] = dataclasses.replace(tokens[i], role=role)
 
@@ -99,24 +82,22 @@ def post_rules(state: ParseState) -> ParseState:
     # attaches to are the family, and any name words beyond that
     # read by position." (v1 handle_non_first_name_prefix; history:
     # decisions.md#P1)
-    # DEVIATION #364: the fold below still takes every remaining name
-    # word, not just the particle run's own -- de Mesnil Juan gives
-    # family=de Mesnil Juan where the rule says family=de Mesnil plus
-    # given=Juan. Pinned by the deviates: markers on P1.
-    # Values written unquoted deliberately: this note sits INSIDE the
-    # citation block above (# decisions.md#P1) does not close it --
-    # _CITE_RE wants a colon after the ID), and the excerpt check
-    # takes the first quoted span in the block.
-    # Code-local: a lone PIECE is the test at both sites, so a
-    # particle group already chained forward is not a lone particle,
-    # and rule H1 above cannot be what produces the fold's family
-    # reading -- H1 is gated on `not families`.
-    sites = (_leading_name_piece(state, tokens), tuple(givens))
-    if len(givens) + len(middles) + len(families) > 1 and any(
+    # This is P1's SECOND site only. The opening-the-name half moved to
+    # assign in #390, where the claim can happen before positions are
+    # handed out; what is left here is the case assign cannot see --
+    # a lone particle that positional assignment has already dropped
+    # into the GIVEN role (Mesnil de under FAMILY_FIRST). It has no
+    # piece after it to attach to, so the fold is backward and the
+    # whole remainder is one word anyway.
+    # Code-local: a lone TOKEN in the given role is the test, so a
+    # particle group already chained forward never reaches it, and
+    # rule H1 above cannot be what produces this family reading --
+    # H1 is gated on `not families`.
+    site = tuple(givens)
+    if len(givens) + len(middles) + len(families) > 1 and (
             len(site) == 1
             and "particle" in tokens[site[0]].tags
-            and "vocab:particle-ambiguous" not in tokens[site[0]].tags
-            for site in sites):
+            and "vocab:particle-ambiguous" not in tokens[site[0]].tags):
         for i in givens + middles:
             _retag(tokens, i, Role.FAMILY)
         # downstream rules key on the role counts: recompute

tests/test_particles.py

@@ -390,9 +390,14 @@ def test_leading_non_first_name_prefix_with_patronymic_name_order(self) -> None:
         self.m(hn.last, "de Mesnil", hn)
 
     def test_leading_non_first_name_prefix_with_middle_name_as_last(self) -> None:
-        # handle_non_first_name_prefix runs first and empties middle_list, so
+        # The leading-particle claim runs first and empties middle_list, so
         # the later opt-in handle_middle_name_as_last has nothing left to do.
+        # The claim takes the particle and the ONE piece it attaches to
+        # (#390), so 'Garcia' survives as the given name -- before that it
+        # was swept into the family and first was ''. The interaction this
+        # test guards is unchanged: middle is empty either way.
         constants = Constants(middle_name_as_last=True)
         hn = HumanName("de Mesnil Garcia", constants=constants)
-        self.m(hn.first, "", hn)
-        self.m(hn.last, "de Mesnil Garcia", hn)
+        self.m(hn.first, "Garcia", hn)
+        self.m(hn.middle, "", hn)
+        self.m(hn.last, "de Mesnil", hn)

tests/v2/pipeline/test_post_rules.py

@@ -133,10 +133,9 @@ def test_degenerate_bare_particle_stays_given() -> None:
     # the leading particle chains the rest of the name into the family
     ("de Mesnil", "de Mesnil", "", ""),
     ("de la Vega", "de la Vega", "", ""),
-    # three pieces, so the fold has a MIDDLE to move as well as the
-    # given -- the `givens + middles` half of the repair, and the only
-    # no-comma corpus name that reaches it
-    ("de Mesnil Garcia", "de Mesnil Garcia", "", ""),
+    # three pieces, so a name word survives the claim -- the only
+    # no-comma corpus name that reaches it, and the one #390 moved
+    ("de Mesnil Garcia", "de Mesnil", "Garcia", ""),
     # ... and the trailing suffix run is peeled before the rule looks,
     # comma or no comma (NO_COMMA and SUFFIX_COMMA both fold)
     ("de Mesnil MD", "de Mesnil", "", "MD"),
@@ -152,21 +151,17 @@ def test_family_first_folds_leading_never_given_particle(
 
 
 @pytest.mark.parametrize("policy", _FAMILY_FIRST)
-def test_leading_piece_scan_skips_pieces_that_hold_no_name(
+def test_titles_are_peeled_before_the_leading_particle_claim(
         policy: Policy) -> None:
-    # `_leading_name_piece` walks PAST pieces carrying no name role
-    # rather than reading piece 0 -- and past the first such piece, not
-    # only over a single title. 'Mr. de Mesnil' cannot show that: its
-    # particle is chained into one piece with 'Mesnil', so the scan
-    # lands on a two-token piece and the rule declines either way.
-    # Here a mid-name suffix word breaks that chain, leaving the
-    # particle a piece of its own BEHIND a title piece. Without the
-    # skip, or reading only pieces[0], the scan finds the title (or
-    # nothing) and the name splits: given='MD', middle='Mesnil',
-    # family='de'.
-    out = _parsed("Dr. de MD Mesnil", policy)
+    # The claim reads the first NAME piece, not pieces[0]: a title in
+    # front must not hide the particle behind it. assign gets this by
+    # peeling titles before it counts name pieces (#390 moved the claim
+    # there from post_rules, retiring the _leading_name_piece scan that
+    # used to walk past non-name pieces). Without the peel the title is
+    # piece 0, no claim fires, and the name splits by position.
+    out = _parsed("Dr. de Mesnil", policy)
     assert _by_role(out, Role.TITLE) == "Dr."
-    assert _by_role(out, Role.FAMILY) == "de MD Mesnil"
+    assert _by_role(out, Role.FAMILY) == "de Mesnil"
     assert not _by_role(out, Role.GIVEN)
     assert not _by_role(out, Role.MIDDLE)
 
@@ -198,8 +193,12 @@ def test_family_first_leading_particle_cases_that_do_not_fold(
 @pytest.mark.parametrize("text,title,given,middle,family,suffix", [
     ("de Mesnil", "", "", "", "de Mesnil", ""),
     ("de la Vega", "", "", "", "de la Vega", ""),
-    ("de Mesnil Garcia", "", "", "", "de Mesnil Garcia", ""),
-    ("Dr. de MD Mesnil", "Dr.", "", "", "de MD Mesnil", ""),
+    ("de Mesnil Garcia", "", "Garcia", "", "de Mesnil", ""),
+    # garbage in, garbage out: MD is suffix vocabulary sitting
+    # mid-name, so the piece the particle attaches to is "MD".
+    # Both readings of this input are garbage; pinned only so the
+    # claim's reach is visible, never as a shape to design around
+    ("Dr. de MD Mesnil", "Dr.", "Mesnil", "", "de MD", ""),
     ("de Mesnil MD", "", "", "", "de Mesnil", "MD"),
     ("De Mesnil, MD", "", "", "", "De Mesnil", "MD"),
     ("Mr. de Mesnil", "Mr.", "", "", "de Mesnil", ""),
@@ -216,7 +215,7 @@ def test_family_first_leading_particle_cases_that_do_not_fold(
     ("Smith, de Mesnil", "", "", "", "Smith de Mesnil", ""),
     ("Smith, van Gogh", "", "van", "Gogh", "Smith", ""),
 ])
-def test_default_order_is_unchanged_by_the_family_first_fold(
+def test_leading_particle_claim_reads_alike_in_the_default_order(
         text: str, title: str, given: str, middle: str, family: str,
         suffix: str) -> None:
     out = _parsed(text)

tests/v2/test_ledger_guards.py

@@ -1020,6 +1020,9 @@ def _claim(rule: dict) -> _Claim:
             _Claim(215, ('given', 'suffix', 'title'), "f16a0e79cba3"),
         "fix(comma-precomma-family) pre-comma run reads as family, not given":
             _Claim(215, ('family', 'given'), "f16a0e79cba3"),
+        "fix(#390) the leading-particle claim takes the particle's group, "
+        "not the rest of the name":
+            _Claim(1, ('family', 'given'), "caf836e3556c"),
         "fix(suffix-routing) two-token name with unambiguous trailing suffix stays suffix":
             _Claim(751, ('family', 'given', 'suffix'), "231640fc7535"),
         "fix(suffix-delimiter-rendering) no-space delimiter core token kept whole":
@@ -1064,10 +1067,16 @@ def _claim(rule: dict) -> _Claim:
             _Claim(1, ('family', 'given', 'suffix', 'title'), "1d45596e6fdb"),
         "fix(#367) a title no longer displaces a leading particle out of the leading position":
             _Claim(1, ('family', 'given', 'middle'), "dce0ae6df4be"),
+        "fix(#390) the leading-particle claim takes the particle's group, "
+        "not the rest of the name":
+            _Claim(1, ('family', 'given'), "caf836e3556c"),
     },
     "expected_since_2.1.0.toml": {
         "fix(#367) a title no longer displaces a leading particle out of the leading position":
             _Claim(1, ('family', 'given', 'middle'), "dce0ae6df4be"),
+        "fix(#390) the leading-particle claim takes the particle's group, "
+        "not the rest of the name":
+            _Claim(1, ('family', 'given'), "caf836e3556c"),
     },
 }
 

tools/differential/expected_since_1.4.0.toml

@@ -179,6 +179,28 @@ issue = "fix(comma-precomma-family) pre-comma run reads as family, not given"
 name_regex = "^[\\u0000-\\u024f]*,[\\u0000-\\u024f]*$"
 fields = ["given", "family"]
 
+[[change]]
+issue = "fix(#390) the leading-particle claim takes the particle's group, not the rest of the name"
+# 'de Mesnil Garcia': 1.4's handle_non_first_name_prefix swept every
+# remaining name word into the family, giving last='de Mesnil Garcia'
+# with no first. rules.md#P1 says the claim takes the particle and the
+# ONE piece it attaches to, so 'Garcia' survives by position. Diff is
+# exactly {given, family} -- no suffix moves, nothing is consumed.
+#
+# Its own rule for the reason #372 gave the three names above one: it
+# was falling to the fields-only fix(suffix-routing) below, whose
+# prose is "two-token name with unambiguous trailing suffix stays
+# suffix" and whose own comment says the prose fits all four of its
+# names. 'Garcia' is not a suffix and nothing here routes one, so
+# leaving it there would make that claim false for a fifth name.
+#
+# Narrower than the leading-particle shapes that do NOT diff
+# ('de la Vega', 'de Mesnil Jr.'): those are one group, or two name
+# words with the third a suffix, so the claim takes everything and
+# 1.4 agrees. Only three-or-more-name-word shapes reach this rule.
+name_regex = "^de Mesnil Garcia$"
+fields = ["given", "family"]
+
 [[change]]
 issue = "fix(suffix-routing) two-token name with unambiguous trailing suffix stays suffix"
 # 'Johnson PhD' / 'Smith Jr.' / 'John V' / 'QC MP': v1 routed a lone

tools/differential/expected_since_2.0.0.toml

@@ -283,3 +283,18 @@ issue = "fix(#367) a title no longer displaces a leading particle out of the lea
 # classification.
 name_regex = "(?i)^mr\\.\\s+van\\b"
 fields = ["given", "middle", "family"]
+
+[[change]]
+issue = "fix(#390) the leading-particle claim takes the particle's group, not the rest of the name"
+# 'de Mesnil Garcia': the claim now takes the particle and the ONE
+# piece it attaches to (rules.md#P1), so 'Garcia' survives by position
+# where it used to be swept into the family. Diff is exactly
+# {given, family} -- no suffix moves and nothing is consumed.
+#
+# The leading-particle shapes that do NOT diff are the check on this
+# rule's reach: 'de la Vega' is one group start to finish, and
+# 'de Mesnil Jr.' has two name words because Jr. is a suffix, so the
+# claim takes everything in both and the baseline agrees. Only
+# three-or-more-name-word shapes reach here, and the corpus holds one.
+name_regex = "^de Mesnil Garcia$"
+fields = ["given", "family"]

tools/differential/expected_since_2.1.0.toml

@@ -94,3 +94,18 @@ issue = "fix(#367) a title no longer displaces a leading particle out of the lea
 # classification.
 name_regex = "(?i)^mr\\.\\s+van\\b"
 fields = ["given", "middle", "family"]
+
+[[change]]
+issue = "fix(#390) the leading-particle claim takes the particle's group, not the rest of the name"
+# 'de Mesnil Garcia': the claim now takes the particle and the ONE
+# piece it attaches to (rules.md#P1), so 'Garcia' survives by position
+# where it used to be swept into the family. Diff is exactly
+# {given, family} -- no suffix moves and nothing is consumed.
+#
+# The leading-particle shapes that do NOT diff are the check on this
+# rule's reach: 'de la Vega' is one group start to finish, and
+# 'de Mesnil Jr.' has two name words because Jr. is a suffix, so the
+# claim takes everything in both and the baseline agrees. Only
+# three-or-more-name-word shapes reach here, and the corpus holds one.
+name_regex = "^de Mesnil Garcia$"
+fields = ["given", "family"]