diff --git a/docs/design/rules.md b/docs/design/rules.md index c5eac187..485b8742 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -108,8 +108,8 @@ P1. Rationale: a never-given particle standing alone cannot be "de la Vega" → family="de la Vega" "Sir de Mesnil" → family="de Mesnil" "Mesnil de" family-first → family="Mesnil de" - "de Mesnil Juan" → family="de Mesnil" deviates: #364 (today: family="de Mesnil Juan") - "de Mesnil Juan" → given="Juan" deviates: #364 (today: given="") + "de Mesnil Juan" → family="de Mesnil" + "de Mesnil Juan" → given="Juan" "van Gogh" → given="van" · boundary Accepted: a bare "de" stays the given name — there is nothing to fold into, and inventing a surname would be worse. @@ -121,7 +121,7 @@ P1. Rationale: a never-given particle standing alone cannot be the bare particle reading as a given name, not any name part that begins with one. "Juan de la Vega" family-first → family="Juan" - history: decisions.md#P1 · interacts: P2, P4, P6 · implemented: nameparser/_pipeline/_post_rules.py + history: decisions.md#P1 · interacts: P2, P4, P6 · implemented: nameparser/_pipeline/_assign.py, nameparser/_pipeline/_post_rules.py P2. Rationale: a particle is written as part of the surname it precedes, and a title stands outside the name entirely. diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index 15b5ddab..57b83a1f 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -268,7 +268,37 @@ def _assign_main(seg_idx: int, state: ParseState, order = _effective_order(state.policy, [pieces[i] for i in name_pieces], tokens, dot_divided=bool(state.interpunct_offsets)) - roles = _name_positions(order, len(name_pieces)) + # rules.md#P1: "a never-given particle standing alone where the + # given name would go — or opening the name — marks the name as + # surname-only: the particle run and the one name word it + # attaches to are the family, and any name words beyond that read + # by position." (history: decisions.md#P1) + # The claim happens HERE, before positions are handed out, because + # the particle is evidence about the name and name_order governs + # only what no vocabulary has claimed (decisions.md#O4). Doing it + # after assignment is what made the fold's reach depend on which + # role the order happened to give the particle (#364, #365). + claimed: list[int] = [] + if len(name_pieces) >= 2: + head = pieces[name_pieces[0]] + if (len(head) == 1 + and "particle" in tokens[head[0]].tags + and "vocab:particle-ambiguous" not in tokens[head[0]].tags): + claimed, name_pieces = name_pieces[:2], name_pieces[2:] + if claimed: + # The family slot is taken, so the remainder positions itself + # as if it did not exist: ask for one more slot than there are + # pieces and drop FAMILY. Reusing _name_positions rather than + # re-deriving the order keeps one definition of "by position" + # -- under FAMILY_FIRST a plain count would hand the leftover + # the family a second time. + roles: list[Role] = [ + r for r in _name_positions(order, len(name_pieces) + 1) + if r is not Role.FAMILY] + for piece_idx in claimed: + _set_roles(tokens, pieces[piece_idx], Role.FAMILY) + else: + roles = list(_name_positions(order, len(name_pieces))) for pos, piece_idx in enumerate(name_pieces): _set_roles(tokens, pieces[piece_idx], roles[pos]) for piece_idx in suffix_pieces: diff --git a/nameparser/_pipeline/_post_rules.py b/nameparser/_pipeline/_post_rules.py index 906db723..63c5f5c6 100644 --- a/nameparser/_pipeline/_post_rules.py +++ b/nameparser/_pipeline/_post_rules.py @@ -1,9 +1,7 @@ """Stage: post_rules. -Consumes: tokens (roles assigned), plus pieces and structure -- the -particle fold reads the opening piece of segment 0, or of segment 1 -under a family comma (#359). structure was always read here, for the -rotation gate. +Consumes: tokens (roles assigned), plus structure for the rotation +gate. Produces: tokens with roles adjusted by the post rules. Reads: Policy.patronymic_rules, Policy.middle_as_family; Lexicon.given_name_titles. @@ -11,6 +9,15 @@ Implements rules H1, P1, O1, O2 and O3 of docs/design/rules.md; each is cited at its code below, and P1/O1/O2's history lives in docs/design/decisions.md. + +P1 is SPLIT across two stages as of #390, and the split is the rule's +own shape rather than an accident: a leading particle CLAIMS the +family before positions exist, so that half lives in assign; a lone +particle that positioning has already dropped into the GIVEN role can +only be seen afterwards, so that half stays here. The pieces scan the +leading half used (`_leading_name_piece`, and with it the reading of +state.pieces) went with it -- assign reaches the same piece by peeling +titles before it counts name pieces. """ from __future__ import annotations @@ -37,34 +44,10 @@ r"^(оглу|оглы|оғлу|ўғли|угли|кызы|гызы|қызы|қизи|улы|ұлы|уулу)$", re.I) -_NAME_ROLES = (Role.GIVEN, Role.MIDDLE, Role.FAMILY) - - def _idx(tokens: list[WorkToken], role: Role) -> list[int]: return [i for i, t in enumerate(tokens) if t.role is role] -def _leading_name_piece(state: ParseState, - tokens: list[WorkToken]) -> tuple[int, ...]: - """The piece that OPENS the name, whatever role name_order gave it: - the first piece holding a GIVEN, MIDDLE or FAMILY token, in the - segment the positional read governs. Every piece holding none of - those is walked past -- title and suffix pieces, but NICKNAME and - MAIDEN as well, and anything assign left unroled -- and any number - of them, not only a single leading title. The segment is 0, except - under a family comma, where segment 0 is already fixed as the - surname and the name continues in segment 1. Empty on either of - two exits: that segment does not exist, or none of its pieces - holds a name token.""" - seg = 1 if state.structure is Structure.FAMILY_COMMA else 0 - if seg >= len(state.pieces): - return () - for piece in state.pieces[seg]: - if any(tokens[i].role in _NAME_ROLES for i in piece): - return piece - return () - - def _retag(tokens: list[WorkToken], i: int, role: Role) -> None: tokens[i] = dataclasses.replace(tokens[i], role=role) @@ -99,24 +82,22 @@ def post_rules(state: ParseState) -> ParseState: # attaches to are the family, and any name words beyond that # read by position." (v1 handle_non_first_name_prefix; history: # decisions.md#P1) - # DEVIATION #364: the fold below still takes every remaining name - # word, not just the particle run's own -- de Mesnil Juan gives - # family=de Mesnil Juan where the rule says family=de Mesnil plus - # given=Juan. Pinned by the deviates: markers on P1. - # Values written unquoted deliberately: this note sits INSIDE the - # citation block above (# decisions.md#P1) does not close it -- - # _CITE_RE wants a colon after the ID), and the excerpt check - # takes the first quoted span in the block. - # Code-local: a lone PIECE is the test at both sites, so a - # particle group already chained forward is not a lone particle, - # and rule H1 above cannot be what produces the fold's family - # reading -- H1 is gated on `not families`. - sites = (_leading_name_piece(state, tokens), tuple(givens)) - if len(givens) + len(middles) + len(families) > 1 and any( + # This is P1's SECOND site only. The opening-the-name half moved to + # assign in #390, where the claim can happen before positions are + # handed out; what is left here is the case assign cannot see -- + # a lone particle that positional assignment has already dropped + # into the GIVEN role (Mesnil de under FAMILY_FIRST). It has no + # piece after it to attach to, so the fold is backward and the + # whole remainder is one word anyway. + # Code-local: a lone TOKEN in the given role is the test, so a + # particle group already chained forward never reaches it, and + # rule H1 above cannot be what produces this family reading -- + # H1 is gated on `not families`. + site = tuple(givens) + if len(givens) + len(middles) + len(families) > 1 and ( len(site) == 1 and "particle" in tokens[site[0]].tags - and "vocab:particle-ambiguous" not in tokens[site[0]].tags - for site in sites): + and "vocab:particle-ambiguous" not in tokens[site[0]].tags): for i in givens + middles: _retag(tokens, i, Role.FAMILY) # downstream rules key on the role counts: recompute diff --git a/tests/test_particles.py b/tests/test_particles.py index 3edeb863..ca1f87fb 100644 --- a/tests/test_particles.py +++ b/tests/test_particles.py @@ -390,9 +390,14 @@ def test_leading_non_first_name_prefix_with_patronymic_name_order(self) -> None: self.m(hn.last, "de Mesnil", hn) def test_leading_non_first_name_prefix_with_middle_name_as_last(self) -> None: - # handle_non_first_name_prefix runs first and empties middle_list, so + # The leading-particle claim runs first and empties middle_list, so # the later opt-in handle_middle_name_as_last has nothing left to do. + # The claim takes the particle and the ONE piece it attaches to + # (#390), so 'Garcia' survives as the given name -- before that it + # was swept into the family and first was ''. The interaction this + # test guards is unchanged: middle is empty either way. constants = Constants(middle_name_as_last=True) hn = HumanName("de Mesnil Garcia", constants=constants) - self.m(hn.first, "", hn) - self.m(hn.last, "de Mesnil Garcia", hn) + self.m(hn.first, "Garcia", hn) + self.m(hn.middle, "", hn) + self.m(hn.last, "de Mesnil", hn) diff --git a/tests/v2/pipeline/test_post_rules.py b/tests/v2/pipeline/test_post_rules.py index ad456f60..12e9f1c1 100644 --- a/tests/v2/pipeline/test_post_rules.py +++ b/tests/v2/pipeline/test_post_rules.py @@ -133,10 +133,9 @@ def test_degenerate_bare_particle_stays_given() -> None: # the leading particle chains the rest of the name into the family ("de Mesnil", "de Mesnil", "", ""), ("de la Vega", "de la Vega", "", ""), - # three pieces, so the fold has a MIDDLE to move as well as the - # given -- the `givens + middles` half of the repair, and the only - # no-comma corpus name that reaches it - ("de Mesnil Garcia", "de Mesnil Garcia", "", ""), + # three pieces, so a name word survives the claim -- the only + # no-comma corpus name that reaches it, and the one #390 moved + ("de Mesnil Garcia", "de Mesnil", "Garcia", ""), # ... and the trailing suffix run is peeled before the rule looks, # comma or no comma (NO_COMMA and SUFFIX_COMMA both fold) ("de Mesnil MD", "de Mesnil", "", "MD"), @@ -152,21 +151,17 @@ def test_family_first_folds_leading_never_given_particle( @pytest.mark.parametrize("policy", _FAMILY_FIRST) -def test_leading_piece_scan_skips_pieces_that_hold_no_name( +def test_titles_are_peeled_before_the_leading_particle_claim( policy: Policy) -> None: - # `_leading_name_piece` walks PAST pieces carrying no name role - # rather than reading piece 0 -- and past the first such piece, not - # only over a single title. 'Mr. de Mesnil' cannot show that: its - # particle is chained into one piece with 'Mesnil', so the scan - # lands on a two-token piece and the rule declines either way. - # Here a mid-name suffix word breaks that chain, leaving the - # particle a piece of its own BEHIND a title piece. Without the - # skip, or reading only pieces[0], the scan finds the title (or - # nothing) and the name splits: given='MD', middle='Mesnil', - # family='de'. - out = _parsed("Dr. de MD Mesnil", policy) + # The claim reads the first NAME piece, not pieces[0]: a title in + # front must not hide the particle behind it. assign gets this by + # peeling titles before it counts name pieces (#390 moved the claim + # there from post_rules, retiring the _leading_name_piece scan that + # used to walk past non-name pieces). Without the peel the title is + # piece 0, no claim fires, and the name splits by position. + out = _parsed("Dr. de Mesnil", policy) assert _by_role(out, Role.TITLE) == "Dr." - assert _by_role(out, Role.FAMILY) == "de MD Mesnil" + assert _by_role(out, Role.FAMILY) == "de Mesnil" assert not _by_role(out, Role.GIVEN) assert not _by_role(out, Role.MIDDLE) @@ -198,8 +193,12 @@ def test_family_first_leading_particle_cases_that_do_not_fold( @pytest.mark.parametrize("text,title,given,middle,family,suffix", [ ("de Mesnil", "", "", "", "de Mesnil", ""), ("de la Vega", "", "", "", "de la Vega", ""), - ("de Mesnil Garcia", "", "", "", "de Mesnil Garcia", ""), - ("Dr. de MD Mesnil", "Dr.", "", "", "de MD Mesnil", ""), + ("de Mesnil Garcia", "", "Garcia", "", "de Mesnil", ""), + # garbage in, garbage out: MD is suffix vocabulary sitting + # mid-name, so the piece the particle attaches to is "MD". + # Both readings of this input are garbage; pinned only so the + # claim's reach is visible, never as a shape to design around + ("Dr. de MD Mesnil", "Dr.", "Mesnil", "", "de MD", ""), ("de Mesnil MD", "", "", "", "de Mesnil", "MD"), ("De Mesnil, MD", "", "", "", "De Mesnil", "MD"), ("Mr. de Mesnil", "Mr.", "", "", "de Mesnil", ""), @@ -216,7 +215,7 @@ def test_family_first_leading_particle_cases_that_do_not_fold( ("Smith, de Mesnil", "", "", "", "Smith de Mesnil", ""), ("Smith, van Gogh", "", "van", "Gogh", "Smith", ""), ]) -def test_default_order_is_unchanged_by_the_family_first_fold( +def test_leading_particle_claim_reads_alike_in_the_default_order( text: str, title: str, given: str, middle: str, family: str, suffix: str) -> None: out = _parsed(text) diff --git a/tests/v2/test_ledger_guards.py b/tests/v2/test_ledger_guards.py index dc8d7d2e..916d081d 100644 --- a/tests/v2/test_ledger_guards.py +++ b/tests/v2/test_ledger_guards.py @@ -1020,6 +1020,9 @@ def _claim(rule: dict) -> _Claim: _Claim(215, ('given', 'suffix', 'title'), "f16a0e79cba3"), "fix(comma-precomma-family) pre-comma run reads as family, not given": _Claim(215, ('family', 'given'), "f16a0e79cba3"), + "fix(#390) the leading-particle claim takes the particle's group, " + "not the rest of the name": + _Claim(1, ('family', 'given'), "caf836e3556c"), "fix(suffix-routing) two-token name with unambiguous trailing suffix stays suffix": _Claim(751, ('family', 'given', 'suffix'), "231640fc7535"), "fix(suffix-delimiter-rendering) no-space delimiter core token kept whole": @@ -1064,10 +1067,16 @@ def _claim(rule: dict) -> _Claim: _Claim(1, ('family', 'given', 'suffix', 'title'), "1d45596e6fdb"), "fix(#367) a title no longer displaces a leading particle out of the leading position": _Claim(1, ('family', 'given', 'middle'), "dce0ae6df4be"), + "fix(#390) the leading-particle claim takes the particle's group, " + "not the rest of the name": + _Claim(1, ('family', 'given'), "caf836e3556c"), }, "expected_since_2.1.0.toml": { "fix(#367) a title no longer displaces a leading particle out of the leading position": _Claim(1, ('family', 'given', 'middle'), "dce0ae6df4be"), + "fix(#390) the leading-particle claim takes the particle's group, " + "not the rest of the name": + _Claim(1, ('family', 'given'), "caf836e3556c"), }, } diff --git a/tools/differential/expected_since_1.4.0.toml b/tools/differential/expected_since_1.4.0.toml index dde79d00..5351132b 100644 --- a/tools/differential/expected_since_1.4.0.toml +++ b/tools/differential/expected_since_1.4.0.toml @@ -179,6 +179,28 @@ issue = "fix(comma-precomma-family) pre-comma run reads as family, not given" name_regex = "^[\\u0000-\\u024f]*,[\\u0000-\\u024f]*$" fields = ["given", "family"] +[[change]] +issue = "fix(#390) the leading-particle claim takes the particle's group, not the rest of the name" +# 'de Mesnil Garcia': 1.4's handle_non_first_name_prefix swept every +# remaining name word into the family, giving last='de Mesnil Garcia' +# with no first. rules.md#P1 says the claim takes the particle and the +# ONE piece it attaches to, so 'Garcia' survives by position. Diff is +# exactly {given, family} -- no suffix moves, nothing is consumed. +# +# Its own rule for the reason #372 gave the three names above one: it +# was falling to the fields-only fix(suffix-routing) below, whose +# prose is "two-token name with unambiguous trailing suffix stays +# suffix" and whose own comment says the prose fits all four of its +# names. 'Garcia' is not a suffix and nothing here routes one, so +# leaving it there would make that claim false for a fifth name. +# +# Narrower than the leading-particle shapes that do NOT diff +# ('de la Vega', 'de Mesnil Jr.'): those are one group, or two name +# words with the third a suffix, so the claim takes everything and +# 1.4 agrees. Only three-or-more-name-word shapes reach this rule. +name_regex = "^de Mesnil Garcia$" +fields = ["given", "family"] + [[change]] issue = "fix(suffix-routing) two-token name with unambiguous trailing suffix stays suffix" # 'Johnson PhD' / 'Smith Jr.' / 'John V' / 'QC MP': v1 routed a lone diff --git a/tools/differential/expected_since_2.0.0.toml b/tools/differential/expected_since_2.0.0.toml index d8cb0fe7..a42f3124 100644 --- a/tools/differential/expected_since_2.0.0.toml +++ b/tools/differential/expected_since_2.0.0.toml @@ -283,3 +283,18 @@ issue = "fix(#367) a title no longer displaces a leading particle out of the lea # classification. name_regex = "(?i)^mr\\.\\s+van\\b" fields = ["given", "middle", "family"] + +[[change]] +issue = "fix(#390) the leading-particle claim takes the particle's group, not the rest of the name" +# 'de Mesnil Garcia': the claim now takes the particle and the ONE +# piece it attaches to (rules.md#P1), so 'Garcia' survives by position +# where it used to be swept into the family. Diff is exactly +# {given, family} -- no suffix moves and nothing is consumed. +# +# The leading-particle shapes that do NOT diff are the check on this +# rule's reach: 'de la Vega' is one group start to finish, and +# 'de Mesnil Jr.' has two name words because Jr. is a suffix, so the +# claim takes everything in both and the baseline agrees. Only +# three-or-more-name-word shapes reach here, and the corpus holds one. +name_regex = "^de Mesnil Garcia$" +fields = ["given", "family"] diff --git a/tools/differential/expected_since_2.1.0.toml b/tools/differential/expected_since_2.1.0.toml index 796b1062..c136315f 100644 --- a/tools/differential/expected_since_2.1.0.toml +++ b/tools/differential/expected_since_2.1.0.toml @@ -94,3 +94,18 @@ issue = "fix(#367) a title no longer displaces a leading particle out of the lea # classification. name_regex = "(?i)^mr\\.\\s+van\\b" fields = ["given", "middle", "family"] + +[[change]] +issue = "fix(#390) the leading-particle claim takes the particle's group, not the rest of the name" +# 'de Mesnil Garcia': the claim now takes the particle and the ONE +# piece it attaches to (rules.md#P1), so 'Garcia' survives by position +# where it used to be swept into the family. Diff is exactly +# {given, family} -- no suffix moves and nothing is consumed. +# +# The leading-particle shapes that do NOT diff are the check on this +# rule's reach: 'de la Vega' is one group start to finish, and +# 'de Mesnil Jr.' has two name words because Jr. is a suffix, so the +# claim takes everything in both and the baseline agrees. Only +# three-or-more-name-word shapes reach here, and the corpus holds one. +name_regex = "^de Mesnil Garcia$" +fields = ["given", "family"]