From 510810c5c56e0c92739d676bbe53657d3bb1cbdb Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Mon, 7 Sep 2026 23:50:45 -0700 Subject: [PATCH 1/5] feat(types): AmbiguityKind.GIVEN_OR_FAMILY, reported where O5's convention picked the field MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit rules.md#O5 fixes a reading it does not determine: at one name word the positional rule has nothing to compare, so the library picks one of two equally consistent readings and picks it the same way every time. Until now it picked silently. It now reports `given-or-family`, with `detail` naming the field the convention chose -- the field follows the read order, which is why the kind cannot name it, the PARTICLE_OR_GIVEN precedent. The report is as narrow as the convention, which the naive condition is not: the one-name-piece branch in assign runs for 141 of the 1119 corpus names, because a title peel, a nickname, a maiden clause, a trailing credential or a script order can each leave one piece standing. Guarded on O5's own carve-out list plus two the rule states without naming -- a script whose convention settles the order (W4), and a piece with no letter or digit in it (A2) -- the population is 27 corpus names. `abdul` reports nothing, bound-given vocabulary having claimed it; `de` reports nothing, a lone particle's reading being P4's. W4 silences the report by AUTHORSHIP, not by agreement. Comparing the order used against the order declared would have called `毛泽东` and `高橋一郎` undecided under `Policy(name_order=FAMILY_FIRST)`, where the Han entry and the declaration happen to agree -- 26 of the 31 W4-decided CJK corpus names, measured. `_effective_order` now returns an `Order` saying whether a `script_orders` entry RESOLVED the order, and the emitter reads that instead; all 31 stay silent under both policies, while `マイケル`, `王·Smith` and `田中、太郎` -- three different script rules DECLINING -- still report. A suffix beside the lone word decides nothing, so it does not silence the report either: `"Smith Jr."` and O5's own `"'Smitty' Jones Jr."` are S2's peel taking the suffix and the convention placing the one name word left. Nine of the 27 are that shape. The bare-suffix carve-out is the opposite case and stays out, the count being read off the peel rather than off the name pieces: reporting `PhD` and `QC MP` is H4's, and arrives with it. No parse changes: every role of every corpus name is identical before and after, measured against the parent tree, and the three 2.x ledgers classify the 27 on `_ambiguities` alone -- except at 2.0.0, where the two glued-honorific names carry #308's peel in the same diff and their two rules must admit it. Co-Authored-By: Claude Fable 5.1 --- docs/design/rules.md | 21 +++- nameparser/_pipeline/_assign.py | 98 +++++++++++++-- nameparser/_types.py | 13 ++ tests/v2/cases.py | 107 ++++++++++++++-- tests/v2/test_contracts.py | 1 + tests/v2/test_facade_cases.py | 7 ++ tests/v2/test_ledger_guards.py | 123 +++++++++++++++++++ tools/differential/corpus_rules.jsonl | 2 + tools/differential/expected_since_2.0.0.toml | 86 ++++++++++++- tools/differential/expected_since_2.1.0.toml | 78 ++++++++++++ tools/differential/expected_since_2.2.0.toml | 78 ++++++++++++ 11 files changed, 588 insertions(+), 26 deletions(-) diff --git a/docs/design/rules.md b/docs/design/rules.md index fb49dfe5..cd82e80c 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -1068,10 +1068,15 @@ O5. Rationale: O4 reads a name by comparing where its words stand, suffix stands beside them too, which its count does not set aside; and a maiden name beside it does the same (M4), except where the vocabulary or the word's own shape has claimed the - word already. What the library should SAY about a reading it - merely fixed — whether the convention is worth reporting as an - ambiguity — is open, and #449 holds the measurement that makes - it a design question rather than an implementation one. + word already. The convention is reported: a name whose one name + word nothing else decided carries a `given-or-family` ambiguity + naming the field the convention chose. The report is exactly as + narrow as the convention, so every rule named above silences it + where it fires, a comma silences it, a script whose own + convention settles the order (W4) silences it, and so does the + word's own claim — a particle, a bound given name, an initial's + shape. A word with no letter or digit in it is no name word and + reports nothing (A2). "Smith" → given="Smith" "Garcia" family-first → family="Garcia" "Sir John" → given="John" @@ -1080,6 +1085,14 @@ O5. Rationale: O4 reads a name by comparing where its words stand, "Mr. Johnson" → family="Johnson" · boundary "'Smitty' Jones" → family="Jones" · boundary "Smith née Jones" → family="Smith" · boundary + "Andrew" → ambiguities=("given-or-family",) + "Garcia" family-first → ambiguities=("given-or-family",) + "Juan & Garcia" → ambiguities=("given-or-family",) + "'Smitty' Jones Jr." → ambiguities=("given-or-family",) + "Smith Jr." → ambiguities=("given-or-family",) + "Dr. Smith" → ambiguities=() · boundary + "Smith née Jones" → ambiguities=() + "abd née Jones" → ambiguities=() interacts: O4, H1, N3, M4 · implemented: nameparser/_pipeline/_assign.py ## Scripts & writing systems (W) diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index cb057f04..1cf92a84 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -38,6 +38,7 @@ import dataclasses from collections.abc import Sequence, Set +from typing import NamedTuple from nameparser._lexicon import Lexicon from nameparser._pipeline._vocab import ( @@ -59,6 +60,17 @@ def _set_roles(tokens: list[WorkToken], piece: tuple[int, ...], tokens[i] = dataclasses.replace(tokens[i], role=role) +#: Tags that say the word's own reading was claimed before position +#: could speak, so O5's convention decided nothing. Two are M4's +#: `_NEVER_FLIPPED` pair, for M4's reasons -- a bound given-name word is +#: vocabulary claiming the word as a given name, `initial` is the shape +#: claim -- and `particle` is here because a lone particle's reading is +#: P4's. None is a predicate this emitter owns: they are read off the +#: tags classify already recorded (mechanisms.md#TWO-LAYER-ASSIGN). +_WORD_ALREADY_CLAIMED = frozenset({ + "particle", "vocab:bound-given", "initial"}) + + # rules.md#H2: "an abbreviation opening the part of the name that # carries the given name — the whole name, or the part after a # family comma — reads as a title even when unlisted" -- the count is @@ -75,6 +87,20 @@ def _peel_leading_titles(pieces: tuple[tuple[int, ...], ...], return n +class Order(NamedTuple): + """What _effective_order made of a name's scripts. `roles` is the + order the positional read uses; `by_script` says a script_orders + entry RESOLVED it, which is not the same as `roles` happening to + equal the declared name_order -- under a declared family-first + order a Han name's W4 entry and the declaration agree, and only + this flag distinguishes the script rule DECIDING the reading from + the caller's declaration standing unopposed. O5's convention + report reads it (#449).""" + + roles: tuple[Role, Role, Role] + by_script: bool + + # rules.md#W4: "a name written wholly in one East Asian script, or in # the kana-licensed Japanese repertoire, reads family-first whatever # order the caller declared; a wholly-katakana name keeps the declared @@ -82,7 +108,7 @@ def _peel_leading_titles(pieces: tuple[tuple[int, ...], ...], def _effective_order(policy: Policy, pieces: list[tuple[int, ...]], tokens: list[WorkToken], - *, dot_divided: bool) -> tuple[Role, Role, Role]: + *, dot_divided: bool) -> Order: """script_orders resolution (#271): when every name piece is written wholly in ONE script that has an entry, that script's order governs the positional read; anything else -- Latin, mixed @@ -107,13 +133,18 @@ def _effective_order(policy: Policy, resolves the ORDER for a whole name; `_vocab.effective_script` resolves the SCRIPT for a single token. This function calls that one per token below. + + Returns an Order: the roles triple, and `by_script` set only on + the one path where an entry answered. Every fallback below is a + script rule DECLINING, and reports it as such. """ + declared = Order(policy.name_order, by_script=False) # #298 transcription marker -- see the docstring; codepoint-scoped # (only U+00B7 records; decisions.md#T3) if dot_divided: - return policy.name_order + return declared if not policy.script_orders: - return policy.name_order + return declared # Collect every token's script rather than comparing pairwise as # tokens are seen: the kana license needs the WHOLE set (a Han # piece and a Hiragana piece only license together, never one at a @@ -124,15 +155,19 @@ def _effective_order(policy: Policy, script = effective_script(tokens[i].text) if script is None: # Latin, mixed, or a script with no entry: never a key - return policy.name_order + return declared found.add(script) resolved = resolve_script_set(found) if resolved is None: # e.g. Han+Hangul: two scripts, neither the kana license's # Han/Hiragana/Katakana repertoire -- no single tradition - return policy.name_order - return next((order for s, order in policy.script_orders - if s is resolved), policy.name_order) + return declared + for script, order in policy.script_orders: + if script is resolved: + return Order(order, by_script=True) + # the resolved script has no entry: the declaration stands, and + # nothing about the writing system decided the reading + return declared # rules.md#O4: "words no vocabulary has claimed read by position. In @@ -224,14 +259,57 @@ def _assign_main(seg_idx: int, state: ParseState, # AFTER both peels, and load-bearing: the script test sees the NAME # pieces only, so a Latin title or suffix ('Dr. 毛 泽东', '毛 泽东, # PhD') cannot make a wholly-CJK name look mixed-script. - order = _effective_order(state.policy, - [pieces[i] for i in name_pieces], tokens, - dot_divided=bool(state.interpunct_offsets)) + resolved = _effective_order(state.policy, + [pieces[i] for i in name_pieces], tokens, + dot_divided=bool(state.interpunct_offsets)) + order = resolved.roles roles = _name_positions(order, len(name_pieces)) for pos, piece_idx in enumerate(name_pieces): _set_roles(tokens, pieces[piece_idx], roles[pos]) for piece_idx in suffix_pieces: _set_roles(tokens, pieces[piece_idx], Role.SUFFIX) + # rules.md#O5's convention, reported at the site that applies it + # (mechanisms.md#AMBIGUITY-AT-THE-DECISION-SITE). The guard is + # O5's own carve-out list read as code -- a leading title, a + # maiden name, a group-flagged credential and the word's own claim + # each mean something else decided -- plus one the rule states + # without naming: a script whose order convention settles the + # reading (W4) decided it too, which is `resolved.by_script` + # rather than a comparison against name_order, a declared + # family-first order agreeing with a Han name's entry being + # agreement and not authorship. The two shapes O5 also names never + # reach this line: a nickname beside a lone name word is N3's and + # returns above, and a family comma names the family before + # segment 0 is read positionally, so neither needs a clause here. + # A suffix beside the word is NOT such a shape -- 'Smith Jr.' and + # "'Smitty' Jones Jr." are the convention placing a lone name word, + # which is why S2's peel does not silence it. The count comes off + # the PEEL rather than off name_pieces, which is what leaves the + # bare-suffix carve-out above (peeled.names == 0, the first + # post-nominal read as the name for want of anything else) out: + # that reading is a different convention, and reporting it is + # H4's. The role comes off the token for the reason stated at the + # particle emitter below. + if (peeled.names == 1 and n == 0 + and not resolved.by_script + and not any(t.role is Role.MAIDEN for t in tokens)): + head = pieces[name_pieces[0]] + text = " ".join(tokens[i].text for i in head) + if (not any(_WORD_ALREADY_CLAIMED & tokens[i].tags for i in head) + # A2's content test: a piece with no alphanumeric + # character is no name word, and the name it sits in + # assembles empty -- so a convention report there would + # describe a reading nobody got. parse("(") keeps its + # unbalanced-delimiter report and gains nothing here. + and any(c.isalnum() for c in text)): + token = tokens[head[0]] + assert token.role is not None + ambiguities.append(PendingAmbiguity( + AmbiguityKind.GIVEN_OR_FAMILY, + f"{text!r} is the only name word and nothing else " + f"decides it; read as a {token.role.value} name by " + f"convention, which follows the read order", + tuple(head))) for piece in peeled.picks: # every pick is in rest, so the loops above just gave it a role token = tokens[piece[0]] diff --git a/nameparser/_types.py b/nameparser/_types.py index 0e0d47dc..8b1893c8 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -425,6 +425,19 @@ class AmbiguityKind(StrEnum): #: is the family and nothing about this, so the fork is real and #: ``detail`` names the word it turned on. PARTICLE_OR_GIVEN = "particle-or-given" + #: A name of one name word that nothing else decided had to be read + #: as one field or the other, and both readings fit it equally well + #: -- "Andrew", "Smith". The convention picks the given name under + #: the default order and the family name under a declared + #: family-first one, the same way every time, so ``detail`` names + #: the field the convention chose rather than the kind naming it: + #: the same reason PARTICLE_OR_GIVEN cannot. A name something DID + #: decide reports nothing -- a title, a nickname, a maiden name, a + #: comma, a script whose own convention settles the order, or the + #: vocabulary claiming the word (a particle, a bound given name, an + #: initial's shape) each settle the reading, and a settled reading + #: is not a fork. + GIVEN_OR_FAMILY = "given-or-family" #: A nickname/maiden delimiter opened without closing (or closed #: without opening); the text was kept as literal name content, so #: the tokens are the one the stray character ended up inside. diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 85fac3bb..76e25fba 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -497,6 +497,7 @@ def _check_cjk_shape_purity(self) -> None: Case("by_design_trailing_mc_reads_as_a_credential", "Donald Mc", {"given": "Donald", "suffix": "Mc"}, classification="fix(suffix-routing)", + ambiguities=("given-or-family",), notes="#454, dispositioned by design with this bundle. " "1.4.0 read last 'Mc'; the 1.4.0 ledger's " "fix(suffix-routing) rule -- a two-token name ending " @@ -517,7 +518,10 @@ def _check_cjk_shape_purity(self) -> None: "Mc', both because 'mc' is a never-given particle " "(#360), which is the membership that actually bears " "on #454's example. See " - "decisions.md#suffix-acronym-collisions"), + "decisions.md#suffix-acronym-collisions. The peel " + "leaves 'Donald' the only name word, so O5's " + "convention places it and says so (#449): roles " + "parity, the flag is #449's"), Case("ambiguous_acronym_suffix_with_middle", "John Q Smith MA", {"given": "John", "middle": "Q", "family": "Smith", "suffix": "MA"}, @@ -1180,9 +1184,12 @@ def _check_cjk_shape_purity(self) -> None: Case("doubled_comma_given_kept", "Doe, John,, Jr.", {"given": "John", "family": "Doe", "suffix": "Jr."}), Case("single_trailing_comma_cosmetic", "John,", - {"given": "John"}, + {"given": "John"}, ambiguities=("given-or-family",), notes="v1 collapse_whitespace strips exactly ONE trailing " - "comma before parsing"), + "comma before parsing -- and a comma stripped before " + "parsing decided nothing, so what is left is one name " + "word and O5's convention reports it (#449). Roles " + "parity; the flag is #449's"), Case("double_trailing_comma_structural", "Doe,,", {"family": "Doe"}, notes="one trailing comma is cosmetic, the second is " @@ -1339,7 +1346,11 @@ def _check_cjk_shape_purity(self) -> None: notes="shape 2's post-comma Middle slot, in the form it is " "usually written after a family comma -- an initial", shape=2), - Case("single", "John", {"given": "John"}), + Case("single", "John", {"given": "John"}, + ambiguities=("given-or-family",), + notes="the plainest O5 shape there is: nothing decides one " + "name word, so the convention picks the field and says " + "so (#449). Roles parity; the flag is #449's"), Case("title_only", "Dr.", {"title": "Dr."}), Case("double_comma_suffix", "Smith, John, Jr.", {"given": "John", "family": "Smith", "suffix": "Jr."}), @@ -2212,6 +2223,55 @@ def _check_cjk_shape_purity(self) -> None: "which is what makes this differ from 1.4.0 (first " "'J.' / middle 'née Jones' / last 'Smith' / suffix " "'V', measured 2026-08-27)"), + Case("lone_name_word_reports_the_convention", "Andrew", + {"given": "Andrew"}, ambiguities=("given-or-family",), + classification="feat(#449)", + notes="rules.md#O5 -- nothing decided this reading, so the " + "convention picked the field and says so"), + Case("lone_name_word_reports_the_convention_family_first", "Garcia", + {"family": "Garcia"}, policy=Policy(name_order=FAMILY_FIRST), + ambiguities=("given-or-family",), classification="feat(#449)", + notes="the kind cannot name the field -- the convention " + "follows the read order, the PARTICLE_OR_GIVEN precedent"), + Case("lone_name_word_joined_by_a_connective_is_one_word", "Juan & Garcia", + {"given": "Juan & Garcia"}, ambiguities=("given-or-family",), + classification="feat(#449)", + notes="rules.md#P3 -- a joined part is one name word wherever " + "another rule counts them, and O5 counts them"), + Case("lone_name_word_bound_given_vocabulary_decides", "abdul", + {"given": "abdul"}, classification="parity", + notes="negative control: the vocabulary claimed the word as a " + "given name, so the convention decided nothing"), + Case("lone_name_word_particle_reading_is_p4s", "de", + {"given": "de"}, classification="parity", + notes="negative control: a lone particle's reading is P4's, " + "not O5's convention"), + Case("lone_name_word_beside_a_suffix_still_reports", "Smith Jr.", + {"given": "Smith", "suffix": "Jr."}, + ambiguities=("given-or-family",), classification="feat(#449)", + notes="rules.md#S2 peels the suffix and leaves ONE name word, " + "which O5's convention then places -- the peel decided " + "the suffix, not the field"), + Case("lone_name_word_beside_a_nickname_and_a_suffix_reports", + "'Smitty' Jones Jr.", + {"given": "Jones", "suffix": "Jr.", "nickname": "Smitty"}, + ambiguities=("given-or-family",), classification="feat(#449)", + notes="rules.md#O5's own example: N3's count does not set " + "aside a suffix standing beside the nickname, so N3 " + "declines and the convention is what places 'Jones'"), + Case("lone_name_word_title_decides_it", "Dr. Smith", + {"title": "Dr.", "family": "Smith"}, classification="parity", + notes="negative control: H1 decided it"), + Case("lone_name_word_nickname_decides_it", "'Smitty' Smith", + {"family": "Smith", "nickname": "Smitty"}, classification="parity", + notes="boundary: N3 returns before the O5 site is reached, so " + "the convention never ran -- not a control of any guard " + "clause, which is why the emitter carries none"), + Case("lone_name_word_comma_decides_it", "Smith, Andrew", + {"given": "Andrew", "family": "Smith"}, classification="parity", + notes="boundary: the comma named the family before the " + "positional read, so one name word never stood alone " + "here -- not a control of any guard clause"), Case("marker_led_clause_in_a_quote_pair", 'Jane Smith "née Jones"', {"given": "Jane", "family": "Smith", "maiden": "Jones"}, @@ -2563,7 +2623,11 @@ def _check_cjk_shape_purity(self) -> None: ambiguities=("suffix-or-name",)), Case("phd_split", "John Ph. D.", {"given": "John", "suffix": "Ph. D."}, - notes="v1 fix_phd; healed via the stable 'joined' tag"), + ambiguities=("given-or-family",), + notes="v1 fix_phd; healed via the stable 'joined' tag. Roles " + "parity; the flag is #449's -- a split credential beside " + "the lone word decides the field no more than a whole " + "one does"), Case("phd_split_mid_name", "Dr. John Ph. D. Smith", {"title": "Dr.", "given": "John", "family": "Smith", "suffix": "Ph. D."}), @@ -2604,9 +2668,12 @@ def _check_cjk_shape_purity(self) -> None: Case("suffix_stays_suffix", "Johnson PhD", {"given": "Johnson", "suffix": "PhD"}, classification="fix(suffix-routing)", + ambiguities=("given-or-family",), notes="v1 routes a lone trailing suffix to family " "(first=Johnson last=PhD); v2 keeps recognized " - "suffixes in suffix"), + "suffixes in suffix -- which leaves 'Johnson' the one " + "name word O5's convention places, reported since " + "#449. Roles parity; the flag is #449's"), Case("suffix_stays_suffix_title", "Mr. Johnson PhD", {"title": "Mr.", "family": "Johnson", "suffix": "PhD"}, classification="fix(#410)", @@ -3295,6 +3362,14 @@ def _check_cjk_shape_purity(self) -> None: notes="no default Han segmentation: one token, and a lone " "wholly-Han token takes the script order's first " "role = family"), + Case("han_unspaced_family_first_declared_reports_nothing", "毛泽东", + {"family": "毛泽东"}, policy=Policy(name_order=FAMILY_FIRST), + notes="W4 AUTHORED this reading, and a declared family-first " + "order agreeing with the Han entry is agreement, not " + "authorship: the script rule resolved the order, so O5's " + "convention decided nothing and reports nothing (#449). " + "The one row that would fail if the emitter compared the " + "order used against the order declared"), Case("mixed_script_untouched_by_script_orders", "John 王", {"given": "John", "family": "王"}, notes="effective_script is None for a mixed name: script_orders " @@ -3395,10 +3470,13 @@ def _check_cjk_shape_purity(self) -> None: notes="hiragana earns a script_orders entry in its own right: " "a lone token takes the entry's first role"), Case("ja_lone_katakana_stays_given", "マイケル", - {"given": "マイケル"}, + {"given": "マイケル"}, ambiguities=("given-or-family",), notes="parity: katakana deliberately has no entry, so the " "positional default holds -- transcribed foreign names " - "keep source order"), + "keep source order. W4 DECLINING is what leaves the " + "reading to O5's convention, which reports it (#449); " + "a name whose script order decides it stays silent. " + "Roles parity; the flag is #449's"), Case("ja_iteration_mark_is_han", "佐々木 太郎", {"family": "佐々木", "given": "太郎"}, classification="fix(#272)", @@ -3492,13 +3570,15 @@ def _check_cjk_shape_purity(self) -> None: "comma: the marker is structure-independent", tolerated=True), Case("zh_interpunct_half_flanked_stays", "王·Smith", - {"given": "王·Smith"}, + {"given": "王·Smith"}, ambiguities=("given-or-family",), notes="one classified neighbor is not enough: the guard " "requires both, so the undivided dot remains part of " "the word -- declining, not deciding. Swept as a " "2026-09-01 tolerated candidate and declined on the " "same boundary as 'John 王': the Latin is a name part, " - "not a wrapper"), + "not a wrapper. T3 declining is what leaves the " + "reading to O5's convention, which reports it (#449). " + "Roles parity; the flag is #449's"), Case("zh_honorific_suffix_spaced", "王小明 先生", {"family": "王小明", "suffix": "先生"}, classification="fix(#307) + fix(#271)", @@ -3761,6 +3841,7 @@ def _check_cjk_shape_purity(self) -> None: Case("latin_stem_glued_kana_honorific", "Andersonさん", {"given": "Anderson", "suffix": "さん"}, classification="fix(#308)", + ambiguities=("given-or-family",), notes="no script precondition on the remainder -- the tail " "is the license. Japanese text about a foreigner, and " "the Latin remainder keeps the positional default. " @@ -3777,6 +3858,7 @@ def _check_cjk_shape_purity(self) -> None: Case("latin_stem_glued_hangul_honorific", "Anderson선생님", {"given": "Anderson", "suffix": "선생님"}, classification="fix(#308)", + ambiguities=("given-or-family",), notes="the hangul twin of latin_stem_glued_kana_honorific, " "and the one that shows why a post-nominal is not a " "surname site: 선 is a listed census surname, so the " @@ -3784,7 +3866,10 @@ def _check_cjk_shape_purity(self) -> None: "-- the stage dissecting the honorific it had just " "manufactured. Single-issue for the same reason as its " "kana twin: the remainder is Latin, so #271 never " - "applies"), + "applies -- and a remainder no script order can place " + "is a remainder O5's convention places, which is what " + "#449 reports here and on the kana twin. Roles " + "parity; the flag is #449's"), Case("ko_honorific_glued_doctor", "김민준박사님", {"family": "김", "given": "민준", "suffix": "박사님"}, classification="fix(#308) + fix(#271)", diff --git a/tests/v2/test_contracts.py b/tests/v2/test_contracts.py index 7699c127..07e1c340 100644 --- a/tests/v2/test_contracts.py +++ b/tests/v2/test_contracts.py @@ -13,6 +13,7 @@ AmbiguityKind.COMMA_STRUCTURE: "Smith, John, Extra, Jr.", AmbiguityKind.SUFFIX_OR_NICKNAME: "JEFFREY (JD) BRICKEN", AmbiguityKind.SUFFIX_OR_NAME: "John Smith MA", + AmbiguityKind.GIVEN_OR_FAMILY: "Andrew", # no emitter yet -- arrives with locale-pack order detection (2.x) AmbiguityKind.ORDER: None, # 남 and 남궁 are both shipped surnames, so longest-first picks diff --git a/tests/v2/test_facade_cases.py b/tests/v2/test_facade_cases.py index 926a510e..b4def303 100644 --- a/tests/v2/test_facade_cases.py +++ b/tests/v2/test_facade_cases.py @@ -88,6 +88,13 @@ # initial veto surviving #430 is core-only "family_comma_strict_keeps_the_initial_veto", "ja_honorific_glued_family_comma_credential_pair_strict_knob", + # #449: O5's convention under a DECLARED family-first order, which + # is the half of the rule v1 has no way to say -- the given-first + # half ("Andrew") is an ordinary row and runs here. The Han row + # beside it declares the same order to pin W4 AUTHORING the + # reading, and is core-only for the same reason. + "lone_name_word_reports_the_convention_family_first", + "han_unspaced_family_first_declared_reports_nothing", }) diff --git a/tests/v2/test_ledger_guards.py b/tests/v2/test_ledger_guards.py index d297eae5..525c8872 100644 --- a/tests/v2/test_ledger_guards.py +++ b/tests/v2/test_ledger_guards.py @@ -919,6 +919,28 @@ def test_case_shape_ids_exist_in_the_inventory() -> None: "fix(#346) a renunciate title and one name word leave the name a given name": ("Swami Vivekananda Saraswati", "Guru Gobind Singh", "Rabbi Cohen", "Swami", "Mr Guru Nanak"), + # #449's six rules. The five CJK ones are literal-anchored on one + # corpus name each, so _CORPUS_CLAIMS cannot move under a widening + # that reaches only names the corpora lack; these probes are the + # wall. Each rule's boundary is the same name with something + # DECIDING it -- a title, a second name word, a comma -- because + # that is the widening the rule invites. A SUFFIX beside the word + # is deliberately not such a probe: it decides nothing, and + # 'Smith Jr.' is a member of the alternation rather than a wall + # against it. + "feat(#449) a lone name word reports given-or-family": + ("Dr. Andrew", "Andrew Smith", "Smith, Andrew", "abdul", "de", + "Dr. Smith Jr."), + "feat(#449) a wholly-katakana name keeps the declared order, so the convention decides it": + ("マイケル ジャクソン", "マイケル・ジャクソン", "Dr. マイケル"), + "feat(#449) an interpunct transcription declines the script order, so the convention decides it": + ("王·Smith Jones", "王 Smith", "Dr. 王·Smith"), + "feat(#449) an ideographic comma leaves one name word, and the convention decides it": + ("田中、太郎、次郎", "田中 太郎", "Dr. 田中、太郎"), + "feat(#449) a glued Japanese honorific leaves a Latin name word the script cannot order": + ("Anderson Smithさん", "Dr. Andersonさん", "Anderson, さん"), + "feat(#449) a glued Korean honorific leaves a Latin name word the script cannot order": + ("Anderson Smith선생님", "Dr. Anderson선생님", "Anderson, 선생님"), } @@ -1564,6 +1586,27 @@ class _LatinCopy(NamedTuple): # regression. One set, identical in all four ledgers. frozenset({"Ahmad Jayadi, CHA", "Aishwarya Rai", "John Smith RAI", "John Smith, RAI", "Lala Lajpat Rai"}), + # #449's movers, one corpus name per alternative -- a list of + # names, not a copy of any wordlist, so there is no vocabulary for + # it to drift from. What selects these names is a SHAPE the + # vocabulary only participates in negatively (a lone name word NO + # vocabulary claimed), and a member copying any wordlist would + # reach names that report nothing: 'abdul' is bound-given + # vocabulary and 'de' a particle, and neither moves. Seven members + # carry a trailing suffix -- 'Smith Jr.', "'Smitty' Jones Jr.", + # 'John V', 'Jack M.A.', 'Carod i', 'Donald mc', 'Mohamad X' -- + # where the peel took the suffix and the convention placed the ONE + # name word left, so a suffix wordlist is not what this list copies + # either. One set, identical in all three 2.x ledgers. + frozenset({r"'Smitty' Jones Jr\.", "Andrew", "Carod i", + "Dean of Chemistry", "Donald mc", "Duke of Edinburgh", + "Duke of Wellington", "Garcia", r"Jack M\.A\.", + "John & Jane", "John V", "John of the Doe", + "Juan & Garcia", "Juan and Garcia", "Mohamad X", + "Smith", r"Smith Jr\.", "e and e", + "part1 of The part2 of the part3 and part4", + "part1 of and The part2 of the part3 And part4", + "test", "سلمان،"}), # fix(#445)'s movers, one corpus name per alternative -- a list of # names, not a copy of any wordlist, so there is no vocabulary for # it to drift from. Two sets because the ledgers group the nine @@ -2272,6 +2315,36 @@ def _claim(rule: dict) -> _Claim: # change the row here before it reached the gate. "fix(#436/#437) a space-separated post-nominal run renders with spaces, not commas": _Claim(10, ('suffix',), "30f5314a2662", None), + # #449's six rules, second in every 2.x ledger. The + # alternation reaches twenty-two corpus names and + # `_ambiguities` alone: no role moves anywhere in this change, + # so a widening that took a role would change the row here + # before it reached the gate, and a member reaching a name + # something DECIDED ('abdul', 'de', 'Dr. Smith') would move + # the count. The five CJK rules are literal-anchored on one + # corpus name each, a reach _CORPUS_CLAIMS cannot police on + # its own -- a widening into names the corpora lack leaves it + # unmoved -- so _MUST_NOT_MATCH carries the boundary probes + # beside them. The digests are the same in all three 2.x + # ledgers because the regexes are the same strings and the + # corpora are one set; the ROLES are not, and the two glued + # honorific rules are why -- only at this baseline is #308's + # peel still in the diff beside the report, and the gate + # refuses a rule declaring a role no diff it explains moves. + "feat(#449) a lone name word reports given-or-family": + _Claim(22, ('_ambiguities',), "0ef8cf9a9272", None), + "feat(#449) a wholly-katakana name keeps the declared order, so the convention decides it": + _Claim(1, ('_ambiguities',), "80777383a11a", None), + "feat(#449) an interpunct transcription declines the script order, so the convention decides it": + _Claim(1, ('_ambiguities',), "ba9a4258c864", None), + "feat(#449) an ideographic comma leaves one name word, and the convention decides it": + _Claim(1, ('_ambiguities',), "4b2858642238", None), + "feat(#449) a glued Japanese honorific leaves a Latin name word the script cannot order": + _Claim(1, ('_ambiguities', 'given', 'suffix'), "528858346d14", + None), + "feat(#449) a glued Korean honorific leaves a Latin name word the script cannot order": + _Claim(1, ('_ambiguities', 'given', 'suffix'), "30ec95e25f05", + None), # #346's alternation. Four corpus names, `family` and # `given` together: the fold moves both roles at once, so a # widening taking one alone would change the roles here @@ -2453,6 +2526,31 @@ def _claim(rule: dict) -> _Claim: # change the row here before it reached the gate. "fix(#436/#437) a space-separated post-nominal run renders with spaces, not commas": _Claim(10, ('suffix',), "30f5314a2662", None), + # #449's six rules, second in every 2.x ledger. The + # alternation reaches twenty-two corpus names and + # `_ambiguities` alone: no role moves anywhere in this change, + # so a widening that took a role would change the row here + # before it reached the gate, and a member reaching a name + # something DECIDED ('abdul', 'de', 'Dr. Smith') would move + # the count. The five CJK rules are literal-anchored on one + # corpus name each, a reach _CORPUS_CLAIMS cannot police on + # its own -- a widening into names the corpora lack leaves it + # unmoved -- so _MUST_NOT_MATCH carries the boundary probes + # beside them. The digests are the same in all three 2.x + # ledgers because the regexes are the same strings and the + # corpora are one set. + "feat(#449) a lone name word reports given-or-family": + _Claim(22, ('_ambiguities',), "0ef8cf9a9272", None), + "feat(#449) a wholly-katakana name keeps the declared order, so the convention decides it": + _Claim(1, ('_ambiguities',), "80777383a11a", None), + "feat(#449) an interpunct transcription declines the script order, so the convention decides it": + _Claim(1, ('_ambiguities',), "ba9a4258c864", None), + "feat(#449) an ideographic comma leaves one name word, and the convention decides it": + _Claim(1, ('_ambiguities',), "4b2858642238", None), + "feat(#449) a glued Japanese honorific leaves a Latin name word the script cannot order": + _Claim(1, ('_ambiguities',), "528858346d14", None), + "feat(#449) a glued Korean honorific leaves a Latin name word the script cannot order": + _Claim(1, ('_ambiguities',), "30ec95e25f05", None), # #346's alternation. Four corpus names, `family` and # `given` together: the fold moves both roles at once, so a # widening taking one alone would change the roles here @@ -2488,6 +2586,31 @@ def _claim(rule: dict) -> _Claim: # change the row here before it reached the gate. "fix(#436/#437) a space-separated post-nominal run renders with spaces, not commas": _Claim(10, ('suffix',), "30f5314a2662", None), + # #449's six rules, second in every 2.x ledger. The + # alternation reaches twenty-two corpus names and + # `_ambiguities` alone: no role moves anywhere in this change, + # so a widening that took a role would change the row here + # before it reached the gate, and a member reaching a name + # something DECIDED ('abdul', 'de', 'Dr. Smith') would move + # the count. The five CJK rules are literal-anchored on one + # corpus name each, a reach _CORPUS_CLAIMS cannot police on + # its own -- a widening into names the corpora lack leaves it + # unmoved -- so _MUST_NOT_MATCH carries the boundary probes + # beside them. The digests are the same in all three 2.x + # ledgers because the regexes are the same strings and the + # corpora are one set. + "feat(#449) a lone name word reports given-or-family": + _Claim(22, ('_ambiguities',), "0ef8cf9a9272", None), + "feat(#449) a wholly-katakana name keeps the declared order, so the convention decides it": + _Claim(1, ('_ambiguities',), "80777383a11a", None), + "feat(#449) an interpunct transcription declines the script order, so the convention decides it": + _Claim(1, ('_ambiguities',), "ba9a4258c864", None), + "feat(#449) an ideographic comma leaves one name word, and the convention decides it": + _Claim(1, ('_ambiguities',), "4b2858642238", None), + "feat(#449) a glued Japanese honorific leaves a Latin name word the script cannot order": + _Claim(1, ('_ambiguities',), "528858346d14", None), + "feat(#449) a glued Korean honorific leaves a Latin name word the script cannot order": + _Claim(1, ('_ambiguities',), "30ec95e25f05", None), # #346's alternation. Four corpus names, `family` and # `given` together: the fold moves both roles at once, so a # widening taking one alone would change the roles here diff --git a/tools/differential/corpus_rules.jsonl b/tools/differential/corpus_rules.jsonl index f2eef6c3..692bc03d 100644 --- a/tools/differential/corpus_rules.jsonl +++ b/tools/differential/corpus_rules.jsonl @@ -11,6 +11,7 @@ "Ali Ahmad Vali oglu" "Ali Ahmad Vali oglu Jr." "Ali Ahmad oglu" +"Andrew" "Andrew (Andy) Perkins" "Andrew Perkins (Andy)" "Andrew Perkins (MBA)" @@ -32,6 +33,7 @@ "Del Toro" "Dr. John van der Berg" "Dr. Juan Q. Xavier de la Vega III" +"Dr. Smith" "Dr. Smith née Jones" "Dr. Smith, John" "Dr. abdul salam" diff --git a/tools/differential/expected_since_2.0.0.toml b/tools/differential/expected_since_2.0.0.toml index 9cba00dc..e3272fd1 100644 --- a/tools/differential/expected_since_2.0.0.toml +++ b/tools/differential/expected_since_2.0.0.toml @@ -57,6 +57,88 @@ issue = "fix(#436/#437) a space-separated post-nominal run renders with spaces, name_regex = "^(?:JOHN DOE PHD MD|John Doe MD PhD|John Smith MD PhD|John Smith Mc V|Kenneth Clarke QC MP|Smith, John PhD I\\.|The Rt Hon Kenneth Clarke QC MP, HMG|Washington Jr\\. MD, Franklin|abdul Smith Jr Ma|abdul Smith Jr V)$" fields = ["suffix"] +# The six #449 rules go SECOND, not first: the rule above +# declares its own first-in-file position with a measurement, +# and `fields = ["suffix"]` and `fields = ["_ambiguities"]` +# are disjoint, so nothing orders the two against each other. +# They DO sit ahead of every rule below whose `fields` is a +# strict superset of `_ambiguities`, which is the narrow-first +# default (#382). +[[change]] +issue = "feat(#449) a lone name word reports given-or-family" +# rules.md#O5's convention, now reported. Twenty-two corpus names, and +# the alternation is a list of NAMES rather than a copy of any +# wordlist -- what selects them is the SHAPE (one name word, and no +# title, maiden name, comma, script order or vocabulary claim deciding +# it), so it is declared in _NOT_A_VOCABULARY_COPY. A suffix beside +# the word does NOT decide it: 'Smith Jr.' and "'Smitty' Jones Jr." +# are the peel taking the suffix and the convention placing what is +# left. `_ambiguities` alone: no role moves anywhere in this change. +# The five CJK-bearing names in the same population ride five rules of +# their own below, because a CJK member in an alternation is claimed +# by the honorific pin, which would demand this be a copy of +# GLUED_HONORIFICS. +name_regex = "(?i)^(?:'Smitty' Jones Jr\\.|Andrew|Carod i|Dean of Chemistry|Donald mc|Duke of Edinburgh|Duke of Wellington|Garcia|Jack M\\.A\\.|John & Jane|John V|John of the Doe|Juan & Garcia|Juan and Garcia|Mohamad X|Smith|Smith Jr\\.|e and e|part1 of The part2 of the part3 and part4|part1 of and The part2 of the part3 And part4|test|سلمان،)$" +fields = ["_ambiguities"] + +[[change]] +issue = "feat(#449) a wholly-katakana name keeps the declared order, so the convention decides it" +# 'マイケル'. rules.md#W4 gives katakana no order of its own -- a +# wholly-katakana name is usually a transcription and keeps whatever +# the caller declared -- so the script rule DECLINES and O5's +# convention is what is left. Literal-anchored on one corpus name, so +# _CORPUS_CLAIMS cannot see a widening and _MUST_NOT_MATCH carries the +# wall. +name_regex = "^マイケル$" +fields = ["_ambiguities"] + +[[change]] +issue = "feat(#449) an interpunct transcription declines the script order, so the convention decides it" +# '王·Smith'. rules.md#T3: the 間隔号 marks a transcription and +# suppresses the script_orders lookup whole, so name_order governs and +# the one remaining name word is O5's. Literal-anchored, one corpus +# name. +name_regex = "^王·Smith$" +fields = ["_ambiguities"] + +[[change]] +issue = "feat(#449) an ideographic comma leaves one name word, and the convention decides it" +# '田中、太郎'. The comma is inside the token, so the text resolves to +# no single script and the order lookup declines; what is left is one +# name word and O5. A rules.md#W3 shape -- tolerated, read +# best-effort -- and the report says only what the parse actually +# called. Literal-anchored, one corpus name. +name_regex = "^田中、太郎$" +fields = ["_ambiguities"] + +[[change]] +issue = "feat(#449) a glued Japanese honorific leaves a Latin name word the script cannot order" +# 'Andersonさん'. rules.md#W2 peels the glued honorific to SUFFIX and +# leaves 'Anderson' -- Latin, so effective_script answers None and no +# script order is resolved at all. One name word, and O5's convention +# places it. Literal-anchored, one corpus name. +# +# `given` and `suffix` ride beside `_ambiguities` here and in this +# ledger ALONE: at 2.0.0 the name reads given 'Andersonさん' with an +# empty suffix, so #308's peel and #449's report arrive in one diff +# and one rule has to admit both. By 2.1.0 the peel is already in the +# baseline and `_ambiguities` is the whole diff, which is why the +# other two ledgers spell these two rules with the narrower `fields` +# -- the gate's OVER-DECLARED check refuses a rule declaring a role no +# diff it explains moves, so the three copies cannot be identical. +name_regex = "^Andersonさん$" +fields = ["_ambiguities", "given", "suffix"] + +[[change]] +issue = "feat(#449) a glued Korean honorific leaves a Latin name word the script cannot order" +# 'Anderson선생님'. The Japanese rule's argument, in Hangul: the +# honorific is peeled, the Latin stem resolves to no script, and the +# convention places the one name word left. Literal-anchored, one +# corpus name, and `fields` carries the peel for the reason given +# above. +name_regex = "^Anderson선생님$" +fields = ["_ambiguities", "given", "suffix"] + # #346: swami, guru, baba and lama moved from the TITLES-only block # into GIVEN_NAME_TITLES on 2026-09-06. rules.md#H1's Accepted clause # -- "a given-name title plus one name word leaves the family empty" @@ -174,7 +256,9 @@ fields = ["given", "middle", "family", "_ambiguities"] [[change]] issue = "fix(#308/#312/#319/#320) glued CJK honorific peeled off the name into suffix" -# '田中さん', '김민준씨', '王小明先生', 'Andersonさん': #308 splits an +# '田中さん', '김민준씨', '王小明先生' (and 'Andersonさん', which the +# feat(#449) glued-Japanese rule above claims first since 2026-09-08): +# #308 splits an # honorific written against the name off the end of its token and # routes it to `suffix`, where 2.0 left it inside the name. #312 lets # the peel reach across a comma ('김, 민준씨', '田中, 太郎さん', diff --git a/tools/differential/expected_since_2.1.0.toml b/tools/differential/expected_since_2.1.0.toml index 68e20774..d6e1cef2 100644 --- a/tools/differential/expected_since_2.1.0.toml +++ b/tools/differential/expected_since_2.1.0.toml @@ -81,6 +81,84 @@ issue = "fix(#436/#437) a space-separated post-nominal run renders with spaces, name_regex = "^(?:JOHN DOE PHD MD|John Doe MD PhD|John Smith MD PhD|John Smith Mc V|Kenneth Clarke QC MP|Smith, John PhD I\\.|The Rt Hon Kenneth Clarke QC MP, HMG|Washington Jr\\. MD, Franklin|abdul Smith Jr Ma|abdul Smith Jr V)$" fields = ["suffix"] +# The six #449 rules go SECOND, not first: the rule above +# declares its own first-in-file position with a measurement, +# and `fields = ["suffix"]` and `fields = ["_ambiguities"]` +# are disjoint, so nothing orders the two against each other. +# They DO sit ahead of every rule below whose `fields` is a +# strict superset of `_ambiguities`, which is the narrow-first +# default (#382). +[[change]] +issue = "feat(#449) a lone name word reports given-or-family" +# rules.md#O5's convention, now reported. Twenty-two corpus names, and +# the alternation is a list of NAMES rather than a copy of any +# wordlist -- what selects them is the SHAPE (one name word, and no +# title, maiden name, comma, script order or vocabulary claim deciding +# it), so it is declared in _NOT_A_VOCABULARY_COPY. A suffix beside +# the word does NOT decide it: 'Smith Jr.' and "'Smitty' Jones Jr." +# are the peel taking the suffix and the convention placing what is +# left. `_ambiguities` alone: no role moves anywhere in this change. +# The five CJK-bearing names in the same population ride five rules of +# their own below, because a CJK member in an alternation is claimed +# by the honorific pin, which would demand this be a copy of +# GLUED_HONORIFICS. +name_regex = "(?i)^(?:'Smitty' Jones Jr\\.|Andrew|Carod i|Dean of Chemistry|Donald mc|Duke of Edinburgh|Duke of Wellington|Garcia|Jack M\\.A\\.|John & Jane|John V|John of the Doe|Juan & Garcia|Juan and Garcia|Mohamad X|Smith|Smith Jr\\.|e and e|part1 of The part2 of the part3 and part4|part1 of and The part2 of the part3 And part4|test|سلمان،)$" +fields = ["_ambiguities"] + +[[change]] +issue = "feat(#449) a wholly-katakana name keeps the declared order, so the convention decides it" +# 'マイケル'. rules.md#W4 gives katakana no order of its own -- a +# wholly-katakana name is usually a transcription and keeps whatever +# the caller declared -- so the script rule DECLINES and O5's +# convention is what is left. Literal-anchored on one corpus name, so +# _CORPUS_CLAIMS cannot see a widening and _MUST_NOT_MATCH carries the +# wall. +name_regex = "^マイケル$" +fields = ["_ambiguities"] + +[[change]] +issue = "feat(#449) an interpunct transcription declines the script order, so the convention decides it" +# '王·Smith'. rules.md#T3: the 間隔号 marks a transcription and +# suppresses the script_orders lookup whole, so name_order governs and +# the one remaining name word is O5's. Literal-anchored, one corpus +# name. +name_regex = "^王·Smith$" +fields = ["_ambiguities"] + +[[change]] +issue = "feat(#449) an ideographic comma leaves one name word, and the convention decides it" +# '田中、太郎'. The comma is inside the token, so the text resolves to +# no single script and the order lookup declines; what is left is one +# name word and O5. A rules.md#W3 shape -- tolerated, read +# best-effort -- and the report says only what the parse actually +# called. Literal-anchored, one corpus name. +name_regex = "^田中、太郎$" +fields = ["_ambiguities"] + +[[change]] +issue = "feat(#449) a glued Japanese honorific leaves a Latin name word the script cannot order" +# 'Andersonさん'. rules.md#W2 peels the glued honorific to SUFFIX and +# leaves 'Anderson' -- Latin, so effective_script answers None and no +# script order is resolved at all. One name word, and O5's convention +# places it. Literal-anchored, one corpus name. +# +# `_ambiguities` alone here, where the 2.0.0 ledger's twin also carries +# `given` and `suffix`: #308's peel is already in THIS baseline, so +# the report is the whole diff. The gate's OVER-DECLARED check is what +# forces the two spellings apart -- a rule may not declare a role no +# diff it explains moves. +name_regex = "^Andersonさん$" +fields = ["_ambiguities"] + +[[change]] +issue = "feat(#449) a glued Korean honorific leaves a Latin name word the script cannot order" +# 'Anderson선생님'. The Japanese rule's argument, in Hangul: the +# honorific is peeled, the Latin stem resolves to no script, and the +# convention places the one name word left. Literal-anchored, one +# corpus name, and `_ambiguities` alone for the reason given above. +name_regex = "^Anderson선생님$" +fields = ["_ambiguities"] + # The four CJK names of #436/#437's class, one rule each. Not one # alternation: an alternation holding a script-classified member is # claimed by the honorific pin in tests/v2/test_ledger_guards.py, diff --git a/tools/differential/expected_since_2.2.0.toml b/tools/differential/expected_since_2.2.0.toml index fe0f9a87..5e1dbfe6 100644 --- a/tools/differential/expected_since_2.2.0.toml +++ b/tools/differential/expected_since_2.2.0.toml @@ -70,6 +70,84 @@ issue = "fix(#436/#437) a space-separated post-nominal run renders with spaces, name_regex = "^(?:JOHN DOE PHD MD|John Doe MD PhD|John Smith MD PhD|John Smith Mc V|Kenneth Clarke QC MP|Smith, John PhD I\\.|The Rt Hon Kenneth Clarke QC MP, HMG|Washington Jr\\. MD, Franklin|abdul Smith Jr Ma|abdul Smith Jr V)$" fields = ["suffix"] +# The six #449 rules go SECOND, not first: the rule above +# declares its own first-in-file position with a measurement, +# and `fields = ["suffix"]` and `fields = ["_ambiguities"]` +# are disjoint, so nothing orders the two against each other. +# They DO sit ahead of every rule below whose `fields` is a +# strict superset of `_ambiguities`, which is the narrow-first +# default (#382). +[[change]] +issue = "feat(#449) a lone name word reports given-or-family" +# rules.md#O5's convention, now reported. Twenty-two corpus names, and +# the alternation is a list of NAMES rather than a copy of any +# wordlist -- what selects them is the SHAPE (one name word, and no +# title, maiden name, comma, script order or vocabulary claim deciding +# it), so it is declared in _NOT_A_VOCABULARY_COPY. A suffix beside +# the word does NOT decide it: 'Smith Jr.' and "'Smitty' Jones Jr." +# are the peel taking the suffix and the convention placing what is +# left. `_ambiguities` alone: no role moves anywhere in this change. +# The five CJK-bearing names in the same population ride five rules of +# their own below, because a CJK member in an alternation is claimed +# by the honorific pin, which would demand this be a copy of +# GLUED_HONORIFICS. +name_regex = "(?i)^(?:'Smitty' Jones Jr\\.|Andrew|Carod i|Dean of Chemistry|Donald mc|Duke of Edinburgh|Duke of Wellington|Garcia|Jack M\\.A\\.|John & Jane|John V|John of the Doe|Juan & Garcia|Juan and Garcia|Mohamad X|Smith|Smith Jr\\.|e and e|part1 of The part2 of the part3 and part4|part1 of and The part2 of the part3 And part4|test|سلمان،)$" +fields = ["_ambiguities"] + +[[change]] +issue = "feat(#449) a wholly-katakana name keeps the declared order, so the convention decides it" +# 'マイケル'. rules.md#W4 gives katakana no order of its own -- a +# wholly-katakana name is usually a transcription and keeps whatever +# the caller declared -- so the script rule DECLINES and O5's +# convention is what is left. Literal-anchored on one corpus name, so +# _CORPUS_CLAIMS cannot see a widening and _MUST_NOT_MATCH carries the +# wall. +name_regex = "^マイケル$" +fields = ["_ambiguities"] + +[[change]] +issue = "feat(#449) an interpunct transcription declines the script order, so the convention decides it" +# '王·Smith'. rules.md#T3: the 間隔号 marks a transcription and +# suppresses the script_orders lookup whole, so name_order governs and +# the one remaining name word is O5's. Literal-anchored, one corpus +# name. +name_regex = "^王·Smith$" +fields = ["_ambiguities"] + +[[change]] +issue = "feat(#449) an ideographic comma leaves one name word, and the convention decides it" +# '田中、太郎'. The comma is inside the token, so the text resolves to +# no single script and the order lookup declines; what is left is one +# name word and O5. A rules.md#W3 shape -- tolerated, read +# best-effort -- and the report says only what the parse actually +# called. Literal-anchored, one corpus name. +name_regex = "^田中、太郎$" +fields = ["_ambiguities"] + +[[change]] +issue = "feat(#449) a glued Japanese honorific leaves a Latin name word the script cannot order" +# 'Andersonさん'. rules.md#W2 peels the glued honorific to SUFFIX and +# leaves 'Anderson' -- Latin, so effective_script answers None and no +# script order is resolved at all. One name word, and O5's convention +# places it. Literal-anchored, one corpus name. +# +# `_ambiguities` alone here, where the 2.0.0 ledger's twin also carries +# `given` and `suffix`: #308's peel is already in THIS baseline, so +# the report is the whole diff. The gate's OVER-DECLARED check is what +# forces the two spellings apart -- a rule may not declare a role no +# diff it explains moves. +name_regex = "^Andersonさん$" +fields = ["_ambiguities"] + +[[change]] +issue = "feat(#449) a glued Korean honorific leaves a Latin name word the script cannot order" +# 'Anderson선생님'. The Japanese rule's argument, in Hangul: the +# honorific is peeled, the Latin stem resolves to no script, and the +# convention places the one name word left. Literal-anchored, one +# corpus name, and `_ambiguities` alone for the reason given above. +name_regex = "^Anderson선생님$" +fields = ["_ambiguities"] + # The four CJK names of #436/#437's class, one rule each. Not one # alternation: an alternation holding a script-classified member is # claimed by the honorific pin in tests/v2/test_ledger_guards.py, From 5af3d981f7ef334375f80a2158fda167e411bf70 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Tue, 8 Sep 2026 00:42:25 -0700 Subject: [PATCH 2/5] feat(types): AmbiguityKind.TITLE_OR_NAME, reported where the title peel leaves one title word as the name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Handed a string the title peel eats down to one last word that is itself title vocabulary, the parser reads that word as the name -- 'Lord Chancellor' gives family 'Chancellor', and the Queen's Bench string family 'Division'. The reading stands: this is a name parser, not a title parser, and a title with no name at all is worse output. What was wrong is that the guess was silent, which is what #491 asked for. ONE site reports it: the assignment that places the lone name word, which is where that reading is actually chosen. Reporting at H1's retag instead was measured wrong -- H1 only moves the word between fields, and under `Policy(name_order=FAMILY_FIRST)` (or FAMILY_FIRST_GIVEN_LAST) assign places it in the family directly and the retag never runs, so an all-titles input was silent under a declared family-first order while giving byte-identical roles. At the assignment the report is order-independent, and all three orders are now measured to report 'Lord Chancellor', 'Dr. King', 'His Holiness the Dalai Lama', the Queen's Bench string, 'The Rt Hon', 'Mr. Mrs.' and 'Xyz. King', and to stay silent on 'Dr. Smith', 'Dr. Prof. Smith', 'Sir', 'Dr.', 'King', 'King Charles', 'Coach' and 'Prince of Wales'. The detail names no FIELD for the same reason: H1 would leave one named at emit time stale, and the fork the kind reports is title-versus-name, which no field answers. Six corpus names carry it, three contract-tier and three radar, and 'Dr. King' is one of them: `king` is title vocabulary for the addressing forms, so the peel leaves one title word and the rule claims it. A LONE title word reports nothing -- the peel takes 'Dr.' and 'Prince of Wales' whole and leaves no word to be read as a name -- and so do 'Dr. King MD' and 'Dr King Jr', where the peel also leaves nothing and the post-nominal becomes the name through the bare-suffix carve-out, which is scoped to a run no title preceded. The title-vs-given-name collision on a word like 'Baron' is a question about the vocabulary, left to #348. One case row moves off the corpus: 'Dr. King, Jr.' reports now, a suffix beside the word not being a shape that silences the convention, and its row records that. The new rules.md#H4 states both halves. The suffix half is the existing SUFFIX_OR_NAME at assign's bare-suffix carve-out: 'Rinpoche' and 'QC MP' read a post-nominal as the name because nothing else was left to be one. 'Jr.' alone is not one of them -- H2's opening-abbreviation shape reads it as a title before the vocabulary is asked -- and the emitter is scoped away from a lone CJK honorific ('さん', '씨', '선생님'), the same shape read through the glued-honorific rules and the script's own order, whose report is left to the arc that revisits those readings rather than settled here. A maiden name beside the credential says the input is not post-nominal vocabulary and nothing else, so 'abd née Jones' is out for M4's reason. The same site's O5 branch gains the third emitter, promised with reports title-or-name rather than given-or-family, the fork there being whether the title word inside the unit is a title at all. 'John of Prince' and 'Smith and Prince' are the measured inputs that reach it -- `prince` is in TITLES -- and no corpus name does, a join led by a title word being a title run ('Prince of Wales', silent like a lone 'Dr.'). The test is the `vocab:title` TAG rather than a lexicon lookup, which is what the module header promises: assign reads the lexicon only through what classify tagged. No parse changes: every role of every corpus name, under every order the corpora declare, is identical to the parent tree's (1123 names, measured), and the eight names that move do so on `_ambiguities` alone at all three 2.x baselines. The gate is 368 / 296 / 210 / 72 intentional diffs at 1.4.0 / 2.0.0 / 2.1.0 / 2.2.0, unexplained 0 at each. Co-Authored-By: Claude Fable 5.1 --- docs/design/rules.md | 63 ++++++++- nameparser/_pipeline/_assign.py | 136 +++++++++++++++---- nameparser/_types.py | 22 +++ tests/v2/cases.py | 85 +++++++++++- tests/v2/test_contracts.py | 1 + tests/v2/test_facade_cases.py | 5 + tests/v2/test_ledger_guards.py | 65 +++++++++ tools/differential/corpus_rules.jsonl | 7 + tools/differential/expected_since_2.0.0.toml | 29 ++++ tools/differential/expected_since_2.1.0.toml | 29 ++++ tools/differential/expected_since_2.2.0.toml | 29 ++++ 11 files changed, 438 insertions(+), 33 deletions(-) diff --git a/docs/design/rules.md b/docs/design/rules.md index cd82e80c..5f328da4 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -29,7 +29,7 @@ The marker's unit is the whole rule, because that is the unit `tools/differentia ## Titles & honorifics (H) -Background: an honorific title precedes a name and is not itself part of it; it addresses or ranks the person. Most titles address by surname ("Mr. Johnson"), but a few — knighthoods, some clerical and courtesy titles — address by given name ("Sir John"). The library keeps a vocabulary of titles and, separately, of these given-name titles. What a TRAILING title-vocabulary word should do is unresolved (#316): today "John Smith Prof." keeps Prof. a name word while "Smith, Prof." reads it as a title — the two comma paths disagree, and TITLES holding ordinary surnames (king, judge, bishop) is what bars the blanket vocabulary-wins answer. Two criteria govern two different questions here. Membership in the given-name-title list follows HOW THE TITLE ADDRESSES: a title that precedes and addresses by the given name belongs (Sir, Sheikh, the Arabic honorifics الدكتور/الشيخ — which qualify even though those traditions fully retain family names). Whether an EMPTY FAMILY is correct output is the separate question, governed by surname retention: renunciation abolishes the surname, so for Swami, Guru, Baba or Lama family="" is right (#346), while rabbi and imam traditions keep surnames — "Rabbi Cohen" addresses by title and keeps family "Cohen". Conflating the two criteria either ejects the Arabic entries or sweeps in titles that break +Background: an honorific title precedes a name and is not itself part of it; it addresses or ranks the person. Most titles address by surname ("Mr. Johnson"), but a few — knighthoods, some clerical and courtesy titles — address by given name ("Sir John"). The library keeps a vocabulary of titles and, separately, of these given-name titles. What a TRAILING title-vocabulary word should do is unresolved (#316): today "John Smith Prof." keeps Prof. a name word while "Smith, Prof." reads it as a title — the two comma paths disagree, and TITLES holding ordinary surnames (king, judge, bishop) is what bars the blanket vocabulary-wins answer. An input the title peel eats down to one last title-vocabulary word is H4's, and what it does with that word is a convention rather than a reading of the vocabulary. Two criteria govern two different questions here. Membership in the given-name-title list follows HOW THE TITLE ADDRESSES: a title that precedes and addresses by the given name belongs (Sir, Sheikh, the Arabic honorifics الدكتور/الشيخ — which qualify even though those traditions fully retain family names). Whether an EMPTY FAMILY is correct output is the separate question, governed by surname retention: renunciation abolishes the surname, so for Swami, Guru, Baba or Lama family="" is right (#346), while rabbi and imam traditions keep surnames — "Rabbi Cohen" addresses by title and keeps family "Cohen". Conflating the two criteria either ejects the Arabic entries or sweeps in titles that break "Rabbi Cohen". H1. Rationale: a title normally addresses by surname, so a title @@ -103,6 +103,61 @@ H3. Rationale: compound titles are written as a run of title words, "Dr. Smith, John" → family="Dr. Smith" interacts: C1 · implemented: nameparser/_pipeline/_pieces.py +H4. Rationale: this is a name parser, not a title parser. Handed a + string the title peel eats down to one last word which is itself + title vocabulary, it still has to name somebody, and that word is + the only candidate there is — a guess fixed in advance, so the + same input reads the same way every time, not a claim that the + word is a surname. The same reasoning covers a string that is + nothing but post-nominal vocabulary: something has to be the name. + An input whose only remaining name word after the title peel is + itself title vocabulary reads that word as the name by convention + and reports `title-or-name`; an input whose every word is + post-nominal vocabulary reads its first word as a name and + reports `suffix-or-name`. A single title word reads as a title + with no name beside it and reports nothing: the peel took the + whole string, so no word was left standing to be read as a name + and no reading was chosen. + "Lord Chancellor" → family="Chancellor" + "Lord Chancellor" → ambiguities=("title-or-name",) + "The Right Hon. the President of the Queen's Bench Division" → family="Division" + "The Right Hon. the President of the Queen's Bench Division" → ambiguities=("title-or-name",) + "Dr. King" → ambiguities=("title-or-name",) + "Dr." → title="Dr." · boundary + "Dr." → ambiguities=() + "Dr. Smith" → ambiguities=() + Accepted: ONE site takes both halves — the assignment that places + the lone name word — so neither report depends on the declared + order. Under the default order H1 retags the title-vocabulary + word from given to family afterwards; under a declared + family-first order the assignment places it in the family + directly and H1 never runs. Same reading either way, so the same + kind is reported either way. + Accepted: the suffix half is reached only where no title was + peeled first, so a title in front of the run takes the input out + of this rule and leaves it H1's — "Dr. King MD" reports nothing, + the credential having been read as the name after a title peel + rather than for want of one. + "Rinpoche" → ambiguities=("suffix-or-name",) + "QC MP" → given="QC" + "Jr." → title="Jr." + Accepted: `Jr.` is post-nominal vocabulary and still reads as a + title, H2's opening-abbreviation shape outranking the vocabulary + where it fires — so the suffix half never sees a dotted lone + credential. + Accepted: the suffix half is scoped to names no script order + placed, so a lone CJK honorific written by itself — さん, 씨, + 선생님 — reports nothing. The same shape, read through the + glued-honorific rules (W2) and the script's own order, and left + to the arc that revisits those readings. + Accepted: one name word that is a join (P3) carrying title + vocabulary reports `title-or-name` as well, the fork there being + whether the title word inside the unit is a title at all. The + measured inputs, `John of Prince` and `Smith and Prince`, are + pinned in the case table rather than here: a join led by a title + word is a title run (H3), so no corpus name reaches the branch. + interacts: H1, H2, H3, S2, O5 · implemented: nameparser/_pipeline/_assign.py + ## Particles & surname prefixes (P) Background: particles ("de", "la", "van", "von", "bin") link forward to a surname and are written as part of it. Some are never anyone's given name; others ("Van", "Bin") are ordinary given names in some cultures, so the vocabulary distinguishes never-given particles from ambiguous ones, and only the never-given ones license special treatment. A separate small vocabulary binds forward to a GIVEN name instead: words like "abdul" that are not complete given names alone (P5). Which particles fall on which side of the never-given line is its own open question (#360). @@ -1076,7 +1131,11 @@ O5. Rationale: O4 reads a name by comparing where its words stand, convention settles the order (W4) silences it, and so does the word's own claim — a particle, a bound given name, an initial's shape. A word with no letter or digit in it is no name word and - reports nothing (A2). + reports nothing (A2). Where the one name word is a join (P3) + carrying title vocabulary, or is itself title vocabulary left + standing by the title peel, the doubt reported is `title-or-name` + instead, which H4 states — so a title silences THIS kind, not + every report at the site. "Smith" → given="Smith" "Garcia" family-first → family="Garcia" "Sir John" → given="John" diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index 1cf92a84..2dbd3e6d 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -9,8 +9,8 @@ across pieces); token/piece tags; Lexicon only through tags already applied by classify (plus the leading-title period rule). -Implements rules H2, N3, O4 and W4 of docs/design/rules.md, each -cited at its code below. Ports v1's assignment loops. +Implements rules H2, H4, N3, O4, O5 and W4 of docs/design/rules.md, +each cited at its code below. Ports v1's assignment loops. NO_COMMA (per name_order): leading title pieces chain while no given-position name has been seen (a title needs a following piece, unless the whole name is one title); @@ -253,8 +253,10 @@ def _assign_main(seg_idx: int, state: ParseState, f"letter there would be a middle initial", peeled.numeral)) name_pieces, suffix_pieces = rest[:peeled.names], rest[peeled.names:] + bare_suffix = False if not name_pieces and suffix_pieces: # everything suffix-shaped after titles: first one is the name + bare_suffix = True name_pieces, suffix_pieces = suffix_pieces[:1], suffix_pieces[1:] # AFTER both peels, and load-bearing: the script test sees the NAME # pieces only, so a Latin title or suffix ('Dr. 毛 泽东', '毛 泽东, @@ -268,16 +270,48 @@ def _assign_main(seg_idx: int, state: ParseState, _set_roles(tokens, pieces[piece_idx], roles[pos]) for piece_idx in suffix_pieces: _set_roles(tokens, pieces[piece_idx], Role.SUFFIX) - # rules.md#O5's convention, reported at the site that applies it - # (mechanisms.md#AMBIGUITY-AT-THE-DECISION-SITE). The guard is - # O5's own carve-out list read as code -- a leading title, a - # maiden name, a group-flagged credential and the word's own claim - # each mean something else decided -- plus one the rule states - # without naming: a script whose order convention settles the - # reading (W4) decided it too, which is `resolved.by_script` - # rather than a comparison against name_order, a declared - # family-first order agreeing with a Han name's entry being - # agreement and not authorship. The two shapes O5 also names never + # rules.md#H4: "an input whose every word is post-nominal + # vocabulary reads its first word as a name and reports + # `suffix-or-name`" -- reported at the carve-out above that + # applies it (mechanisms.md#AMBIGUITY-AT-THE-DECISION-SITE), and + # only the word made into a name reports. `n == 0` is what + # keeps a titled run out -- 'Dr King Jr' and 'MD DDS' peel a + # title first, and after a title the reading is H1's rather than + # this convention's. A maiden name beside the credential says the + # input is not post-nominal vocabulary and nothing else, so 'abd + # née Jones' is out for the same reason M4 keeps it out of O5's + # report below. `resolved.by_script` is a SCOPE line rather than a + # claim that something else decided: a lone CJK honorific ('さん', + # '씨', '선생님') is the same shape read through the glued-honorific + # rules and the script's own order (W2, #271/#308), and whether + # those readings should report is left to the arc that revisits + # them rather than settled here. The role comes off the token for + # the reason stated at the particle emitter below. + if (bare_suffix and n == 0 and not resolved.by_script + and not any(t.role is Role.MAIDEN for t in tokens)): + head = pieces[name_pieces[0]] + token = tokens[head[0]] + assert token.role is not None + text = " ".join(tokens[i].text for i in head) + ambiguities.append(PendingAmbiguity( + AmbiguityKind.SUFFIX_OR_NAME, + f"{text!r} is post-nominal vocabulary with no name word " + f"beside it; read as a {token.role.value} name rather than " + f"a post-nominal, nothing else being left to be the name", + tuple(head))) + # The site that places a lone name word, and so the site that + # reports both conventions which turn on one + # (mechanisms.md#AMBIGUITY-AT-THE-DECISION-SITE): H4's below, + # then O5's. The outer guard holds only what silences BOTH -- a + # maiden name (M4 decided it) and a script whose order convention + # settles the reading (W4 decided it too, which the rule states + # without naming), the latter `resolved.by_script` rather than a + # comparison against name_order, a declared family-first order + # agreeing with a Han name's entry being agreement and not + # authorship. The rest of O5's carve-out list -- a leading title, + # a group-flagged credential, the word's own claim -- sits on + # O5's own branch, because a leading title does not silence H4: + # it is the shape H4 is about. The two shapes O5 also names never # reach this line: a nickname beside a lone name word is N3's and # returns above, and a family comma names the family before # segment 0 is read positionally, so neither needs a clause here. @@ -286,30 +320,78 @@ def _assign_main(seg_idx: int, state: ParseState, # which is why S2's peel does not silence it. The count comes off # the PEEL rather than off name_pieces, which is what leaves the # bare-suffix carve-out above (peeled.names == 0, the first - # post-nominal read as the name for want of anything else) out: - # that reading is a different convention, and reporting it is - # H4's. The role comes off the token for the reason stated at the - # particle emitter below. - if (peeled.names == 1 and n == 0 - and not resolved.by_script + # post-nominal read as the name for want of anything else) out of + # both branches: that reading is a different convention, and + # reporting it is H4's suffix half above. The role comes off the + # token for the reason stated at the particle emitter below. + if (peeled.names == 1 and not resolved.by_script and not any(t.role is Role.MAIDEN for t in tokens)): head = pieces[name_pieces[0]] text = " ".join(tokens[i].text for i in head) - if (not any(_WORD_ALREADY_CLAIMED & tokens[i].tags for i in head) + token = tokens[head[0]] + assert token.role is not None + # rules.md#H4: "an input whose only remaining name word after + # the title peel is itself title vocabulary reads that word as + # the name by convention and reports `title-or-name`" -- the + # word this line places is the whole of what the parse got to + # call a name, and calling it one is a convention: a name + # parser, not a title parser. ONE site, ahead of the `n == 0` + # clause below rather than inside it, which is what makes the + # report order-independent: under the default order H1 retags + # this word from given to family AFTERWARDS, and under a + # declared family-first order it is placed in the family here + # and H1 never runs -- the same reading, so the same report. + # A LONE title word never reaches this line at all ('Dr.', + # 'Prince of Wales'): the leading-title peel takes the whole + # name and `rest` is empty. The peeled titles are not tested: + # H2 makes an unlisted abbreviation a title by SHAPE, and + # 'Xyz. Smith' is not this input -- what the rule turns on is + # the word left standing as the name. The detail names no + # FIELD, unlike O5's below: under the default order H1 retags + # this word after assign, so a field named here would be the + # one it was placed in and not the one it ends in -- and the + # fork the kind reports is title-versus-name, which no field + # answers either way. + if len(head) == 1 and "vocab:title" in token.tags: + ambiguities.append(PendingAmbiguity( + AmbiguityKind.TITLE_OR_NAME, + f"{text!r} is title vocabulary and the only name word " + f"the title peel left standing; read as the name by " + f"convention rather than as more title", + tuple(head))) + elif (n == 0 + and not any(_WORD_ALREADY_CLAIMED & tokens[i].tags + for i in head) # A2's content test: a piece with no alphanumeric # character is no name word, and the name it sits in # assembles empty -- so a convention report there would # describe a reading nobody got. parse("(") keeps its # unbalanced-delimiter report and gains nothing here. and any(c.isalnum() for c in text)): - token = tokens[head[0]] - assert token.role is not None - ambiguities.append(PendingAmbiguity( - AmbiguityKind.GIVEN_OR_FAMILY, - f"{text!r} is the only name word and nothing else " - f"decides it; read as a {token.role.value} name by " - f"convention, which follows the read order", - tuple(head))) + # rules.md#O5's exception: the one name word is a JOIN + # (P3) and one of the words it joins is title vocabulary, + # so the doubt is not which field the unit takes but + # whether that word is a title at all -- 'John of Prince' + # and 'Smith and Prince', the measured inputs that reach + # this branch. A join whose FIRST word is title vocabulary + # never does: H3 chains it into a title run ('Prince of + # Wales'), which reports nothing, the same silence as a + # lone 'Dr.'. + if len(head) > 1 and any( + "vocab:title" in tokens[i].tags for i in head): + ambiguities.append(PendingAmbiguity( + AmbiguityKind.TITLE_OR_NAME, + f"{text!r} is the only name unit and joins title " + f"vocabulary to a name word; read as a " + f"{token.role.value} name by convention", + tuple(head))) + else: + ambiguities.append(PendingAmbiguity( + AmbiguityKind.GIVEN_OR_FAMILY, + f"{text!r} is the only name word and nothing else " + f"decides it; read as a {token.role.value} name by " + f"convention, which follows the read order", + tuple(head))) for piece in peeled.picks: # every pick is in rest, so the loops above just gave it a role token = tokens[piece[0]] diff --git a/nameparser/_types.py b/nameparser/_types.py index 8b1893c8..de09987b 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -397,7 +397,29 @@ class AmbiguityKind(StrEnum): #: vd" takes ``vd`` for *van der* and declines the decoration. #: Which name part was declined depends on position and #: ``name_order``, so ``detail`` names it rather than the kind. + #: The same doubt covers an input that is nothing BUT post-nominal + #: vocabulary ("Rinpoche", "QC MP"): with no name word beside it + #: the first post-nominal is read as the name, because something + #: has to be one, and only that word reports. SUFFIX_OR_NAME = "suffix-or-name" + #: An input the title peel eats down to one last word which is + #: itself title vocabulary still has to name somebody, so that + #: word is read as the name -- "Lord Chancellor" gives family + #: "Chancellor". A convention, not evidence: this is a name parser + #: rather than a title parser, and handed a string whose every + #: remaining word is a title the alternative is a title with no + #: name at all. ``detail`` names that word. The report does not + #: depend on ``name_order``: it is made where the word is placed, + #: which is before the rule that moves it between fields. + #: A LONE title word reports nothing: "Dr." reads as a title + #: standing by itself, the peel left no word to be read as a + #: name, and no fork was taken. + #: One name word that is a JOIN carrying title vocabulary reports + #: this kind too ("John of Prince"): the unit is read as a name, + #: and whether the title word inside it is a title is the fork. + #: Not the title-vs-given-name collision on a word like "Baron", + #: which is a question about the VOCABULARY and is not this kind. + TITLE_OR_NAME = "title-or-name" #: An ambiguous particle is either a particle or a name in its own #: right -- "Van Johnson" is the actor's given name, a bare #: "Van Buren" the presidential surname, and the two-word shape diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 76e25fba..fad20e73 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -614,7 +614,7 @@ def _check_cjk_shape_purity(self) -> None: "'John', middle 'née', last 'Jones')"), Case("title_plus_one_word_comma_suffix", "Dr. King, Jr.", {"title": "Dr.", "family": "King", "suffix": "Jr."}, - classification="fix(#410)", + ambiguities=("title-or-name",), classification="fix(#410)", notes="the most ordinary shape H1's widening touches, and " "the one the suite could least afford to leave " "unpinned: mutating H1 to decline on any name carrying " @@ -625,7 +625,10 @@ def _check_cjk_shape_purity(self) -> None: "'Smith, Dr.' takes its family from the comma rule " "(C1), not from H1. 1.4.0 had the same empty family " "here (title 'Dr.', first 'King', last ''), so this " - "row records a v1 bug fixed, not a v2 divergence"), + "row records a v1 bug fixed, not a v2 divergence. The " + "`title-or-name` flag is H4's and not #410's: `king` is " + "title vocabulary and a suffix beside the word is not a " + "shape that silences the convention"), Case("title_plus_one_word_multi_word_maiden", "Dr. Smith née Mary Jones", {"title": "Dr.", "family": "Smith", "maiden": "Mary Jones"}, @@ -1351,7 +1354,11 @@ def _check_cjk_shape_purity(self) -> None: notes="the plainest O5 shape there is: nothing decides one " "name word, so the convention picks the field and says " "so (#449). Roles parity; the flag is #449's"), - Case("title_only", "Dr.", {"title": "Dr."}), + Case("title_only", "Dr.", {"title": "Dr."}, + notes="rules.md#H4's boundary as well as a shape row: a LONE " + "title word demotes nothing, so no reading was chosen " + "and nothing is reported. The title-vs-given-name " + "collision on a word like `Baron` is #348's"), Case("double_comma_suffix", "Smith, John, Jr.", {"given": "John", "family": "Smith", "suffix": "Jr."}), Case("bound_given_two", "abdul rahman", @@ -2261,7 +2268,9 @@ def _check_cjk_shape_purity(self) -> None: "declines and the convention is what places 'Jones'"), Case("lone_name_word_title_decides_it", "Dr. Smith", {"title": "Dr.", "family": "Smith"}, classification="parity", - notes="negative control: H1 decided it"), + notes="negative control for both conventions: H1 decided it, " + "and `smith` is not title vocabulary, so H4 does not " + "claim the input either"), Case("lone_name_word_nickname_decides_it", "'Smitty' Smith", {"family": "Smith", "nickname": "Smitty"}, classification="parity", notes="boundary: N3 returns before the O5 site is reached, so " @@ -2272,6 +2281,74 @@ def _check_cjk_shape_purity(self) -> None: notes="boundary: the comma named the family before the " "positional read, so one name word never stood alone " "here -- not a control of any guard clause"), + Case("all_titles_input_reports_the_demoted_word", "Lord Chancellor", + {"title": "Lord", "family": "Chancellor"}, + ambiguities=("title-or-name",), classification="feat(#491)", + notes="rules.md#H4 -- nothing but title vocabulary, so the " + "last title word is the name by convention and says so"), + Case("all_titles_input_the_queens_bench_string", + "The Right Hon. the President of the Queen's Bench Division", + {"title": "The Right Hon. the President of the Queen's Bench", + "family": "Division"}, + ambiguities=("title-or-name",), classification="feat(#491)", + notes="decisions.md#v1-xfail-triage's fourth NOT FIXED entry: " + "the reading stands, and the guess stops being silent"), + Case("all_titles_input_a_title_vocabulary_surname", "Dr. King", + {"title": "Dr.", "family": "King"}, + ambiguities=("title-or-name",), classification="feat(#491)", + notes="`king` is title vocabulary for the addressing forms, so " + "the peel leaves one title-vocabulary word and the rule " + "claims it"), + Case("all_titles_input_family_first", "Lord Chancellor", + {"title": "Lord", "family": "Chancellor"}, + policy=Policy(name_order=FAMILY_FIRST), + ambiguities=("title-or-name",), classification="feat(#491)", + notes="the report is made where the word is PLACED, so a " + "declared family-first order -- which puts 'Chancellor' " + "in the family directly and leaves H1's retag no work -- " + "gives the same reading and the same kind. The site was " + "H1's retag until the family-first orders were measured " + "silent there"), + Case("title_run_then_a_credential_reports_nothing", "Dr. King MD", + {"title": "Dr. King", "family": "MD"}, classification="parity", + notes="rules.md#H4's boundary, both halves -- the title peel " + "took `Dr. King` and left no name word at all, so the " + "credential is the name by the bare-suffix carve-out -- " + "which is scoped to a run no title preceded, and the " + "title half needs a name word left standing. Neither " + "claims it"), + Case("title_run_then_a_bare_generational_reports_nothing", + "Dr King Jr", {"title": "Dr King", "family": "Jr"}, + classification="parity", + notes="the same boundary in the shape the assign comment " + "names: `Dr King` peels whole, `Jr` is read as the name " + "for want of another, and after a title that reading is " + "H1's rather than either of H4's conventions"), + Case("all_suffix_input_reports_suffix_or_name", "Rinpoche", + {"given": "Rinpoche"}, ambiguities=("suffix-or-name",), + classification="feat(#491)", + notes="rules.md#H4's suffix half: post-nominal vocabulary with " + "no name word beside it, read as the name"), + Case("all_suffix_input_two_words", "QC MP", + {"given": "QC", "suffix": "MP"}, ambiguities=("suffix-or-name",), + classification="feat(#491)", + notes="the first word is the name and the rest the run; only " + "the word made into a name reports"), + Case("lone_joined_unit_carrying_title_vocabulary", "John of Prince", + {"given": "John of Prince"}, ambiguities=("title-or-name",), + classification="feat(#491)", + notes="rules.md#O5's exception: the one name unit is a join " + "(P3) carrying title vocabulary, so the doubt is " + "whether `Prince` is a title rather than which field " + "the unit takes -- the one input measured to reach " + "that branch, `prince` being in TITLES"), + Case("lone_joined_unit_led_by_a_title_is_a_title_run", + "Prince of Wales", {"title": "Prince of Wales"}, + classification="parity", + notes="boundary: a join whose FIRST word is title vocabulary " + "chains into a title run (H3) and never reaches the " + "assignment site at all, so it reports nothing -- the " + "same silence as a lone `Dr.`"), Case("marker_led_clause_in_a_quote_pair", 'Jane Smith "née Jones"', {"given": "Jane", "family": "Smith", "maiden": "Jones"}, diff --git a/tests/v2/test_contracts.py b/tests/v2/test_contracts.py index 07e1c340..fb05ee20 100644 --- a/tests/v2/test_contracts.py +++ b/tests/v2/test_contracts.py @@ -14,6 +14,7 @@ AmbiguityKind.SUFFIX_OR_NICKNAME: "JEFFREY (JD) BRICKEN", AmbiguityKind.SUFFIX_OR_NAME: "John Smith MA", AmbiguityKind.GIVEN_OR_FAMILY: "Andrew", + AmbiguityKind.TITLE_OR_NAME: "Lord Chancellor", # no emitter yet -- arrives with locale-pack order detection (2.x) AmbiguityKind.ORDER: None, # 남 and 남궁 are both shipped surnames, so longest-first picks diff --git a/tests/v2/test_facade_cases.py b/tests/v2/test_facade_cases.py index b4def303..be631d42 100644 --- a/tests/v2/test_facade_cases.py +++ b/tests/v2/test_facade_cases.py @@ -95,6 +95,11 @@ # reading, and is core-only for the same reason. "lone_name_word_reports_the_convention_family_first", "han_unspaced_family_first_declared_reports_nothing", + # #491: H4's report under a DECLARED family-first order, the half + # that proves the report is order-independent -- the order has no + # v1 spelling, so the row is core-only. The default-order twin + # ("Lord Chancellor") is an ordinary row and runs here. + "all_titles_input_family_first", }) diff --git a/tests/v2/test_ledger_guards.py b/tests/v2/test_ledger_guards.py index 525c8872..e7145d08 100644 --- a/tests/v2/test_ledger_guards.py +++ b/tests/v2/test_ledger_guards.py @@ -941,6 +941,14 @@ def test_case_shape_ids_exist_in_the_inventory() -> None: ("Anderson Smithさん", "Dr. Andersonさん", "Anderson, さん"), "feat(#449) a glued Korean honorific leaves a Latin name word the script cannot order": ("Anderson Smith선생님", "Dr. Anderson선생님", "Anderson, 선생님"), + # #491's two rules. Each boundary is a name the vocabulary reaches + # and the SHAPE does not: a title with an ordinary surname behind + # it, a credential with a name word beside it, and the lone title + # word the peel takes whole, leaving nothing to read as a name. + "feat(#491) an all-titles input reports title-or-name for the word the title peel left as the name": + ("Dr. Smith", "Rabbi Cohen", "Mrs. Garcia", "Dr.", "King, Dr Jr"), + "feat(#491) an all-suffix input reports suffix-or-name for the word it made the name": + ("John Smith QC MP", "Lama Zopa Rinpoche", "Smith Jr.", "Jr."), } @@ -1607,6 +1615,21 @@ class _LatinCopy(NamedTuple): "part1 of The part2 of the part3 and part4", "part1 of and The part2 of the part3 And part4", "test", "سلمان،"}), + # #491's title movers, one corpus name per alternative. A list of + # names, not a copy of TITLES: the rule's subject is a SHAPE the + # vocabulary participates in (the title peel leaves one name word + # and that word is in it), and a member copying the wordlist + # would reach 'Dr. Smith', 'Rabbi Cohen' and 'Mrs. Garcia', which + # do not move. One set, identical in all three 2.x ledgers. + frozenset({"Dr\\. King", "His Holiness", "His Holiness the Dalai Lama", + "Lord Chancellor", + "The Right Hon\\. the President of the Queen's Bench Division", + "The Rt Hon"}), + # #491's suffix movers. A list of names, not a copy of SUFFIX_WORDS + # or SUFFIX_ACRONYMS: a member copying either would reach every + # credential-bearing name in the corpus, and what selects these two + # is that no name word stood beside the credential. + frozenset({"QC MP", "Rinpoche"}), # fix(#445)'s movers, one corpus name per alternative -- a list of # names, not a copy of any wordlist, so there is no vocabulary for # it to drift from. Two sets because the ledgers group the nine @@ -2345,6 +2368,20 @@ def _claim(rule: dict) -> _Claim: "feat(#449) a glued Korean honorific leaves a Latin name word the script cannot order": _Claim(1, ('_ambiguities', 'given', 'suffix'), "30ec95e25f05", None), + # #491's two rules, beside #449's for the same reason: both + # classify on `_ambiguities` alone, and no role moves anywhere + # in this change, so a widening that took a role would change + # the row here before it reached the gate. Six title names and + # two suffix names, one alternative each; a member reaching a + # name the SHAPE does not select ('Dr. Smith', 'Rabbi Cohen', + # 'Smith Jr.', the lone 'Dr.' the peel takes whole) would move + # the count, and _MUST_NOT_MATCH carries those probes. Same + # digests in all three 2.x ledgers: the regexes are the same + # strings and the corpora are one set. + "feat(#491) an all-titles input reports title-or-name for the word the title peel left as the name": + _Claim(6, ('_ambiguities',), "9ee2a07d96c8", None), + "feat(#491) an all-suffix input reports suffix-or-name for the word it made the name": + _Claim(2, ('_ambiguities',), "e0756e2e1cd4", None), # #346's alternation. Four corpus names, `family` and # `given` together: the fold moves both roles at once, so a # widening taking one alone would change the roles here @@ -2551,6 +2588,20 @@ def _claim(rule: dict) -> _Claim: _Claim(1, ('_ambiguities',), "528858346d14", None), "feat(#449) a glued Korean honorific leaves a Latin name word the script cannot order": _Claim(1, ('_ambiguities',), "30ec95e25f05", None), + # #491's two rules, beside #449's for the same reason: both + # classify on `_ambiguities` alone, and no role moves anywhere + # in this change, so a widening that took a role would change + # the row here before it reached the gate. Six title names and + # two suffix names, one alternative each; a member reaching a + # name the SHAPE does not select ('Dr. Smith', 'Rabbi Cohen', + # 'Smith Jr.', the lone 'Dr.' the peel takes whole) would move + # the count, and _MUST_NOT_MATCH carries those probes. Same + # digests in all three 2.x ledgers: the regexes are the same + # strings and the corpora are one set. + "feat(#491) an all-titles input reports title-or-name for the word the title peel left as the name": + _Claim(6, ('_ambiguities',), "9ee2a07d96c8", None), + "feat(#491) an all-suffix input reports suffix-or-name for the word it made the name": + _Claim(2, ('_ambiguities',), "e0756e2e1cd4", None), # #346's alternation. Four corpus names, `family` and # `given` together: the fold moves both roles at once, so a # widening taking one alone would change the roles here @@ -2611,6 +2662,20 @@ def _claim(rule: dict) -> _Claim: _Claim(1, ('_ambiguities',), "528858346d14", None), "feat(#449) a glued Korean honorific leaves a Latin name word the script cannot order": _Claim(1, ('_ambiguities',), "30ec95e25f05", None), + # #491's two rules, beside #449's for the same reason: both + # classify on `_ambiguities` alone, and no role moves anywhere + # in this change, so a widening that took a role would change + # the row here before it reached the gate. Six title names and + # two suffix names, one alternative each; a member reaching a + # name the SHAPE does not select ('Dr. Smith', 'Rabbi Cohen', + # 'Smith Jr.', the lone 'Dr.' the peel takes whole) would move + # the count, and _MUST_NOT_MATCH carries those probes. Same + # digests in all three 2.x ledgers: the regexes are the same + # strings and the corpora are one set. + "feat(#491) an all-titles input reports title-or-name for the word the title peel left as the name": + _Claim(6, ('_ambiguities',), "9ee2a07d96c8", None), + "feat(#491) an all-suffix input reports suffix-or-name for the word it made the name": + _Claim(2, ('_ambiguities',), "e0756e2e1cd4", None), # #346's alternation. Four corpus names, `family` and # `given` together: the fold moves both roles at once, so a # widening taking one alone would change the roles here diff --git a/tools/differential/corpus_rules.jsonl b/tools/differential/corpus_rules.jsonl index 692bc03d..123f24dd 100644 --- a/tools/differential/corpus_rules.jsonl +++ b/tools/differential/corpus_rules.jsonl @@ -31,8 +31,10 @@ "Berg, abdul van" "Berg, abdul vd" "Del Toro" +"Dr." "Dr. John van der Berg" "Dr. Juan Q. Xavier de la Vega III" +"Dr. King" "Dr. Smith" "Dr. Smith née Jones" "Dr. Smith, John" @@ -108,6 +110,7 @@ "Jong, Anke de" "Jong, Piet de" "Jose E Maria Santos" +"Jr." "Juan & Garcia" "Juan McDonald" "Juan and Garcia" @@ -118,6 +121,7 @@ "Juan y Eva Garcia" "Juan y Garcia" "Juan y Garcia née Jones" +"Lord Chancellor" "Mari' Aube'" "Maria Kowalska (z domu Nowak)" "Maria Kowalska (z domu)" @@ -142,7 +146,9 @@ "Nguyễn, Thị Vân" "Ph. D. Van Johnson" "Ph. D., John" +"QC MP" "Rev. John Smith" +"Rinpoche" "SHIRLEY MACLAINE" "Salam, abd Allah" "Sean O'Connor" @@ -182,6 +188,7 @@ "Smith, PhD" "Smith, Sr." "Smith, de Mesnil Jean" +"The Right Hon. the President of the Queen's Bench Division" "Van Johnson" "Vega, Juan de la" "Vincent van Gogh van Beethoven" diff --git a/tools/differential/expected_since_2.0.0.toml b/tools/differential/expected_since_2.0.0.toml index e3272fd1..cc784fa4 100644 --- a/tools/differential/expected_since_2.0.0.toml +++ b/tools/differential/expected_since_2.0.0.toml @@ -139,6 +139,35 @@ issue = "feat(#449) a glued Korean honorific leaves a Latin name word the script name_regex = "^Anderson선생님$" fields = ["_ambiguities", "given", "suffix"] +# The two #491 rules sit beside #449's, same position rule: both +# declare `fields = ["_ambiguities"]`, so they are disjoint from the +# `suffix` rule that owns first place and ahead of every rule below +# whose fields are a strict superset of theirs (#382's narrow-first +# default). +[[change]] +issue = "feat(#491) an all-titles input reports title-or-name for the word the title peel left as the name" +# rules.md#H4. Six corpus names, one alternative each -- a list of +# NAMES, not a copy of TITLES: what selects them is the SHAPE (the +# title peel leaves exactly one name word, and that word is itself +# title vocabulary), and a member copying the wordlist would reach +# 'Dr. Smith', 'Rabbi Cohen' and every other title-plus-surname name, +# none of which moves. Declared in _NOT_A_VOCABULARY_COPY for that +# reason. `_ambiguities` alone: the reading each of these gets is +# unchanged, only the silence about it. +name_regex = "(?i)^(?:Dr\\. King|His Holiness|His Holiness the Dalai Lama|Lord Chancellor|The Right Hon\\. the President of the Queen's Bench Division|The Rt Hon)$" +fields = ["_ambiguities"] + +[[change]] +issue = "feat(#491) an all-suffix input reports suffix-or-name for the word it made the name" +# rules.md#H4's suffix half, emitted at assign's bare-suffix +# carve-out. Two corpus names. A list of names, not a copy of +# SUFFIX_WORDS or SUFFIX_ACRONYMS: a member copying either would reach +# 'John Smith MD', 'Smith Jr.' and the rest of the corpus's credential +# names, none of which moves -- what selects these two is that nothing +# was left to be the name. +name_regex = "(?i)^(?:QC MP|Rinpoche)$" +fields = ["_ambiguities"] + # #346: swami, guru, baba and lama moved from the TITLES-only block # into GIVEN_NAME_TITLES on 2026-09-06. rules.md#H1's Accepted clause # -- "a given-name title plus one name word leaves the family empty" diff --git a/tools/differential/expected_since_2.1.0.toml b/tools/differential/expected_since_2.1.0.toml index d6e1cef2..6af933c4 100644 --- a/tools/differential/expected_since_2.1.0.toml +++ b/tools/differential/expected_since_2.1.0.toml @@ -159,6 +159,35 @@ issue = "feat(#449) a glued Korean honorific leaves a Latin name word the script name_regex = "^Anderson선생님$" fields = ["_ambiguities"] +# The two #491 rules sit beside #449's, same position rule: both +# declare `fields = ["_ambiguities"]`, so they are disjoint from the +# `suffix` rule that owns first place and ahead of every rule below +# whose fields are a strict superset of theirs (#382's narrow-first +# default). +[[change]] +issue = "feat(#491) an all-titles input reports title-or-name for the word the title peel left as the name" +# rules.md#H4. Six corpus names, one alternative each -- a list of +# NAMES, not a copy of TITLES: what selects them is the SHAPE (the +# title peel leaves exactly one name word, and that word is itself +# title vocabulary), and a member copying the wordlist would reach +# 'Dr. Smith', 'Rabbi Cohen' and every other title-plus-surname name, +# none of which moves. Declared in _NOT_A_VOCABULARY_COPY for that +# reason. `_ambiguities` alone: the reading each of these gets is +# unchanged, only the silence about it. +name_regex = "(?i)^(?:Dr\\. King|His Holiness|His Holiness the Dalai Lama|Lord Chancellor|The Right Hon\\. the President of the Queen's Bench Division|The Rt Hon)$" +fields = ["_ambiguities"] + +[[change]] +issue = "feat(#491) an all-suffix input reports suffix-or-name for the word it made the name" +# rules.md#H4's suffix half, emitted at assign's bare-suffix +# carve-out. Two corpus names. A list of names, not a copy of +# SUFFIX_WORDS or SUFFIX_ACRONYMS: a member copying either would reach +# 'John Smith MD', 'Smith Jr.' and the rest of the corpus's credential +# names, none of which moves -- what selects these two is that nothing +# was left to be the name. +name_regex = "(?i)^(?:QC MP|Rinpoche)$" +fields = ["_ambiguities"] + # The four CJK names of #436/#437's class, one rule each. Not one # alternation: an alternation holding a script-classified member is # claimed by the honorific pin in tests/v2/test_ledger_guards.py, diff --git a/tools/differential/expected_since_2.2.0.toml b/tools/differential/expected_since_2.2.0.toml index 5e1dbfe6..49612904 100644 --- a/tools/differential/expected_since_2.2.0.toml +++ b/tools/differential/expected_since_2.2.0.toml @@ -148,6 +148,35 @@ issue = "feat(#449) a glued Korean honorific leaves a Latin name word the script name_regex = "^Anderson선생님$" fields = ["_ambiguities"] +# The two #491 rules sit beside #449's, same position rule: both +# declare `fields = ["_ambiguities"]`, so they are disjoint from the +# `suffix` rule that owns first place and ahead of every rule below +# whose fields are a strict superset of theirs (#382's narrow-first +# default). +[[change]] +issue = "feat(#491) an all-titles input reports title-or-name for the word the title peel left as the name" +# rules.md#H4. Six corpus names, one alternative each -- a list of +# NAMES, not a copy of TITLES: what selects them is the SHAPE (the +# title peel leaves exactly one name word, and that word is itself +# title vocabulary), and a member copying the wordlist would reach +# 'Dr. Smith', 'Rabbi Cohen' and every other title-plus-surname name, +# none of which moves. Declared in _NOT_A_VOCABULARY_COPY for that +# reason. `_ambiguities` alone: the reading each of these gets is +# unchanged, only the silence about it. +name_regex = "(?i)^(?:Dr\\. King|His Holiness|His Holiness the Dalai Lama|Lord Chancellor|The Right Hon\\. the President of the Queen's Bench Division|The Rt Hon)$" +fields = ["_ambiguities"] + +[[change]] +issue = "feat(#491) an all-suffix input reports suffix-or-name for the word it made the name" +# rules.md#H4's suffix half, emitted at assign's bare-suffix +# carve-out. Two corpus names. A list of names, not a copy of +# SUFFIX_WORDS or SUFFIX_ACRONYMS: a member copying either would reach +# 'John Smith MD', 'Smith Jr.' and the rest of the corpus's credential +# names, none of which moves -- what selects these two is that nothing +# was left to be the name. +name_regex = "(?i)^(?:QC MP|Rinpoche)$" +fields = ["_ambiguities"] + # The four CJK names of #436/#437's class, one rule each. Not one # alternation: an alternation holding a script-classified member is # claimed by the honorific pin in tests/v2/test_ledger_guards.py, From c23e21cbe5e1a10cb233af80e0af4ea6bb4a35ba Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Tue, 8 Sep 2026 01:12:45 -0700 Subject: [PATCH 3/5] =?UTF-8?q?docs(design+release):=20the=20conventions?= =?UTF-8?q?=20are=20reported=20=E2=80=94=20#449=20and=20#491=20recorded?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit decisions.md gains two sections. #O5 records what #449's measurement actually settled -- the population, not the principle: the naive one-name-piece condition reaches 143 of 1123 corpus names, and O5's own population is 27, 12 contract and 15 radar. It names the two guard conditions the rule had not stated (a script order decides, which needed _effective_order to return an Order carrying by_script, and a piece with no letter or digit is no name word), the four conditions dropped as inert with why each was inert, the five CJK-bearing names that report because a script rule DECLINED or a glued honorific left a Latin stem, and the deliberate gap between the rule's scope and the report's. #H4 records #491: the reading the v1-xfail triage accepted as convention is unchanged, and the silence it called the real defect is what ends. Six corpus names, one emitter at assign's lone-name-word site rather than at H1's retag so the report is order-independent, Dr. King in scope on purpose, a lone title word demoting nothing and reporting nothing, the joined-title refinement, the suffix half's two guard exclusions, and the MA-versus-PhD asymmetry noted rather than fixed. Three existing passages are repointed: the 2026-09-07 #471 keystone note now says two thirds of the ambiguity bundle shipped and #348 was left out by choice; the v1-xfail-triage entry's Queen's Bench bullet points at H4; the 2026-08-27 #445 entry's promised re-measurement is taken. No `Open:` pointer is removed -- neither issue ever had one. rules.md#O5 and #H4 gain their history: pointers now the anchors exist. concepts.rst gains one sentence; release_log.rst two Additions bullets. Co-Authored-By: Claude Fable 5.1 --- docs/concepts.rst | 9 ++++++++- docs/design/decisions.md | 31 +++++++++++++++++++++++++++---- docs/design/rules.md | 30 +++++++++++++++++++++--------- docs/release_log.rst | 4 ++++ tests/v2/cases.py | 14 +++++++++++++- 5 files changed, 73 insertions(+), 15 deletions(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index 6d0ce6f6..4e94925a 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -176,7 +176,14 @@ de Souza"`` it sits mid-name, where nothing has to choose between readings — so nothing is recorded. A comma can settle the question before it arises, too: ``"Ma, Jack"`` fixes the family name, so the credential reading never comes up, while ``"John Smith MA"`` has to -call it and says so. +call it and says so. Some decisions are conventions rather than +readings — a name of one name word has nothing to compare, and an +input the title peel eats down to one last title word reads that word +as the name for want of anything else — and since 2.3 those are +reported too, so a field the library merely had to pick is a field you +can see it picked. A title standing entirely alone reports nothing: +``"Dr."``, ``"Sir"`` and ``"Prince of Wales"`` are all title and no +name, so there is no reading to have doubted. An empty ``ambiguities`` is therefore not a certificate of certainty. Reporting is deliberately partial: :class:`~nameparser.AmbiguityKind` diff --git a/docs/design/decisions.md b/docs/design/decisions.md index fb3b84e8..c92b7d17 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -60,9 +60,9 @@ the 2026-08-16 entries below. The survivor is the degenerate bare This SUPERSEDES the REASON the 2026-08-17 entry above gives for the same outcome — "The mixed-language shapes are what make the greedy reading look wrong, and the parser cannot see language." That was right and weaker. The order is not a language judgement at all: it is a declaration about the data, and a caller who has one supplies it whether or not a parser could ever have guessed it. The outcome that entry reached, and everything else in it, stands — and so does its "Why the question kept thrashing" paragraph: the REACH question is a language judgement, which is why the order declaration is where the caller supplies it. What is superseded is only the claim that the parser's blindness to language is the reason. It also answers the 2026-08-16 #364 entry's "Nothing ever argued for 'takes everything'; it was the shape of v1's handle_non_first_name_prefix, not a decision." This entry argues for it. The v1 provenance is unchanged and the argument is new, which is the only thing that had been missing. Measured 2026-09-07 on the tree, and on the 1.4.0, 2.0.0, 2.1.0 and 2.2.0 wheels, which agree on every default-order row: `de la Torre Vega` gives family `de la Torre Vega`; `de la Cruz Juan Carlos` gives family `de la Cruz Juan Carlos`; `de Mesnil Jean` gives family `de Mesnil Jean`; and the comma form `de la Torre Vega, Juan` gives family `de la Torre Vega`, given `Juan` under both orders. Under FAMILY_FIRST `de la Torre Vega` reads family `de la Torre`, given `Vega` — the cost the declaration accepts, and the reason a surname-only record in a family-first source still wants its comma. Recompute with `Parser().parse(name)` and `Parser(policy=Policy(name_order=FAMILY_FIRST)).parse(name)`; for the wheels, `HumanName(name).last` from a directory outside the checkout so the tree cannot shadow them. - Nothing moves. No parse changes; the `parity` row `leading_never_given_particle_two_leftovers` in tests/v2/cases.py and the guard `test_the_family_first_fold_is_not_explained_under_the_default_order` in tests/v2/test_ledger_guards.py stay as the pins of the decided behavior; no release-note bullet is written, there being no behavior to note and no by-design precedent in docs/release_log.rst. rules.md#P1 gains `de la Torre Vega` as an example, which puts the name in the contract corpus corpus_rules.jsonl. Its parity is over the SEVEN roles, not the `initials` view: at the 1.4.0 baseline only, the name falls into the already-recorded class `fix(initials-per-word) a particle chain inside a name part` beside its sibling `de la Cruz Juan Carlos` — the v1→v2 per-word initialling shipped since 2.0.0, not a regression — so the 1.4.0 intentional count goes 367 → 368 and that rule's recorded claim goes 107 → 108 names with a new digest. The three 2.x baselines are unmoved at 263 / 175 / 37, and `unexplained: 0` at all four. + Nothing moves. No parse changes; the `parity` row `leading_never_given_particle_two_leftovers` in tests/v2/cases.py and the guard `test_the_family_first_fold_is_not_explained_under_the_default_order` in tests/v2/test_ledger_guards.py stay as the pins of the decided behavior; no release-note bullet is written, there being no behavior to note and no by-design precedent in docs/release_log.rst. rules.md#P1 gains `de la Torre Vega` as an example, which puts the name in the contract corpus corpus_rules.jsonl. Its parity is over the SEVEN roles, not the `initials` view: at the 1.4.0 baseline only, the name falls into the already-recorded class `fix(initials-per-word) a particle chain inside a name part` beside its sibling `de la Cruz Juan Carlos` — the v1→v2 per-word initialling shipped since 2.0.0, not a regression — so the 1.4.0 intentional count goes 367 → 368 and that rule's recorded claim goes 107 → 108 names with a new digest. The three 2.x baselines are unmoved at 263 / 175 / 37 (296 / 210 / 72 after #449 and #491 landed the same day; #471 itself moved none), and `unexplained: 0` at all four. The `Open:` pointer to #471 that stood at the end of decisions.md#P6's 2026-08-30 #467 entry is removed with this entry, which is where the question is now answered. P1's own `Open:` block below is #360's and is untouched. - Deliberately NOT decided here: reporting an ambiguity on the shapes neither order settles — the honest-output flag — would be an AmbiguityKind and belongs with the ambiguity bundle (#449/#491/#348) if anywhere. Declining the reach change does not decline that. + Deliberately NOT decided here: reporting an ambiguity on the shapes neither order settles — the honest-output flag — would be an AmbiguityKind and belongs with the ambiguity bundle (#449/#491/#348) if anywhere. Declining the reach change does not decline that. Two thirds of that bundle shipped 2026-09-07 as decisions.md#O5 and decisions.md#H4; #348 was left OUT by choice, being a question about the TITLES vocabulary rather than about a decision site, and the shapes this sentence is about are neither one — no rule fixes them by convention, so there is no branch to report at. Declined: @@ -271,7 +271,7 @@ The reconciled v1-style banks (`tests/test_*.py`) carried eight `@pytest.mark.xf - `Dr King Jr` — v1 wanted title 'Dr', family 'King', suffix 'Jr'; it reads title 'Dr King', family 'Jr'. `king` stays in the titles vocabulary: it is there for the addressing forms ("King Charles"), and taking it out to serve the surname reading trades a common use for a rarer one, which is the direction #vocabulary-collisions cuts. TITLES has no ambiguous subset and no AmbiguityKind, so — as with MAIDEN_MARKERS and `roz` — the only two expressions of the criterion available here are ship and do not ship; #348 is the open work that would give this set a third answer. The comma format is the road to the surname reading, and is now pinned alongside rather than left as prose — `King, Dr Jr` reads title 'Dr', family 'King', suffix 'Jr'. The test cites [#27](https://github.com/derek73/python-nameparser/issues/27), which is closed; this is the decision it never got. Two halves, and only one is decided. DECIDED: `king` stays in TITLES. RECORDED, not endorsed: what becomes of the leftover `Jr`. rules.md#S2 predicts suffix 'Jr' with an empty family — its Accepted clause consumes an unambiguous suffix even when nothing is left to be the family (`Smith Jr.` → family "") — but once the title chain has taken `Dr King`, H1 claims the one remaining word and it reads family 'Jr', suffix ''. `Dr Smith Jr` isolates the cause: family 'Smith', suffix 'Jr', exactly as S2 states. S2 now carries a descriptive note saying so. A future change moving `Dr King Jr` toward S2's prediction is an IMPROVEMENT and updates the pin; it is not a regression, and the test says as much so nobody reads the pin as an endorsement. - `Ahmad ben Husain` — v1 wanted family "ben Husain"; it reads given Ahmad, middle ben, family Husain. Already decided in v0.2.5, when `ben` came out of the prefixes, and for the reason that still holds: `ben` collides with the given name Ben, in the position the particle claim would act on — `Ahmad Ben Husain` reads middle 'Ben' today, which is exactly the token a forward-joining particle claim would take. That is C-i's position test, and it keeps `ben` out. Recorded a second time as a standing keep-out in this file's Excluded block for the particle set, because that is where a wordlist sweep meets it: a keep-out that lives only in a triage entry is one the next Arabic/Hebrew patronymic-particle sweep never reads. Worth naming as a failure mode of its own — the marker was an aspiration that outlived its own resolution, and nothing in a bare xfail says which of the eight were like that. - - `The Right Hon. the President of the Queen's Bench Division` — v1 wanted the whole string as one title; it reads title "The Right Hon. the President of the Queen's Bench", family 'Division'. This is a name parser, not a title parser: handed an input that is all titles it assumes the last title-word is the name. Accepted as convention rather than defended as correct — what is actually wrong is that the guess is silent, which is [#491](https://github.com/derek73/python-nameparser/issues/491), not this reading. + - `The Right Hon. the President of the Queen's Bench Division` — v1 wanted the whole string as one title; it reads title "The Right Hon. the President of the Queen's Bench", family 'Division'. This is a name parser, not a title parser: handed an input that is all titles it assumes the last title-word is the name. Accepted as convention rather than defended as correct — and since 2026-09-07 the guess is no longer silent: rules.md#H4 states the convention and the parse reports `title-or-name`, which is what [#491](https://github.com/derek73/python-nameparser/issues/491) asked for and is a report rather than a change of reading. See decisions.md#H4. - 2026-09-01 — FIX CANDIDATES, four. The marker stays and now carries its issue, so `pytest -rx` names the work instead of listing anonymous aspirations: - [#489](https://github.com/derek73/python-nameparser/issues/489) — `Her Majesty Queen Elizabeth` should address by given name (`tests/test_conjunctions.py::test_conjunction_in_an_address_with_a_first_name_title`). - [#490](https://github.com/derek73/python-nameparser/issues/490) — `E.T. Smith` (`tests/test_conjunctions.py::test_two_initials_conflict_with_conjunction`) and `U.S. District Judge Marc Thomas Treadwell` (`tests/test_titles.py::test_chained_title_first_name_title_is_initials`). One issue for two tests deliberately: each test's own comment names the other's shape as what blocks a fix — dotted initials against dotted title and credential vocabulary — so they are one question, and fixing either alone is what has failed before. @@ -339,6 +339,18 @@ Declined (rc1 arc; the full argument is AGENTS.md's gotcha): ACCEPTED: `Parser.revise(suffix="Ph. D.")` now renders 'Ph., D.'. revise() runs a full sub-parse of the string it is given, and a field value has no head for a head-position rule to consult; a second draft carved that out by requiring a name to displace, and the carve-out made the head reading depend on what FOLLOWED it — appending a maiden clause changed whether 'Ph.' was a title, which `test_a_maiden_clause_changes_nothing_else` caught. Dropping the carve-out removed the inconsistency with it. The merge exists for a credential someone TYPED after a name; a caller who writes the spaced form into the suffix field is taken at their word (Derek's call, 2026-08-31). SUPERSEDED 2026-09-06 by #511, and half of the reasoning here still holds: the MERGE is a head-position rule and still does not fire in a field value. The other half — a caller who writes the spaced form is taken at their word — is what #511 reverses (Derek's call, 2026-09-06, choosing the entry rule over the accepted split when the round-trip limit was measured). What changed is that rules.md#R1's entry pass now runs inside `revise()` over the forced roles, and `Ph.` and `D.` share a comma bucket with nothing between them, so the ENTRY pass joins the pair for the reason it joins `MD` and `PhD` — a different rule reaching the same two words. `Parser.revise(suffix="Ph. D.")` renders 'Ph. D.' (decisions.md#C1, 2026-09-06 #511). What the leading `Ph.` becomes is H2's business, not this rule's: an abbreviation before a name is almost always a title, which is the same clause that reads `Esq. van Gogh` as title 'Esq.' — so the pair reads title 'Ph.', given 'D.'. The issue proposed given 'Ph.', middle 'D.'; H2 claims the first word before the positional read sees it, and the two answers differ only in which field holds `Ph.`. +### O5 — the lone name word's convention, and reporting it + +- 2026-09-07 #449 — the convention is REPORTED, as `given-or-family`, with `detail` naming the field it chose. rules.md#O5 has said since #445 that the one-word reading is a guess fixed in advance rather than a determination; what #449 asked is what the library should SAY about it, and the answer is the one A1 gives everywhere else — a caller can only act on doubt that is reported. `detail` names the field rather than the kind naming it, because the field follows the read order and the kind does not: that is the PARTICLE_OR_GIVEN precedent, recorded at that kind's docstring. + What the measurement settled was the POPULATION, not the principle. The naive condition — one name piece placed, nothing else looked at — fires on 143 of the 1123 distinct corpus names, because a title peel, a nickname, a maiden clause, a trailing credential run or a script order can each leave exactly one piece standing where the convention decided nothing. The 2026-08-27 #445 entry below records "19 of 1,085" for a differently written naive condition on a smaller corpus; both numbers are naive and neither is O5's. Guarded on O5's own carve-out list the population is 27 corpus names, 12 contract and 15 radar, and the other 116 report nothing at this site, 6 of them reporting `title-or-name` instead (H4's branch is ahead of O5's at the same line). A first draft wrote 110, which is the subtraction done wrong. Recompute by wrapping `_name_positions` with a recorder that fires at `count == 1`, parsing the deduped `corpus*.jsonl` glob through it, and comparing that set against the names whose parse reports the kind; measured 2026-09-08 on this branch. The guard itself is the code, which is where it should be read. + Two conditions are in the guard that O5's statement had not named, and both are in it now. A script whose own convention settles the order decided the reading, so W4 outranks this the way H1 does — that is what keeps 毛泽东, 高橋みなみ, 씨 and the rest of the script-ordered names out. Asking that question needed a shape change: `_effective_order` used to return the roles triple alone, and a caller comparing that triple against the declared `name_order` cannot tell a script rule DECIDING from a declaration standing unopposed — under a declared family-first order a Han name's W4 entry and the declaration agree. It now returns an `Order(roles, by_script)` and the emitter reads the flag. The second condition is content: a piece with no letter or digit in it is no name word, A2 empties such an input, and a convention report on an empty name would describe a reading nobody got — so `parse("(")` keeps its unbalanced-delimiter report and gains nothing. + Five CJK-bearing names DO report, and none of them is a script rule deciding. Three are a script rule DECLINING: マイケル (W4 gives katakana no order of its own, a wholly-katakana name being read as a transcription), 王·Smith (T3's interpunct suppresses the script_orders lookup whole) and 田中、太郎 (the comma sits inside the token, so the text resolves to no single script). Two are a glued honorific peeled off a Latin stem — Andersonさん and Anderson선생님 — where what is left is Latin and has no script order at all. Recorded because "no CJK name reports" was the expectation going in, and it is wrong in a way that is right: where the script rules decline, what is left IS the one-word convention. + Four conditions were drafted into the guard and none of them shipped, but they were dropped for TWO different reasons, and a first draft of this entry flattened both into "inert". Each was measured the same way: restore that one clause on the O5 branch of a `git archive` copy of this commit, reparse the deduped `corpus*.jsonl` glob, and count which of the 27 lose the report. Two are genuinely INERT, silencing ZERO names (measured 2026-09-08). `len(state.segments) == 1`: the family-comma path names the family before segment 0 is read positionally, so the site is never called with one piece under a comma. `not flagged`: a split credential decides the field no more than a whole one does. The general shape is the one this file keeps recording — a guard written from the cases that prompted it carries terms the code already excludes — and the instrument that finds them is mutation, not reading. + The other two were dropped DELIBERATELY, and the same mutation prices each. `not suffix_pieces` silences NINE of the 27 — `Smith Jr.`, `'Smitty' Jones Jr.`, `Carod i`, `Donald mc`, `Jack M.A.`, `John V`, `Mohamad X`, `Andersonさん`, `Anderson선생님` — and `not has_nickname` silences ONE, `'Smitty' Jones Jr.`. Both were dropped on one principle: a suffix or a nickname standing beside the lone name word decides nothing about that word's FIELD, which is the only question this report is about. `Smith Jr.` and `'Smitty' Jones Jr.` are S2's peel taking the suffix and the convention placing what is left, so they report. A first draft justified the nickname clause with "N3 returns before this site", and that is false as written: `_assign.py`'s N3 branch tests `len(pieces) == 1 and len(rest) == 1 and has_nickname`, so it returns only where the segment holds ONE piece, and `'Smitty' Jones Jr.` holds two and reaches this line. What N3 takes is the nickname-plus-one-word shape, not every name a nickname stands in. + Not reported, deliberately: `abdul`, where bound-given vocabulary claimed the word; `de`, where a lone particle's reading is P4's; and `J.`, where the initial's shape is the claim. Those are M4's `_NEVER_FLIPPED` pair plus the particle, read off the tags classify already recorded rather than off a predicate of this emitter's own. Also not reported: `Sir John`, `'Smitty' Jones` and `abd née Jones`, which O5's example block lists because the convention is what H1, N3 and M4 each left behind on them. The rule's SCOPE and the report's scope differ there, and the difference is stated in the rule rather than left to be discovered: the report is narrowed to an input nothing else stands in, because a report on a name where a title, a nickname or a maiden name is visible in the output tells the caller less than the output already does. + The report is order-independent, which is not the same as the FIELD being order-independent: `Andrew` reports under the default order with `detail` naming given and under FAMILY_FIRST with `detail` naming family. Measured over the whole corpus under the default order, FAMILY_FIRST and FAMILY_FIRST_GIVEN_LAST, the set of names reporting this kind is the same 27 in all three; the only names whose reported kinds move with the order at all are four pre-existing `particle-or-given` reports. + Nothing moves but the report. Every role is identical before and after — measured by parsing all 1123 corpus names under all three orders against a `git archive` of the parent commit, 3369 parses, zero role differences — and the three 2.x ledgers classify the 27 names on `_ambiguities` alone; `_ambiguities` cannot enter a diff below baseline 2.0, so `expected_since_1.4.0.toml` is untouched and its 368 intentional diffs are unchanged. SIX ledger rules rather than one, because an alternation carrying a script-classified member is claimed by the honorific pin in tests/v2/test_ledger_guards.py, which would demand it be a hand copy of GLUED_HONORIFICS: the twenty-two Latin-and-Arabic names ride one alternation, declared in `_NOT_A_VOCABULARY_COPY`, and the five CJK-bearing names take a literal-anchored rule each, which is also where their arguments differ. The three ledger copies are NOT byte-identical, which the drafting expected them to be: in `expected_since_2.0.0.toml` alone the two glued-honorific rules declare `["_ambiguities", "given", "suffix"]`, because at that baseline #308's peel has not shipped and its diff arrives in the same rule as this report, and the gate's over-declared check refuses a rule declaring a role no diff it explains moves. + ### O4 — positional assignment and declared order - 2026-08-07 #83 — romanized Chinese order is answered by DECLARATION, not detection: pinyin carries no signal and diaspora makes locale no guide (the issue's own 2019/2021 conclusions); Policy(name_order=FAMILY_FIRST) is the answer the thread wanted. @@ -386,6 +398,17 @@ Declined (ambiguity kinds for script-resolved names, 2026-07-27): Open: [#316](https://github.com/derek73/python-nameparser/issues/316) what a trailing title-vocabulary word should do (the comma paths disagree today). +### H4 — an input that is nothing but vocabulary still has to name somebody + +- 2026-09-07 #491 — the reading is unchanged and the silence is what ends. Handed a string the title peel eats down to one last word which is itself title vocabulary, the parser reads that word as the name; decisions.md#v1-xfail-triage recorded it in the fourth of its NOT FIXED entries — "this is a name parser, not a title parser" — and said in the same breath that what is actually wrong is the guess being silent. It now reports `title-or-name`, with `detail` naming the word that was made into the name. + Six corpus names gain it: the Queen's Bench string, `Lord Chancellor`, `Dr. King`, `The Rt Hon`, `His Holiness` and `His Holiness the Dalai Lama`. Three read contract-tier and three radar, and the tier split is an artifact of the documentation rather than of the shape — the three contract ones are contract because this bundle made them rules.md examples, which puts them in corpus_rules.jsonl. `Dr. King` is the one worth arguing about and it is deliberate: `king` is in the titles vocabulary for the addressing forms, which the triage entry above decided and did not reopen, so `Dr. King` IS an input whose last standing word is title vocabulary and the rule claims it. Reporting there is honest rather than noisy — the reading came from a convention, not from anything the input says — and a caller who wants only the exotic cases has `detail` to filter on. + ONE emitter, and it is at assign's lone-name-word site rather than at H1's retag, which is where the drafting put it. H1 is not the site: under a declared family-first order the assignment places the word in the family directly and H1 never runs, so an emitter there would report under one order and not the other for a reading that is the same either way. Measured under the default order, FAMILY_FIRST and FAMILY_FIRST_GIVEN_LAST, the six report identically. That placement is also why the `detail` names no field, unlike O5's: under the default order H1 retags the word after assign, so a field named at the emitter would be the one the word was placed in and not the one it ends in — and the fork the kind reports is title-versus-name, which no field answers either way. + What is silent, all measured 2026-09-08. A lone title word: `Dr.`, `Sir`, `King` and the chained `Prince of Wales` are a title run with nothing behind it, the peel takes the whole string, no word is left standing to be read as a name, and nothing was chosen — mechanisms.md#AMBIGUITY-AT-THE-DECISION-SITE's "a branch that runs but changes nothing is not a decision". A title with an ordinary word behind it: `Dr. Smith` and `King Charles` leave a word standing, but not a title-vocabulary one, so H1 alone explains them. And a title followed by post-nominal vocabulary: `Dr King Jr` and `Dr. King MD` peel `Dr King` and `Dr. King` WHOLE, leaving a credential that the bare-suffix carve-out makes the name — a different convention, and its report is scoped to `n == 0`, so a title in front of the run takes the input out of it and leaves the reading H1's. The peeled titles are never tested for anything: H2 makes an unlisted abbreviation a title by SHAPE, and `Xyz. Smith` is not this input. What the rule turns on is the word left standing. + A refinement of Derek's, made after the population was measured and widening the rule past the all-titles shape it was drafted for: a lone name word that is a JOIN (P3) carrying title vocabulary reports `title-or-name` too, the fork there being whether the title word inside the unit is a title at all rather than which field the unit takes. `John of Prince` and `Smith and Prince` are the measured inputs; no corpus name reaches that branch, because a join LED by a title word is chained into a title run by H3 (`Prince of Wales`), so the two are pinned as case rows rather than as rules.md examples. It sits on O5's branch and takes precedence there, which is why O5's statement says a title silences THIS kind and not every report at the site. + The suffix half is the same argument on the other vocabulary and needed no new kind. An input whose every word is post-nominal vocabulary reads its first word as a name — assign's "everything suffix-shaped after titles: first one is the name" carve-out — and that is the doubt SUFFIX_OR_NAME already names, so it reports that. `Rinpoche` and `QC MP` are the corpus names; `PhD`, `MBA` and `III` are the same shape. `Jr.` alone is NOT the shape at all: H2's opening-abbreviation rule reads it as a title before the suffix vocabulary is consulted, which is that rule's stated precedence and is recorded here because the expectation going in was that `Jr.` alone read as a name. The guard carries two exclusions of its own. A maiden name beside the credential says the input is not post-nominal vocabulary and nothing else, so `abd née Jones` is out for the reason M4 keeps it out of O5's report. And the report is scoped to names no script order placed, so a lone glued CJK honorific — さん, 씨, 선생님 — reports nothing: that is the same shape read through the glued-honorific rules (W2, #271/#308) and the script's own order, and whether those readings should report is left to the arc that revisits them rather than settled here. + An asymmetry on the boundary between this rule and O5, noted and deliberately not fixed. `MA` and `Ma` alone report `given-or-family`; `PhD` alone reports `suffix-or-name`. Both are a bare credential with nothing beside it, and what separates them is which gate reads them: `ma` is an AMBIGUOUS acronym, so S2's gate declines to peel it and the word stands as the one name word, which is O5's branch; `phd` is unambiguous, so it peels to suffix, leaves no name piece, and reaches the bare-suffix carve-out, which is this rule's. Two conventions, two kinds, and the reading each name gets is the same either way — the caller sees a report in both cases and only the kind differs. Fixing it would mean one of the two gates changing what it reads, which moves fields for a bundle that moves none. + Nothing moves but the report, and the two rules go in the three 2.x ledgers only. Neither is a copy of a wordlist and both are declared in `_NOT_A_VOCABULARY_COPY`: a member copying TITLES would reach `Dr. Smith`, `Rabbi Cohen` and every other title-plus-surname name, and a member copying the suffix sets would reach `John Smith MD` and `Smith Jr.`, none of which moves. What selects these eight names is a SHAPE — the peel leaving one word, and that word being vocabulary — which no wordlist expresses. + ### W1 — unspaced CJK division - 2026-07-27 #271 (decision; shipped in 2.1.0 via PR #294) — Korean division ships as a default: the census surname list is closed, hangul is self-selecting (a hangul entry can only match hangul text), and being unsplit is recoverable while a wrong split is not — which is also why an unrecognized name stays whole. The filed proposal (#271, 2026-07-07) asked for OPT-IN segmentation for Korean too, "like all localization"; default-on is the later refinement, and the census/self-selecting argument above is what justified promoting Korean past the blanket opt-in stance. @@ -687,7 +710,7 @@ Declined: - 2026-08-27 #445 — the corpus-wide property test is the repair worth copying. tests/v2/test_parser.py's `test_a_maiden_clause_changes_nothing_else` asserts over the corpus that appending " née Jones" adds a maiden name and moves no other field, and M4 falsifies it for fourteen names. Skipping them would have bought a green suite and lost the check; instead the test computes M4's guard from the base parse's tokens and asserts the flip — family takes what `given` held, every other field standing still — so the class that used to be a hole is now the strongest witness the rule has, fourteen names against the six rows cases.py carries. It is also what executes M4's nickname precedence, `'Smitty' Jones Jr.` being one of the fourteen. - 2026-08-27 #445 (review round) — M4 is keyed on the maiden NAME, not on a marker, and the statement was corrected to say so after two reviewers measured the same defect independently. The guard tests for a token in the MAIDEN role, and M1's caller-configured pair produces one with no marker anywhere: under `Policy(maiden_delimiters=frozenset({("(", ")")}))`, `Smith (Jones)` reads family 'Smith', maiden 'Jones', and so do M1's own boundary examples `Smith (Nee)` and `Smith (z domu)`. The BEHAVIOUR is right — the rationale transfers to a declared pair without a word of change, since what announces a former surname is the clause, not the vocabulary that marked it — so the fix was to the rule, which had said "a recognized maiden marker that takes its name and leaves exactly one name word" and now says "a maiden name standing beside exactly one name word". That covers M1, M2 and M3 uniformly, M1 is in M4's `interacts:` and M4 in M1's, and the configured path has an example line of its own. The general lesson is the one this bundle keeps teaching from a new angle: a statement written from the case that prompted the change describes that case, not the code, and only measuring the OTHER paths into the same guard finds the difference. - 2026-08-27 #445 (review round) — a known gap recorded rather than closed, the way H1 carries its own. M4's guard counts GIVEN tokens; H1 counts nothing at all (it tests which roles are unoccupied). So a name word another rule has joined counts as several here and M4 declines where H1 fires: `Dr. Dean of Chemistry` reads family 'Dean of Chemistry' while `Dean of Chemistry née Jones` keeps given 'Dean of Chemistry', the connective join (P3) having left three GIVEN tokens. rules.md#P3 says the joined part is one name word wherever another rule counts them, so the two siblings genuinely disagree and this is a gap rather than a boundary. Not closed here: widening the count to units moves zero corpus names (measured), and a behaviour change nobody has approved does not belong in a branch whose blast radius was settled. The sibling claim in the code comment and in this entry is narrowed to match — sibling except in what it counts. -- 2026-08-27 #445 — the relationship to #449, briefly, because the two look like the same question and are not. #449 asks whether a lone name word with nothing to decide it should REPORT an ambiguity; #445 removes a class of names from that population by giving them something that decides. So #445 reduces #449's reach rather than competing with it, and #449's measurement — a naive "one name word, nothing decided it" condition fires on 19 of 1,085 corpus names, only three or four of them the case it is about — is re-taken after this lands. O5 cites #449 in prose and deliberately carries no `deviates:` marker: that marker asserts an intended output the runner then checks strictly, and #449 has not decided one. +- 2026-08-27 #445 — the relationship to #449, briefly, because the two look like the same question and are not. #449 asks whether a lone name word with nothing to decide it should REPORT an ambiguity; #445 removes a class of names from that population by giving them something that decides. So #445 reduces #449's reach rather than competing with it, and #449's measurement — a naive "one name word, nothing decided it" condition fires on 19 of 1,085 corpus names, only three or four of them the case it is about — is re-taken after this lands. O5 cites #449 in prose and deliberately carries no `deviates:` marker: that marker asserts an intended output the runner then checks strictly, and #449 had not decided one. It has since — 2026-09-07, decisions.md#O5 — and the answer arrived as ordinary example lines rather than as a marker, the report being shipped behavior and not a deviation. The re-measurement this bullet promises is taken there: the naive condition is wider than either number suggests, firing on 143 of the 1123 corpus names, and O5's own population is 27. ### O1 — East Slavic rotation diff --git a/docs/design/rules.md b/docs/design/rules.md index 5f328da4..e863d6ff 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -153,10 +153,20 @@ H4. Rationale: this is a name parser, not a title parser. Handed a Accepted: one name word that is a join (P3) carrying title vocabulary reports `title-or-name` as well, the fork there being whether the title word inside the unit is a title at all. The - measured inputs, `John of Prince` and `Smith and Prince`, are - pinned in the case table rather than here: a join led by a title - word is a title run (H3), so no corpus name reaches the branch. - interacts: H1, H2, H3, S2, O5 · implemented: nameparser/_pipeline/_assign.py + clause reaches EVERY join whose non-leading member is TITLES + vocabulary, not the one word that prompted it: `Smith and King`, + `John and King`, `Smith and Bishop` and `John of Judge` all + report it, as `John of Prince` and `Smith and Prince` do. That + reach is the king/judge/bishop collision set met inside a join, + and the report is the honest answer there rather than a + misfire — a second surname that is also title vocabulary is + exactly the doubt the kind names, and those words stay in the + vocabulary for the addressing forms, which is + decisions.md#vocabulary-collisions' call. + A join LED by a title word is a title run (H3) + instead, so no corpus name reaches the branch at all; the + measured inputs are pinned in the case table rather than here. + history: decisions.md#H4 · interacts: H1, H2, H3, S2, O5 · implemented: nameparser/_pipeline/_assign.py ## Particles & surname prefixes (P) @@ -1127,10 +1137,12 @@ O5. Rationale: O4 reads a name by comparing where its words stand, word nothing else decided carries a `given-or-family` ambiguity naming the field the convention chose. The report is exactly as narrow as the convention, so every rule named above silences it - where it fires, a comma silences it, a script whose own - convention settles the order (W4) silences it, and so does the - word's own claim — a particle, a bound given name, an initial's - shape. A word with no letter or digit in it is no name word and + where it fires, a family comma — a comma with a name segment + after it — silences it, a script whose own convention settles + the order (W4) silences it, and so does the word's own claim — a + particle, a bound given name, an initial's shape. A comma with + nothing after it names no family and silences nothing, so + `سلمان،` and `Smith,` report where `Smith, Andrew` does not. A word with no letter or digit in it is no name word and reports nothing (A2). Where the one name word is a join (P3) carrying title vocabulary, or is itself title vocabulary left standing by the title peel, the doubt reported is `title-or-name` @@ -1152,7 +1164,7 @@ O5. Rationale: O4 reads a name by comparing where its words stand, "Dr. Smith" → ambiguities=() · boundary "Smith née Jones" → ambiguities=() "abd née Jones" → ambiguities=() - interacts: O4, H1, N3, M4 · implemented: nameparser/_pipeline/_assign.py + history: decisions.md#O5 · interacts: O4, H1, N3, M4, H4, W4, P3, A2 · implemented: nameparser/_pipeline/_assign.py ## Scripts & writing systems (W) diff --git a/docs/release_log.rst b/docs/release_log.rst index 5d14c287..c9a7a995 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -30,6 +30,10 @@ Release Log - **Add Bengali honorifics -- the first Bengali vocabulary in the default lexicon (#343):** ``ড``, ``ডঃ``, ``ডক্টর``, ``ডাঃ``, ``ডা``, ``ডাক্তার``, ``শ্রী``, ``শ্রীমতী``, ``জনাব``, ``অধ্যাপক``, ``প্রফেসর``, ``বিচারপতি``, ``মাওলানা``, ``মুফতি``, ``আলহাজ্ব``, ``আলহাজ``, ``মিঃ``, ``মি``, ``মিসেস``, ``মোঃ``, ``মো``, ``মোসাঃ``, ``মোসা``, ``মোছাঃ`` and ``মোছা`` as titles, and ``স্বামী``, ``শ্রীল``, ``গুরু``, ``বাবা`` as given-name titles. ``ড. মুহাম্মদ ইউনূস`` reads title ``ড.``, first ``মুহাম্মদ``, last ``ইউনূস`` -- the vocabulary beats the initial reading -- while real initials are untouched: ``র. কে. নারায়ণ`` is unchanged. ``মোঃ আবদুল করিম`` reads title ``মোঃ``, first ``আবদুল``, last ``করিম``, the mirror of Latin ``Md``; the visarga spelling and the ``মো.`` period spelling both match, and the women's ``মোসাঃ``/``মোসা.`` rides the same pair of entries. ``ঠাকুর`` stays out, being Tagore. Latin transliterations (``Sri``, ``Pandit``, ``Mst``) are not added -- they collide with real given names where the native scripts cannot -- and belong to the opt-in packs of #345 (closes #343) + - **Add AmbiguityKind.GIVEN_OR_FAMILY, reported when a name of one name word had nothing to decide which field it is:** ``parse("Andrew")`` still gives given ``Andrew`` and now says that field was a convention rather than a reading -- one word gives the positional rule nothing to compare, so the library picks the given name under the default order and the family name under a declared family-first one, and ``detail`` names the field it picked. A trailing suffix does not decide it either: ``parse("Smith Jr.")`` reports it too, the suffix being peeled and the convention placing the one name word left. A name something DID decide stays silent -- ``"Dr. Smith"``, ``"Smith née Jones"``, ``"'Smitty' Jones"`` and ``"Smith, Andrew"`` -- and so do ``"abdul"`` and ``"de"``, where the bound given-name and particle vocabularies claimed the word, and ``"J."``, claimed by the initial's own shape. A name whose script settles the order is silent too: ``"毛泽东"`` reads family by convention of the writing system, not of this rule. Twenty-seven names in the differential corpora gain the report, and no field moves anywhere. See the ``O5`` entry of ``docs/design/decisions.md`` (closes #449) + + - **Add AmbiguityKind.TITLE_OR_NAME, reported when an input that is nothing but honorifics had its last word read as the name:** ``parse("Lord Chancellor")`` still gives title ``Lord``, family ``Chancellor``, and now says so -- this is a name parser, not a title parser, so handed a string with no name in it, it reads the last title word as one. ``"The Right Hon. the President of the Queen's Bench Division"`` and ``"His Holiness"`` move the same way, and so does ``"Dr. King"``, ``king`` being title vocabulary for the addressing forms. A title with an ordinary word behind it is silent (``"Dr. Smith"``, ``"King Charles"``), and so is a lone title word: ``parse("Dr.")`` is a title with no name beside it, the peel having taken the whole string, so no word was left standing to be read as a name and nothing was chosen. The same convention on the other vocabulary reports the existing suffix-or-name -- ``parse("Rinpoche")`` gives given ``Rinpoche`` and flags it, as does ``"QC MP"`` -- while ``"Jr."`` is unchanged and unflagged, reading as a title on its shape. Eight names in the differential corpora gain a report, and no field moves. See the ``H4`` entry of ``docs/design/decisions.md`` (closes #491) + * 2.2.0 - August 31, 2026 nameparser 2.2 is a rename plus about thirty parsing fixes. diff --git a/tests/v2/cases.py b/tests/v2/cases.py index fad20e73..d0454b4a 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -2340,8 +2340,20 @@ def _check_cjk_shape_purity(self) -> None: notes="rules.md#O5's exception: the one name unit is a join " "(P3) carrying title vocabulary, so the doubt is " "whether `Prince` is a title rather than which field " - "the unit takes -- the one input measured to reach " + "the unit takes -- one of the inputs measured to reach " "that branch, `prince` being in TITLES"), + Case("lone_joined_unit_carrying_collision_set_title_vocabulary", + "Smith and King", {"given": "Smith and King"}, + ambiguities=("title-or-name",), + classification="feat(#491)", + notes="the clause reaches every join whose non-leading member " + "is TITLES vocabulary, not just `prince`: `John and " + "King`, `Smith and Bishop` and `John of Judge` report " + "it too. That is the king/judge/bishop collision set " + "(decisions.md#vocabulary-collisions) met inside a " + "join, and the report is deliberate -- a second surname " + "that is also title vocabulary is the doubt the kind " + "names"), Case("lone_joined_unit_led_by_a_title_is_a_title_run", "Prince of Wales", {"title": "Prince of Wales"}, classification="parity", From ed06654c5684483fd3a83832756522693ecdf977 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Tue, 8 Sep 2026 02:21:38 -0700 Subject: [PATCH 4/5] review round: PR #518 findings Four review agents on the whole PR. Code: a title after a family comma decided the lone word's field and the convention still reported -- "John V, Dr." named given where H1 then wrote family -- so the comma path's title now silences it; H4's join half is hoisted beside its lone-word half, out from under O5's field-deciding clauses, so a maiden marker or a claimed word no longer silences a title-or-name doubt; Order becomes EffectiveOrder. Two corpus names move, both reached by that hoist -- "Attorney General of Minnesota" and "Deputy Secretary of State", a title peeled in front of a joined unit that stands last, so H3 cannot chain it -- and they take a third #491 ledger rule. No role moves anywhere. Docs: the decider list in GIVEN_OR_FAMILY's docstring, the assign comment and the O5 entry no longer claim a nickname or a bare comma silences the report (the PR's own examples report on both); the segments-clause reason is stated as inert over the corpus, not by construction; TITLE_OR_NAME's docstring says which shape names a field. Tests: the three detail strings are pinned across all three orders; the initial-shape silencer, the family-comma title path, the hoisted join reach and the script_orders positive controls gain rows; the feat(#491) probes wall an anchor drop. Co-Authored-By: Claude Fable 5.1 --- docs/design/decisions.md | 10 +- docs/design/rules.md | 44 +++- docs/release_log.rst | 2 +- nameparser/_pipeline/_assign.py | 239 +++++++++++-------- nameparser/_types.py | 23 +- tests/v2/cases.py | 80 ++++++- tests/v2/pipeline/test_assign.py | 35 +++ tests/v2/test_facade_cases.py | 6 + tests/v2/test_ledger_guards.py | 48 +++- tools/differential/expected_since_2.0.0.toml | 16 ++ tools/differential/expected_since_2.1.0.toml | 16 ++ tools/differential/expected_since_2.2.0.toml | 16 ++ 12 files changed, 405 insertions(+), 130 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index c92b7d17..32c6d96d 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -342,15 +342,17 @@ Declined (rc1 arc; the full argument is AGENTS.md's gotcha): ### O5 — the lone name word's convention, and reporting it - 2026-09-07 #449 — the convention is REPORTED, as `given-or-family`, with `detail` naming the field it chose. rules.md#O5 has said since #445 that the one-word reading is a guess fixed in advance rather than a determination; what #449 asked is what the library should SAY about it, and the answer is the one A1 gives everywhere else — a caller can only act on doubt that is reported. `detail` names the field rather than the kind naming it, because the field follows the read order and the kind does not: that is the PARTICLE_OR_GIVEN precedent, recorded at that kind's docstring. - What the measurement settled was the POPULATION, not the principle. The naive condition — one name piece placed, nothing else looked at — fires on 143 of the 1123 distinct corpus names, because a title peel, a nickname, a maiden clause, a trailing credential run or a script order can each leave exactly one piece standing where the convention decided nothing. The 2026-08-27 #445 entry below records "19 of 1,085" for a differently written naive condition on a smaller corpus; both numbers are naive and neither is O5's. Guarded on O5's own carve-out list the population is 27 corpus names, 12 contract and 15 radar, and the other 116 report nothing at this site, 6 of them reporting `title-or-name` instead (H4's branch is ahead of O5's at the same line). A first draft wrote 110, which is the subtraction done wrong. Recompute by wrapping `_name_positions` with a recorder that fires at `count == 1`, parsing the deduped `corpus*.jsonl` glob through it, and comparing that set against the names whose parse reports the kind; measured 2026-09-08 on this branch. The guard itself is the code, which is where it should be read. + What the measurement settled was the POPULATION, not the principle. The naive condition — one name piece placed, nothing else looked at — fires on 143 of the 1123 distinct corpus names, because a title peel, a nickname, a maiden clause, a trailing credential run or a script order can each leave exactly one piece standing where the convention decided nothing. The 2026-08-27 #445 entry below records "19 of 1,085" for a differently written naive condition on a smaller corpus; both numbers are naive and neither is O5's. Guarded on O5's own carve-out list the population is 27 corpus names, 12 contract and 15 radar, and the other 116 report no `given-or-family`, 8 of them reporting `title-or-name` instead (H4's two branches are ahead of O5's at the same line). A first draft wrote 110, which is the subtraction done wrong. Recompute by wrapping `_name_positions` with a recorder that fires at `count == 1`, parsing the deduped `corpus*.jsonl` glob through it, and comparing that set against the names whose parse reports the kind; measured 2026-09-08 on this branch. The guard itself is the code, which is where it should be read. Two conditions are in the guard that O5's statement had not named, and both are in it now. A script whose own convention settles the order decided the reading, so W4 outranks this the way H1 does — that is what keeps 毛泽东, 高橋みなみ, 씨 and the rest of the script-ordered names out. Asking that question needed a shape change: `_effective_order` used to return the roles triple alone, and a caller comparing that triple against the declared `name_order` cannot tell a script rule DECIDING from a declaration standing unopposed — under a declared family-first order a Han name's W4 entry and the declaration agree. It now returns an `Order(roles, by_script)` and the emitter reads the flag. The second condition is content: a piece with no letter or digit in it is no name word, A2 empties such an input, and a convention report on an empty name would describe a reading nobody got — so `parse("(")` keeps its unbalanced-delimiter report and gains nothing. Five CJK-bearing names DO report, and none of them is a script rule deciding. Three are a script rule DECLINING: マイケル (W4 gives katakana no order of its own, a wholly-katakana name being read as a transcription), 王·Smith (T3's interpunct suppresses the script_orders lookup whole) and 田中、太郎 (the comma sits inside the token, so the text resolves to no single script). Two are a glued honorific peeled off a Latin stem — Andersonさん and Anderson선생님 — where what is left is Latin and has no script order at all. Recorded because "no CJK name reports" was the expectation going in, and it is wrong in a way that is right: where the script rules decline, what is left IS the one-word convention. - Four conditions were drafted into the guard and none of them shipped, but they were dropped for TWO different reasons, and a first draft of this entry flattened both into "inert". Each was measured the same way: restore that one clause on the O5 branch of a `git archive` copy of this commit, reparse the deduped `corpus*.jsonl` glob, and count which of the 27 lose the report. Two are genuinely INERT, silencing ZERO names (measured 2026-09-08). `len(state.segments) == 1`: the family-comma path names the family before segment 0 is read positionally, so the site is never called with one piece under a comma. `not flagged`: a split credential decides the field no more than a whole one does. The general shape is the one this file keeps recording — a guard written from the cases that prompted it carries terms the code already excludes — and the instrument that finds them is mutation, not reading. + Four conditions were drafted into the guard and none of them shipped, but they were dropped for TWO different reasons, and a first draft of this entry flattened both into "inert". Each was measured the same way: restore that one clause on the O5 branch of a `git archive` copy of this commit, reparse the deduped `corpus*.jsonl` glob, and count which of the 27 lose the report. Two are genuinely INERT, silencing ZERO names (measured 2026-09-08, and again after the 2026-09-08 review round below). `len(state.segments) == 1` is inert OVER THE CORPUS rather than by construction, and a first draft of this line gave the construction reason, which is false: a family comma that names a family does read segment 0 wholly as the family and never positionally, but a comma followed by NO name word hands segment 0 back to the positional read, and `Smith V, Dr.` reaches this very site that way. What silences it is not the clause but the title in segment 1 — after the review round below, which is what tells the read that a title stands there. `not flagged`: a split credential decides the field no more than a whole one does. The general shape is the one this file keeps recording — a guard written from the cases that prompted it carries terms the code already excludes — and the instrument that finds them is mutation, not reading. The other two were dropped DELIBERATELY, and the same mutation prices each. `not suffix_pieces` silences NINE of the 27 — `Smith Jr.`, `'Smitty' Jones Jr.`, `Carod i`, `Donald mc`, `Jack M.A.`, `John V`, `Mohamad X`, `Andersonさん`, `Anderson선생님` — and `not has_nickname` silences ONE, `'Smitty' Jones Jr.`. Both were dropped on one principle: a suffix or a nickname standing beside the lone name word decides nothing about that word's FIELD, which is the only question this report is about. `Smith Jr.` and `'Smitty' Jones Jr.` are S2's peel taking the suffix and the convention placing what is left, so they report. A first draft justified the nickname clause with "N3 returns before this site", and that is false as written: `_assign.py`'s N3 branch tests `len(pieces) == 1 and len(rest) == 1 and has_nickname`, so it returns only where the segment holds ONE piece, and `'Smitty' Jones Jr.` holds two and reaches this line. What N3 takes is the nickname-plus-one-word shape, not every name a nickname stands in. Not reported, deliberately: `abdul`, where bound-given vocabulary claimed the word; `de`, where a lone particle's reading is P4's; and `J.`, where the initial's shape is the claim. Those are M4's `_NEVER_FLIPPED` pair plus the particle, read off the tags classify already recorded rather than off a predicate of this emitter's own. Also not reported: `Sir John`, `'Smitty' Jones` and `abd née Jones`, which O5's example block lists because the convention is what H1, N3 and M4 each left behind on them. The rule's SCOPE and the report's scope differ there, and the difference is stated in the rule rather than left to be discovered: the report is narrowed to an input nothing else stands in, because a report on a name where a title, a nickname or a maiden name is visible in the output tells the caller less than the output already does. The report is order-independent, which is not the same as the FIELD being order-independent: `Andrew` reports under the default order with `detail` naming given and under FAMILY_FIRST with `detail` naming family. Measured over the whole corpus under the default order, FAMILY_FIRST and FAMILY_FIRST_GIVEN_LAST, the set of names reporting this kind is the same 27 in all three; the only names whose reported kinds move with the order at all are four pre-existing `particle-or-given` reports. Nothing moves but the report. Every role is identical before and after — measured by parsing all 1123 corpus names under all three orders against a `git archive` of the parent commit, 3369 parses, zero role differences — and the three 2.x ledgers classify the 27 names on `_ambiguities` alone; `_ambiguities` cannot enter a diff below baseline 2.0, so `expected_since_1.4.0.toml` is untouched and its 368 intentional diffs are unchanged. SIX ledger rules rather than one, because an alternation carrying a script-classified member is claimed by the honorific pin in tests/v2/test_ledger_guards.py, which would demand it be a hand copy of GLUED_HONORIFICS: the twenty-two Latin-and-Arabic names ride one alternation, declared in `_NOT_A_VOCABULARY_COPY`, and the five CJK-bearing names take a literal-anchored rule each, which is also where their arguments differ. The three ledger copies are NOT byte-identical, which the drafting expected them to be: in `expected_since_2.0.0.toml` alone the two glued-honorific rules declare `["_ambiguities", "given", "suffix"]`, because at that baseline #308's peel has not shipped and its diff arrives in the same rule as this report, and the gate's over-declared check refuses a rule declaring a role no diff it explains moves. +- 2026-09-08 #518 review round (the entry above ran into a second day; its own measurements are dated 2026-09-08 under a 2026-09-07 header) — a title standing after a family comma now silences the report. `John V, Dr.` is the shape: a comma with no name word after it hands segment 0 back to the positional read, the leading-title peel counts only what stands in segment 0, so `n` is 0 and the report named `given` for a word H1 then wrote to `family` — the caller was told the convention chose a field the output does not show. `_assign_main` takes a keyword-only `titled`, which the family-comma call fills from the gate's own reading (a False in it is a title, not a suffix), and the field-deciding emitters require `not titled`. Measured on this branch: `John V, Dr.`, `Smith V, Prince` and `Smith V, Dr.` keep every role and lose the `given-or-family` report, keeping the roman numeral's; `John V, Sir` does too, and it is the one worth naming — a given-name title leaves the word in `given`, which is the field the report would have named, so the report and the reading agreeing is what a silenced clause looks like when the rule that decided it happens to agree. Zero corpus names move: no corpus name reaches the site under a comma at all, which is also why the `len(state.segments) == 1` clause above measures inert. + ### O4 — positional assignment and declared order - 2026-08-07 #83 — romanized Chinese order is answered by DECLARATION, not detection: pinyin carries no signal and diaspora makes locale no guide (the issue's own 2019/2021 conclusions); Policy(name_order=FAMILY_FIRST) is the answer the thread wanted. @@ -407,7 +409,9 @@ Open: [#316](https://github.com/derek73/python-nameparser/issues/316) what a tra A refinement of Derek's, made after the population was measured and widening the rule past the all-titles shape it was drafted for: a lone name word that is a JOIN (P3) carrying title vocabulary reports `title-or-name` too, the fork there being whether the title word inside the unit is a title at all rather than which field the unit takes. `John of Prince` and `Smith and Prince` are the measured inputs; no corpus name reaches that branch, because a join LED by a title word is chained into a title run by H3 (`Prince of Wales`), so the two are pinned as case rows rather than as rules.md examples. It sits on O5's branch and takes precedence there, which is why O5's statement says a title silences THIS kind and not every report at the site. The suffix half is the same argument on the other vocabulary and needed no new kind. An input whose every word is post-nominal vocabulary reads its first word as a name — assign's "everything suffix-shaped after titles: first one is the name" carve-out — and that is the doubt SUFFIX_OR_NAME already names, so it reports that. `Rinpoche` and `QC MP` are the corpus names; `PhD`, `MBA` and `III` are the same shape. `Jr.` alone is NOT the shape at all: H2's opening-abbreviation rule reads it as a title before the suffix vocabulary is consulted, which is that rule's stated precedence and is recorded here because the expectation going in was that `Jr.` alone read as a name. The guard carries two exclusions of its own. A maiden name beside the credential says the input is not post-nominal vocabulary and nothing else, so `abd née Jones` is out for the reason M4 keeps it out of O5's report. And the report is scoped to names no script order placed, so a lone glued CJK honorific — さん, 씨, 선생님 — reports nothing: that is the same shape read through the glued-honorific rules (W2, #271/#308) and the script's own order, and whether those readings should report is left to the arc that revisits them rather than settled here. An asymmetry on the boundary between this rule and O5, noted and deliberately not fixed. `MA` and `Ma` alone report `given-or-family`; `PhD` alone reports `suffix-or-name`. Both are a bare credential with nothing beside it, and what separates them is which gate reads them: `ma` is an AMBIGUOUS acronym, so S2's gate declines to peel it and the word stands as the one name word, which is O5's branch; `phd` is unambiguous, so it peels to suffix, leaves no name piece, and reaches the bare-suffix carve-out, which is this rule's. Two conventions, two kinds, and the reading each name gets is the same either way — the caller sees a report in both cases and only the kind differs. Fixing it would mean one of the two gates changing what it reads, which moves fields for a bundle that moves none. - Nothing moves but the report, and the two rules go in the three 2.x ledgers only. Neither is a copy of a wordlist and both are declared in `_NOT_A_VOCABULARY_COPY`: a member copying TITLES would reach `Dr. Smith`, `Rabbi Cohen` and every other title-plus-surname name, and a member copying the suffix sets would reach `John Smith MD` and `Smith Jr.`, none of which moves. What selects these eight names is a SHAPE — the peel leaving one word, and that word being vocabulary — which no wordlist expresses. + Nothing moves but the report, and the rules go in the three 2.x ledgers only — two of them at first and a third after the 2026-09-08 round below, which is where the ten names and the join clause's own rule come from. None is a copy of a wordlist and all three are declared in `_NOT_A_VOCABULARY_COPY`: a member copying TITLES would reach `Dr. Smith`, `Rabbi Cohen` and every other title-plus-surname name, and a member copying the suffix sets would reach `John Smith MD` and `Smith Jr.`, none of which moves. What selects these ten names is a SHAPE — the peel leaving one unit, and vocabulary standing in it — which no wordlist expresses. + +- 2026-09-08 #518 review round — the join clause is HOISTED beside the peel clause instead of sitting under O5's. It had been written inside O5's `n == 0` leg, so a leading title, a maiden marker or a vocabulary claim on the word silenced it — and every one of those decides which FIELD the unit takes, which is not what this clause asks. Measured before and after on this branch: `Lord Chancellor née Jones` and `Dr. King née Jones` (maiden), `Dr. Smith and Prince` and `Mr. John and King` (a peeled title), `van and Prince` and `J. and Prince` (a claimed word) were all silent and all now report `title-or-name`, with no role moving on any of them. The maiden pair takes the peel half and the other four the join half. TWO CORPUS NAMES MOVE, which the drafting expected to be zero: `Attorney General of Minnesota` and `Deputy Secretary of State`, both a title peeled in front of a joined unit that stands last — and a title needs a following piece, so H3 cannot chain the join into a title run the way it chains `Prince of Wales`. "No corpus name reaches the branch" was true only of the guarded version, and the claim is corrected in rules.md#H4 as well. They take a third `feat(#491)` ledger rule of their own rather than joining the all-titles alternation, whose issue line describes an all-titles input, which neither of these is. Also this round: the emitter comment's carve-out list no longer names "a group-flagged credential" (no such condition is in the code, and `phd_split` pins the opposite), and the `Order` NamedTuple is `EffectiveOrder` with field `order`, `roles` having been the name of two different things three lines apart. ### W1 — unspaced CJK division diff --git a/docs/design/rules.md b/docs/design/rules.md index e863d6ff..304999d9 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -141,6 +141,7 @@ H4. Rationale: this is a name parser, not a title parser. Handed a "Rinpoche" → ambiguities=("suffix-or-name",) "QC MP" → given="QC" "Jr." → title="Jr." + "Jr." → ambiguities=() Accepted: `Jr.` is post-nominal vocabulary and still reads as a title, H2's opening-abbreviation shape outranking the vocabulary where it fires — so the suffix half never sees a dotted lone @@ -149,10 +150,20 @@ H4. Rationale: this is a name parser, not a title parser. Handed a placed, so a lone CJK honorific written by itself — さん, 씨, 선생님 — reports nothing. The same shape, read through the glued-honorific rules (W2) and the script's own order, and left - to the arc that revisits those readings. - Accepted: one name word that is a join (P3) carrying title + to the arc that revisits those readings. That silence is a SCOPE + on the script rule having placed the name, not a claim about the + honorific: a caller who declares no script orders at all gets the + report on the same input, since nothing then resolves the order + and the word reaches the carve-out as any other lone credential + does. The pair is pinned in the case table rather than here, + a policy that empties the script table having no example spelling + in this document. + Accepted: one name UNIT of more than one word carrying title vocabulary reports `title-or-name` as well, the fork there being - whether the title word inside the unit is a title at all. The + whether the title word inside the unit is a title at all. Usually + a join (P3), and a particle chain (P4) is the same shape and + reports too — `St St née` reads family "St née" with `st` title + vocabulary inside it. The clause reaches EVERY join whose non-leading member is TITLES vocabulary, not the one word that prompted it: `Smith and King`, `John and King`, `Smith and Bishop` and `John of Judge` all @@ -163,9 +174,17 @@ H4. Rationale: this is a name parser, not a title parser. Handed a exactly the doubt the kind names, and those words stay in the vocabulary for the addressing forms, which is decisions.md#vocabulary-collisions' call. - A join LED by a title word is a title run (H3) - instead, so no corpus name reaches the branch at all; the - measured inputs are pinned in the case table rather than here. + A join OPENING the name with a title word is a title run (H3) + instead. Behind a peeled title the join is not: a title needs a + following piece and the join is the last one, so `Attorney General + of Minnesota` and `Deputy Secretary of State` keep `Attorney` and + `Deputy` as the title and report the rest — the two corpus names + the clause reaches, and `General` or `Secretary` being a title is + exactly the fork. Nothing in front of the join silences the + clause, a title, a maiden name or a claimed word each deciding + which FIELD the unit takes and none of them whether the word + inside it is a title. The other measured inputs are pinned in the + case table rather than here. history: decisions.md#H4 · interacts: H1, H2, H3, S2, O5 · implemented: nameparser/_pipeline/_assign.py ## Particles & surname prefixes (P) @@ -1142,10 +1161,14 @@ O5. Rationale: O4 reads a name by comparing where its words stand, the order (W4) silences it, and so does the word's own claim — a particle, a bound given name, an initial's shape. A comma with nothing after it names no family and silences nothing, so - `سلمان،` and `Smith,` report where `Smith, Andrew` does not. A word with no letter or digit in it is no name word and - reports nothing (A2). Where the one name word is a join (P3) - carrying title vocabulary, or is itself title vocabulary left - standing by the title peel, the doubt reported is `title-or-name` + `سلمان،` and `Smith,` report where `Smith, Andrew` does not. A + comma with only a TITLE after it names no family either, but the + title still decides the field wherever it stands, so `John V, Dr.` + is silent for H1's reason and not the comma's. A word with no + letter or digit in it is no name word and + reports nothing (A2). Where the one name unit carries title + vocabulary in a word beside the one it places, or is itself title + vocabulary left standing by the title peel, the doubt reported is `title-or-name` instead, which H4 states — so a title silences THIS kind, not every report at the site. "Smith" → given="Smith" @@ -1162,6 +1185,7 @@ O5. Rationale: O4 reads a name by comparing where its words stand, "'Smitty' Jones Jr." → ambiguities=("given-or-family",) "Smith Jr." → ambiguities=("given-or-family",) "Dr. Smith" → ambiguities=() · boundary + "Sir John" → ambiguities=() "Smith née Jones" → ambiguities=() "abd née Jones" → ambiguities=() history: decisions.md#O5 · interacts: O4, H1, N3, M4, H4, W4, P3, A2 · implemented: nameparser/_pipeline/_assign.py diff --git a/docs/release_log.rst b/docs/release_log.rst index c9a7a995..11dfb45f 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -32,7 +32,7 @@ Release Log - **Add AmbiguityKind.GIVEN_OR_FAMILY, reported when a name of one name word had nothing to decide which field it is:** ``parse("Andrew")`` still gives given ``Andrew`` and now says that field was a convention rather than a reading -- one word gives the positional rule nothing to compare, so the library picks the given name under the default order and the family name under a declared family-first one, and ``detail`` names the field it picked. A trailing suffix does not decide it either: ``parse("Smith Jr.")`` reports it too, the suffix being peeled and the convention placing the one name word left. A name something DID decide stays silent -- ``"Dr. Smith"``, ``"Smith née Jones"``, ``"'Smitty' Jones"`` and ``"Smith, Andrew"`` -- and so do ``"abdul"`` and ``"de"``, where the bound given-name and particle vocabularies claimed the word, and ``"J."``, claimed by the initial's own shape. A name whose script settles the order is silent too: ``"毛泽东"`` reads family by convention of the writing system, not of this rule. Twenty-seven names in the differential corpora gain the report, and no field moves anywhere. See the ``O5`` entry of ``docs/design/decisions.md`` (closes #449) - - **Add AmbiguityKind.TITLE_OR_NAME, reported when an input that is nothing but honorifics had its last word read as the name:** ``parse("Lord Chancellor")`` still gives title ``Lord``, family ``Chancellor``, and now says so -- this is a name parser, not a title parser, so handed a string with no name in it, it reads the last title word as one. ``"The Right Hon. the President of the Queen's Bench Division"`` and ``"His Holiness"`` move the same way, and so does ``"Dr. King"``, ``king`` being title vocabulary for the addressing forms. A title with an ordinary word behind it is silent (``"Dr. Smith"``, ``"King Charles"``), and so is a lone title word: ``parse("Dr.")`` is a title with no name beside it, the peel having taken the whole string, so no word was left standing to be read as a name and nothing was chosen. The same convention on the other vocabulary reports the existing suffix-or-name -- ``parse("Rinpoche")`` gives given ``Rinpoche`` and flags it, as does ``"QC MP"`` -- while ``"Jr."`` is unchanged and unflagged, reading as a title on its shape. Eight names in the differential corpora gain a report, and no field moves. See the ``H4`` entry of ``docs/design/decisions.md`` (closes #491) + - **Add AmbiguityKind.TITLE_OR_NAME, reported when an input that is nothing but honorifics had its last word read as the name:** ``parse("Lord Chancellor")`` still gives title ``Lord``, family ``Chancellor``, and now says so -- this is a name parser, not a title parser, so handed a string with no name in it, it reads the last title word as one. ``"The Right Hon. the President of the Queen's Bench Division"`` and ``"His Holiness"`` move the same way, and so does ``"Dr. King"``, ``king`` being title vocabulary for the addressing forms. A title with an ordinary word behind it is silent (``"Dr. Smith"``, ``"King Charles"``), and so is a lone title word: ``parse("Dr.")`` is a title with no name beside it, the peel having taken the whole string, so no word was left standing to be read as a name and nothing was chosen. The same convention on the other vocabulary reports the existing suffix-or-name -- ``parse("Rinpoche")`` gives given ``Rinpoche`` and flags it, as does ``"QC MP"`` -- while ``"Jr."`` is unchanged and unflagged, reading as a title on its shape. The same doubt inside a joined unit reports too -- ``parse("Attorney General of Minnesota")`` reads title ``Attorney``, family ``General of Minnesota``, and whether ``General`` is a title is the fork. Ten names in the differential corpora gain a report, and no field moves. See the ``H4`` entry of ``docs/design/decisions.md`` (closes #491) * 2.2.0 - August 31, 2026 diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index 2dbd3e6d..cbf3f3ff 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -33,6 +33,15 @@ particles_ambiguous token with more pieces following ("Van Johnson", and since #367 "Dr. Van Johnson" too, a title no longer displacing the particle out of that position) -- whatever role name_order assigns. +Emits SUFFIX_OR_NAME at three sites: the trailing roman numeral, each +ambiguous acronym the trailing peel had to resolve, and the bare-suffix +carve-out where an input that is nothing but post-nominal vocabulary +gets its first word made into the name (H4's suffix half, #491). And +at the one site that places a LONE name word, GIVEN_OR_FAMILY for the +field the convention picked (O5, #449) and TITLE_OR_NAME for the two +shapes where the doubt is whether a word is a title instead (H4, +#491): the peel leaving one title-vocabulary word standing, and a +joined unit carrying title vocabulary. """ from __future__ import annotations @@ -87,17 +96,17 @@ def _peel_leading_titles(pieces: tuple[tuple[int, ...], ...], return n -class Order(NamedTuple): - """What _effective_order made of a name's scripts. `roles` is the +class EffectiveOrder(NamedTuple): + """What _effective_order made of a name's scripts. `order` is the order the positional read uses; `by_script` says a script_orders - entry RESOLVED it, which is not the same as `roles` happening to + entry RESOLVED it, which is not the same as `order` happening to equal the declared name_order -- under a declared family-first order a Han name's W4 entry and the declaration agree, and only this flag distinguishes the script rule DECIDING the reading from the caller's declaration standing unopposed. O5's convention report reads it (#449).""" - roles: tuple[Role, Role, Role] + order: tuple[Role, Role, Role] by_script: bool @@ -108,7 +117,7 @@ class Order(NamedTuple): def _effective_order(policy: Policy, pieces: list[tuple[int, ...]], tokens: list[WorkToken], - *, dot_divided: bool) -> Order: + *, dot_divided: bool) -> EffectiveOrder: """script_orders resolution (#271): when every name piece is written wholly in ONE script that has an entry, that script's order governs the positional read; anything else -- Latin, mixed @@ -134,11 +143,11 @@ def _effective_order(policy: Policy, resolves the SCRIPT for a single token. This function calls that one per token below. - Returns an Order: the roles triple, and `by_script` set only on + Returns an EffectiveOrder: the order triple, and `by_script` set only on the one path where an entry answered. Every fallback below is a script rule DECLINING, and reports it as such. """ - declared = Order(policy.name_order, by_script=False) + declared = EffectiveOrder(policy.name_order, by_script=False) # #298 transcription marker -- see the docstring; codepoint-scoped # (only U+00B7 records; decisions.md#T3) if dot_divided: @@ -164,7 +173,7 @@ def _effective_order(policy: Policy, return declared for script, order in policy.script_orders: if script is resolved: - return Order(order, by_script=True) + return EffectiveOrder(order, by_script=True) # the resolved script has no entry: the declaration stands, and # nothing about the writing system decided the reading return declared @@ -204,9 +213,19 @@ def _name_positions(order: tuple[Role, Role, Role], def _assign_main(seg_idx: int, state: ParseState, tokens: list[WorkToken], ambiguities: list[PendingAmbiguity], + *, titled: bool = False, ) -> tuple[Role, Role, Role] | None: """Returns the order the positional read used, for ParseState.order - -- None on every path that returns before resolving one.""" + -- None on every path that returns before resolving one. + + `titled` is the one fact about ANOTHER segment this read needs: on + the family-comma path a title stands after the comma ('John V, + Dr.'), the caller sends segment 0 here to be read positionally, + and the leading-title peel below counts nothing -- so `n` cannot + see the title and O5's report would name a field H1 then rewrites. + Only the field-deciding emitters read it; H4's two halves ask + whether a WORD is a title, which a title elsewhere does not + answer.""" pieces = state.pieces[seg_idx] ptags = state.piece_tags[seg_idx] has_nickname = any(t.role is Role.NICKNAME for t in tokens) @@ -264,7 +283,7 @@ def _assign_main(seg_idx: int, state: ParseState, resolved = _effective_order(state.policy, [pieces[i] for i in name_pieces], tokens, dot_divided=bool(state.interpunct_offsets)) - order = resolved.roles + order = resolved.order roles = _name_positions(order, len(name_pieces)) for pos, piece_idx in enumerate(name_pieces): _set_roles(tokens, pieces[piece_idx], roles[pos]) @@ -272,22 +291,25 @@ def _assign_main(seg_idx: int, state: ParseState, _set_roles(tokens, pieces[piece_idx], Role.SUFFIX) # rules.md#H4: "an input whose every word is post-nominal # vocabulary reads its first word as a name and reports - # `suffix-or-name`" -- reported at the carve-out above that - # applies it (mechanisms.md#AMBIGUITY-AT-THE-DECISION-SITE), and - # only the word made into a name reports. `n == 0` is what - # keeps a titled run out -- 'Dr King Jr' and 'MD DDS' peel a - # title first, and after a title the reading is H1's rather than - # this convention's. A maiden name beside the credential says the - # input is not post-nominal vocabulary and nothing else, so 'abd - # née Jones' is out for the same reason M4 keeps it out of O5's - # report below. `resolved.by_script` is a SCOPE line rather than a - # claim that something else decided: a lone CJK honorific ('さん', - # '씨', '선생님') is the same shape read through the glued-honorific - # rules and the script's own order (W2, #271/#308), and whether - # those readings should report is left to the arc that revisits - # them rather than settled here. The role comes off the token for - # the reason stated at the particle emitter below. - if (bare_suffix and n == 0 and not resolved.by_script + # `suffix-or-name`" (history: decisions.md#H4) -- reported at the + # carve-out above that applies it + # (mechanisms.md#AMBIGUITY-AT-THE-DECISION-SITE), and only the + # word made into a name reports. `n == 0` is what keeps a titled + # run out -- 'Dr King Jr' and 'MD DDS' peel a title first, and + # after a title the reading is H1's rather than this convention's; + # `titled` is the same exclusion for the title the peel could not + # see, standing after a family comma. A maiden name beside the + # credential says the input is not post-nominal vocabulary and + # nothing else, so 'abd née Jones' is out for the same reason M4 + # keeps it out of O5's report below. `resolved.by_script` is a + # SCOPE line rather than a claim that something else decided: a + # lone CJK honorific ('さん', '씨', '선생님') is the same shape read + # through the glued-honorific rules and the script's own order + # (W2, #271/#308), and whether those readings should report is + # left to the arc that revisits them rather than settled here. The + # role comes off the token for the reason stated at the particle + # emitter below. + if (bare_suffix and n == 0 and not titled and not resolved.by_script and not any(t.role is Role.MAIDEN for t in tokens)): head = pieces[name_pieces[0]] token = tokens[head[0]] @@ -301,57 +323,47 @@ def _assign_main(seg_idx: int, state: ParseState, tuple(head))) # The site that places a lone name word, and so the site that # reports both conventions which turn on one - # (mechanisms.md#AMBIGUITY-AT-THE-DECISION-SITE): H4's below, - # then O5's. The outer guard holds only what silences BOTH -- a - # maiden name (M4 decided it) and a script whose order convention - # settles the reading (W4 decided it too, which the rule states - # without naming), the latter `resolved.by_script` rather than a - # comparison against name_order, a declared family-first order - # agreeing with a Han name's entry being agreement and not - # authorship. The rest of O5's carve-out list -- a leading title, - # a group-flagged credential, the word's own claim -- sits on - # O5's own branch, because a leading title does not silence H4: - # it is the shape H4 is about. The two shapes O5 also names never - # reach this line: a nickname beside a lone name word is N3's and - # returns above, and a family comma names the family before - # segment 0 is read positionally, so neither needs a clause here. - # A suffix beside the word is NOT such a shape -- 'Smith Jr.' and - # "'Smitty' Jones Jr." are the convention placing a lone name word, - # which is why S2's peel does not silence it. The count comes off - # the PEEL rather than off name_pieces, which is what leaves the - # bare-suffix carve-out above (peeled.names == 0, the first - # post-nominal read as the name for want of anything else) out of - # both branches: that reading is a different convention, and - # reporting it is H4's suffix half above. The role comes off the - # token for the reason stated at the particle emitter below. - if (peeled.names == 1 and not resolved.by_script - and not any(t.role is Role.MAIDEN for t in tokens)): + # (mechanisms.md#AMBIGUITY-AT-THE-DECISION-SITE): H4's two halves + # below, then O5's. The outer guard holds only what silences ALL + # THREE -- a script whose order convention settles the reading + # (W4 decided it, which O5's rule states without naming W4), read + # as `resolved.by_script` rather than as a comparison against + # name_order, a declared family-first order agreeing with a Han + # name's entry being agreement and not authorship. Everything + # else on O5's carve-out list decides the FIELD and not whether + # the word is a title, so it sits on O5's own branch: a leading + # title (H1's reading then), a maiden name (M4's), the word's own + # claim, and `titled` for the title a family comma put where the + # peel cannot count it. A suffix beside the word is NOT such a + # shape -- 'Smith Jr.' and "'Smitty' Jones Jr." are the convention + # placing a lone name word, which is why S2's peel does not + # silence it. The count comes off the PEEL rather than off + # name_pieces, which is what leaves the bare-suffix carve-out + # above (peeled.names == 0, the first post-nominal read as the + # name for want of anything else) out of all three branches: that + # reading is a different convention, and reporting it is H4's + # suffix half above. The role comes off the token for the reason + # stated at the particle emitter below. + if peeled.names == 1 and not resolved.by_script: head = pieces[name_pieces[0]] text = " ".join(tokens[i].text for i in head) token = tokens[head[0]] assert token.role is not None # rules.md#H4: "an input whose only remaining name word after # the title peel is itself title vocabulary reads that word as - # the name by convention and reports `title-or-name`" -- the - # word this line places is the whole of what the parse got to - # call a name, and calling it one is a convention: a name - # parser, not a title parser. ONE site, ahead of the `n == 0` - # clause below rather than inside it, which is what makes the - # report order-independent: under the default order H1 retags - # this word from given to family AFTERWARDS, and under a - # declared family-first order it is placed in the family here - # and H1 never runs -- the same reading, so the same report. - # A LONE title word never reaches this line at all ('Dr.', - # 'Prince of Wales'): the leading-title peel takes the whole - # name and `rest` is empty. The peeled titles are not tested: - # H2 makes an unlisted abbreviation a title by SHAPE, and - # 'Xyz. Smith' is not this input -- what the rule turns on is - # the word left standing as the name. The detail names no - # FIELD, unlike O5's below: under the default order H1 retags - # this word after assign, so a field named here would be the - # one it was placed in and not the one it ends in -- and the - # fork the kind reports is title-versus-name, which no field - # answers either way. + # the name by convention and reports `title-or-name`" + # (history: decisions.md#H4). ONE site, ahead of O5's leg + # rather than inside it, which is what makes the report + # order-independent: under the default order H1 retags this + # word from given to family AFTERWARDS, and under a declared + # family-first order it is placed in the family here and H1 + # never runs -- the same reading, so the same report. A LONE + # title word never reaches this line at all ('Dr.', 'Prince of + # Wales'): the leading-title peel takes the whole name and + # `rest` is empty. The peeled titles are not tested: H2 makes + # an unlisted abbreviation a title by SHAPE, and 'Xyz. Smith' + # is not this input -- what the rule turns on is the word left + # standing as the name. if len(head) == 1 and "vocab:title" in token.tags: ambiguities.append(PendingAmbiguity( AmbiguityKind.TITLE_OR_NAME, @@ -359,39 +371,55 @@ def _assign_main(seg_idx: int, state: ParseState, f"the title peel left standing; read as the name by " f"convention rather than as more title", tuple(head))) - elif (n == 0 + # rules.md#H4's join clause, stated at rules.md#O5 as its + # exception: the one name word is a JOIN (P3) and one of the + # words it joins is title vocabulary, so the doubt is not + # which field the unit takes but whether that word is a title + # at all -- 'John of Prince', 'Smith and Prince', 'Dr. Smith + # and Prince', 'van and Prince'. A join LEADING the name with + # a title word does not reach here: H3 chains it into a title + # run ('Prince of Wales'), the same silence as a lone 'Dr.'. + # Behind a peeled title it does, the join being the last piece + # and a title needing a following one -- 'Attorney General of + # Minnesota' and 'Deputy Secretary of State' are the two + # corpus names that arrive that way, and 'General' or + # 'Secretary' being a title is exactly this fork. + # `len(head) > 1` would be redundant beside the `any`: + # a one-token head carrying the tag took the branch above. + # Beside that branch and not under O5's leg, because every + # clause on that leg decides the FIELD -- a title peeled in + # front, a maiden marker, a claimed word -- and none of them + # answers whether the word inside the join is a title. + elif any("vocab:title" in tokens[i].tags for i in head): + ambiguities.append(PendingAmbiguity( + AmbiguityKind.TITLE_OR_NAME, + f"{text!r} is the only name unit and joins title " + f"vocabulary to a name word; read as a " + f"{token.role.value} name by convention", + tuple(head))) + # rules.md#O5: "a name of one name word that nothing else has + # decided reads that word as the given name under the default + # given-first order, and as the family name under a declared + # family-first one" (history: decisions.md#O5). The clauses + # are that rule's carve-out list, each naming the rule that + # decided the field instead: H1 (`n`, and `titled` for the + # title standing after a family comma), M4 (the maiden role), + # and the word's own claim. A2's content test is the last: a + # piece with no alphanumeric character is no name word, and + # the name it sits in assembles empty, so a convention report + # there would describe a reading nobody got -- parse("(") + # keeps its unbalanced-delimiter report and gains nothing. + elif (n == 0 and not titled + and not any(t.role is Role.MAIDEN for t in tokens) and not any(_WORD_ALREADY_CLAIMED & tokens[i].tags for i in head) - # A2's content test: a piece with no alphanumeric - # character is no name word, and the name it sits in - # assembles empty -- so a convention report there would - # describe a reading nobody got. parse("(") keeps its - # unbalanced-delimiter report and gains nothing here. and any(c.isalnum() for c in text)): - # rules.md#O5's exception: the one name word is a JOIN - # (P3) and one of the words it joins is title vocabulary, - # so the doubt is not which field the unit takes but - # whether that word is a title at all -- 'John of Prince' - # and 'Smith and Prince', the measured inputs that reach - # this branch. A join whose FIRST word is title vocabulary - # never does: H3 chains it into a title run ('Prince of - # Wales'), which reports nothing, the same silence as a - # lone 'Dr.'. - if len(head) > 1 and any( - "vocab:title" in tokens[i].tags for i in head): - ambiguities.append(PendingAmbiguity( - AmbiguityKind.TITLE_OR_NAME, - f"{text!r} is the only name unit and joins title " - f"vocabulary to a name word; read as a " - f"{token.role.value} name by convention", - tuple(head))) - else: - ambiguities.append(PendingAmbiguity( - AmbiguityKind.GIVEN_OR_FAMILY, - f"{text!r} is the only name word and nothing else " - f"decides it; read as a {token.role.value} name by " - f"convention, which follows the read order", - tuple(head))) + ambiguities.append(PendingAmbiguity( + AmbiguityKind.GIVEN_OR_FAMILY, + f"{text!r} is the only name word and nothing else " + f"decides it; read as a {token.role.value} name by " + f"convention, which follows the read order", + tuple(head))) for piece in peeled.picks: # every pick is in rest, so the loops above just gave it a role token = tokens[piece[0]] @@ -514,7 +542,14 @@ def assign(state: ParseState) -> ParseState: if reading is not None and sum( 1 for k, piece in enumerate(fam_pieces) if not is_suffix_piece(piece, fam_tags[k], tokens)) > 1: - order = _assign_main(0, state, tokens, ambiguities) + # `titled`: the gate's own reading says which pieces after + # the comma are suffixes, so a False in it is a TITLE + # ('John V, Dr.'). Segment 0's leading-title peel cannot + # count that title -- it stands in the other segment -- + # and H1 reads it, so the field-deciding reports have to + # be told (#449 review round). + order = _assign_main(0, state, tokens, ambiguities, + titled=not all(reading)) else: for k, piece in enumerate(fam_pieces): if k > 0 and is_suffix_piece(piece, fam_tags[k], tokens): diff --git a/nameparser/_types.py b/nameparser/_types.py index de09987b..e4cc6eba 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -378,7 +378,8 @@ class AmbiguityKind(StrEnum): #: Reserved: the name's field order itself is uncertain (e.g. a #: two-word name under a non-default name_order). Not yet emitted; - #: planned for 2.x. + #: planned for 2.x. A lone name word is GIVEN_OR_FAMILY's, not + #: this. ORDER = "order" #: Delimited content is an ambiguous suffix acronym, so it reads #: plausibly as either a post-nominal or a nickname -- "JEFFREY @@ -408,9 +409,11 @@ class AmbiguityKind(StrEnum): #: "Chancellor". A convention, not evidence: this is a name parser #: rather than a title parser, and handed a string whose every #: remaining word is a title the alternative is a title with no - #: name at all. ``detail`` names that word. The report does not - #: depend on ``name_order``: it is made where the word is placed, - #: which is before the rule that moves it between fields. + #: name at all. ``detail`` names that word. The report is made + #: before the rule that moves the word between fields, so the peel + #: shape reports under either order and names no field; the join + #: shape's ``detail`` names the field it was placed in. The peel + #: shape points at one token, the join shape at the whole unit. #: A LONE title word reports nothing: "Dr." reads as a title #: standing by itself, the peel left no word to be read as a #: name, and no fork was taken. @@ -454,11 +457,13 @@ class AmbiguityKind(StrEnum): #: family-first one, the same way every time, so ``detail`` names #: the field the convention chose rather than the kind naming it: #: the same reason PARTICLE_OR_GIVEN cannot. A name something DID - #: decide reports nothing -- a title, a nickname, a maiden name, a - #: comma, a script whose own convention settles the order, or the - #: vocabulary claiming the word (a particle, a bound given name, an - #: initial's shape) each settle the reading, and a settled reading - #: is not a fork. + #: decide reports nothing: a title, a maiden name, a family comma + #: that names a family, a script whose own convention settles the + #: order, or the vocabulary claiming the word (a particle, a bound + #: given name, an initial's shape) each settle the reading, and a + #: settled reading is not a fork. A nickname or a suffix standing + #: BESIDE the one name word does not settle it -- "'Smitty' Jones + #: Jr." and "Smith Jr." both report. GIVEN_OR_FAMILY = "given-or-family" #: A nickname/maiden delimiter opened without closing (or closed #: without opening); the text was kept as literal name content, so diff --git a/tests/v2/cases.py b/tests/v2/cases.py index d0454b4a..587e02aa 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -1657,7 +1657,7 @@ def _check_cjk_shape_purity(self) -> None: Case("maiden_marker_trailing_keeps_the_fork_report", "St St née", {"title": "St", "family": "St née"}, - ambiguities=("particle-or-given",), + ambiguities=("particle-or-given", "title-or-name"), notes="'st' is both a title and an ambiguous particle (#367), " "so this shape reaches group's PARTICLE_OR_GIVEN " "emitter, which is guarded on the chain having merged " @@ -1666,7 +1666,14 @@ def _check_cjk_shape_purity(self) -> None: "silencing the report while still deciding the fork -- " "the shape A1 forbids and #405 closed at P6. Pinned because " "removing a report a caller already sees is worse than " - "never emitting one"), + "never emitting one. The second flag is H4's join " + "clause, gained in the #518 review round: the one name " + "unit is a particle CHAIN rather than a P3 join, and " + "carries `st` -- title vocabulary -- beside the word it " + "places, which is the same fork. The `particle` tag on " + "that word is what used to silence it, and a claimed " + "word decides the FIELD and not whether the word is a " + "title"), Case("maiden_marker_particles_on_both_sides", "Anna von der Müller geb. von der Berg", {"given": "Anna", "family": "von der Müller", @@ -2253,6 +2260,12 @@ def _check_cjk_shape_purity(self) -> None: {"given": "de"}, classification="parity", notes="negative control: a lone particle's reading is P4's, " "not O5's convention"), + Case("lone_name_word_initial_shape_decides_it", "J", + {"given": "J"}, classification="parity", + notes="negative control: the word's own SHAPE claimed it as " + "an initial -- the third member of " + "_WORD_ALREADY_CLAIMED beside 'abdul' and 'de', and " + "the only one of the three that is not vocabulary"), Case("lone_name_word_beside_a_suffix_still_reports", "Smith Jr.", {"given": "Smith", "suffix": "Jr."}, ambiguities=("given-or-family",), classification="feat(#449)", @@ -2361,6 +2374,49 @@ def _check_cjk_shape_purity(self) -> None: "chains into a title run (H3) and never reaches the " "assignment site at all, so it reports nothing -- the " "same silence as a lone `Dr.`"), + Case("lone_joined_unit_behind_a_peeled_title", "Dr. Smith and Prince", + {"title": "Dr.", "family": "Smith and Prince"}, + ambiguities=("title-or-name",), classification="parity", + notes="the join clause is H4's and asks whether a WORD is a " + "title, so a title peeled in FRONT of the unit does not " + "answer it -- which is why the clause sits beside H4's " + "peel half rather than under O5's field-deciding " + "leg (#449 review round). 'Lord Chancellor née Jones' " + "is the same hoist on the maiden clause. Roles parity; " + "the flag is #491's"), + Case("lone_title_word_beside_a_maiden_name_still_reports", + "Lord Chancellor née Jones", + {"title": "Lord", "family": "Chancellor", "maiden": "Jones"}, + ambiguities=("title-or-name",), classification="fix(#410)", + notes="a maiden name decides the FIELD (M4, and H1's widening " + "is what keeps 'Chancellor' in the family), and says " + "nothing about whether 'Chancellor' is a title -- so it " + "silences O5's report and not H4's. 1.4.0 had no maiden " + "SUPPORT and read title 'Lord Chancellor', first 'née', " + "last 'Jones'"), + Case("comma_path_title_decides_the_lone_word", "John V, Dr.", + {"title": "Dr.", "family": "John", "suffix": "V"}, + ambiguities=("suffix-or-name",), classification="fix(#296)", + notes="a comma with no name word after it hands segment 0 " + "back to the positional read, and the title deciding " + "the field stands in the OTHER segment where the " + "leading-title peel cannot count it -- so the read is " + "told (`titled`) and O5 stays silent, where it named " + "`given` for a word H1 then wrote to `family` (#449 " + "review round). The roman numeral still reports, that " + "fork being untouched. 1.4.0 read first 'John', last " + "'V', suffix 'Dr.', 'dr' having been suffix vocabulary " + "before the audit"), + Case("comma_path_given_name_title_decides_the_lone_word", + "John V, Sir", + {"title": "Sir", "given": "John", "suffix": "V"}, + ambiguities=("suffix-or-name",), classification="fix(#296)", + notes="the row above with a GIVEN-NAME title, which leaves " + "the word in `given` -- the field O5 would have named. " + "Silenced all the same: what the report is about is a " + "field NOTHING decided, and this one a title decided, " + "so the two agreeing is not the report being right. " + "1.4.0 read title 'Sir', last 'John V'"), Case("marker_led_clause_in_a_quote_pair", 'Jane Smith "née Jones"', {"given": "Jane", "family": "Smith", "maiden": "Jones"}, @@ -3451,6 +3507,26 @@ def _check_cjk_shape_purity(self) -> None: notes="no default Han segmentation: one token, and a lone " "wholly-Han token takes the script order's first " "role = family"), + Case("han_unspaced_no_script_orders_reports_the_convention", "毛泽东", + {"given": "毛泽东"}, policy=Policy(script_orders=()), + ambiguities=("given-or-family",), classification="feat(#449)", + notes="core-only: an emptied script table has no v1 spelling, " + "so 'parity' could never have been true of this row -- " + "though the ROLES are 1.4.0's, v1 having no script " + "orders to empty. The positive control for `by_script`: " + "with nothing resolving the order the same text takes " + "O4/O5's default read and the convention reports it. " + "The row below is the same input with the entries in " + "place and is SILENT, which is the pair"), + Case("kana_honorific_no_script_orders_reports_the_convention", "さん", + {"given": "さん"}, policy=Policy(script_orders=()), + ambiguities=("suffix-or-name",), classification="feat(#491)", + notes="core-only for the reason given on the Han row above. " + "The suffix half's positive control, and what makes " + "rules.md#H4's CJK-honorific silence a `by_script` " + "scope rather than a claim about the word: empty the " + "script table and the same lone honorific reaches the " + "bare-suffix carve-out and reports"), Case("han_unspaced_family_first_declared_reports_nothing", "毛泽东", {"family": "毛泽东"}, policy=Policy(name_order=FAMILY_FIRST), notes="W4 AUTHORED this reading, and a declared family-first " diff --git a/tests/v2/pipeline/test_assign.py b/tests/v2/pipeline/test_assign.py index 67592634..c55d5e9e 100644 --- a/tests/v2/pipeline/test_assign.py +++ b/tests/v2/pipeline/test_assign.py @@ -104,6 +104,41 @@ def test_leading_particle_detail_names_the_role_it_took( f"read as a {role} name") +@pytest.mark.parametrize("policy,role", [ + (None, "given"), + (Policy(name_order=FAMILY_FIRST), "family"), + (Policy(name_order=FAMILY_FIRST_GIVEN_LAST), "family"), +]) +@pytest.mark.parametrize("text,kind,detail", [ + ("Andrew", AmbiguityKind.GIVEN_OR_FAMILY, + "'Andrew' is the only name word and nothing else decides it; " + "read as a {role} name by convention, which follows the read order"), + ("Rinpoche", AmbiguityKind.SUFFIX_OR_NAME, + "'Rinpoche' is post-nominal vocabulary with no name word beside " + "it; read as a {role} name rather than a post-nominal, nothing " + "else being left to be the name"), + ("John of Prince", AmbiguityKind.TITLE_OR_NAME, + "'John of Prince' is the only name unit and joins title " + "vocabulary to a name word; read as a {role} name by convention"), +]) +def test_convention_details_name_the_role_the_assignment_took( + text: str, kind: AmbiguityKind, detail: str, + policy: Policy | None, role: str) -> None: + # The three conventions this site reports all place a lone name + # word, and all three details have to READ the field back off the + # token rather than hardcode "given": the field follows the read + # order, which is why none of the three kinds names it. The + # PARTICLE_OR_GIVEN test above is the precedent, and these are the + # only other details at this site that name a field -- H4's PEEL + # shape ("Lord Chancellor") deliberately names none, because H1 + # retags that word after assign under the default order. + lex = _LEX.add(suffix_words={"rinpoche"}, conjunctions={"of"}, + titles={"prince"}) + (amb,) = _assigned(text, policy, lexicon=lex).ambiguities + assert amb.kind is kind + assert amb.detail == detail.format(role=role) + + def test_leading_particle_detail_follows_the_effective_order() -> None: # Reading policy.name_order[0] instead of the token's own role # would pass every case above, because there the two agree. They diff --git a/tests/v2/test_facade_cases.py b/tests/v2/test_facade_cases.py index be631d42..975157b0 100644 --- a/tests/v2/test_facade_cases.py +++ b/tests/v2/test_facade_cases.py @@ -100,6 +100,12 @@ # v1 spelling, so the row is core-only. The default-order twin # ("Lord Chancellor") is an ordinary row and runs here. "all_titles_input_family_first", + # #518 review round: the `by_script` scope's positive controls. + # An emptied script table has no v1 spelling -- v1 has no script + # orders to empty -- so both rows are core-only, though the roles + # they assert are 1.4.0's for exactly that reason. + "han_unspaced_no_script_orders_reports_the_convention", + "kana_honorific_no_script_orders_reports_the_convention", }) diff --git a/tests/v2/test_ledger_guards.py b/tests/v2/test_ledger_guards.py index e7145d08..392c693f 100644 --- a/tests/v2/test_ledger_guards.py +++ b/tests/v2/test_ledger_guards.py @@ -930,7 +930,7 @@ def test_case_shape_ids_exist_in_the_inventory() -> None: # against it. "feat(#449) a lone name word reports given-or-family": ("Dr. Andrew", "Andrew Smith", "Smith, Andrew", "abdul", "de", - "Dr. Smith Jr."), + "Dr. Smith Jr.", "J"), "feat(#449) a wholly-katakana name keeps the declared order, so the convention decides it": ("マイケル ジャクソン", "マイケル・ジャクソン", "Dr. マイケル"), "feat(#449) an interpunct transcription declines the script order, so the convention decides it": @@ -946,9 +946,22 @@ def test_case_shape_ids_exist_in_the_inventory() -> None: # it, a credential with a name word beside it, and the lone title # word the peel takes whole, leaving nothing to read as a name. "feat(#491) an all-titles input reports title-or-name for the word the title peel left as the name": - ("Dr. Smith", "Rabbi Cohen", "Mrs. Garcia", "Dr.", "King, Dr Jr"), + ("Dr. Smith", "Rabbi Cohen", "Mrs. Garcia", "Dr.", "King, Dr Jr", + "Lord Chancellor Smith", "The Rt Hon Jane Smith"), "feat(#491) an all-suffix input reports suffix-or-name for the word it made the name": - ("John Smith QC MP", "Lama Zopa Rinpoche", "Smith Jr.", "Jr."), + ("John Smith QC MP", "Lama Zopa Rinpoche", "Smith Jr.", "Jr.", + "Rinpoche Tenzin"), + # The join clause's own rule, added by the #518 review round when + # hoisting it out from under O5's field-deciding clauses reached + # two corpus names. Literal-anchored on the two, so the probes are + # the wall: each member with a name word behind it (the join stops + # being the only unit), the join LED by a title word that H3 + # chains into a title run instead, and a join whose members are no + # title vocabulary at all, which is O5's report and not this one. + "feat(#491) a joined unit carrying title vocabulary reports title-or-name": + ("Attorney General of Minnesota Smith", + "Deputy Secretary of State Jones", "Prince of Wales", + "Duke of Edinburgh"), } @@ -1630,6 +1643,14 @@ class _LatinCopy(NamedTuple): # credential-bearing name in the corpus, and what selects these two # is that no name word stood beside the credential. frozenset({"QC MP", "Rinpoche"}), + # #491's join movers, reached by the #518 review round's hoist. A + # list of two names, not a copy of TITLES: a member copying the + # wordlist would reach 'Prince of Wales', which H3 chains into a + # title run, and every title-plus-surname name besides. What + # selects these two is a SHAPE -- a peeled title, then a joined + # unit standing last with title vocabulary inside it. + frozenset({"Attorney General of Minnesota", + "Deputy Secretary of State"}), # fix(#445)'s movers, one corpus name per alternative -- a list of # names, not a copy of any wordlist, so there is no vocabulary for # it to drift from. Two sets because the ledgers group the nine @@ -2382,6 +2403,13 @@ def _claim(rule: dict) -> _Claim: _Claim(6, ('_ambiguities',), "9ee2a07d96c8", None), "feat(#491) an all-suffix input reports suffix-or-name for the word it made the name": _Claim(2, ('_ambiguities',), "e0756e2e1cd4", None), + # The join clause's own rule, added by the #518 review round: + # hoisting the clause out from under O5's field-deciding + # guards reached two corpus names, both a peeled title in + # front of a joined unit standing last. Literal-anchored on + # the two, so _MUST_NOT_MATCH carries the boundary probes. + "feat(#491) a joined unit carrying title vocabulary reports title-or-name": + _Claim(2, ('_ambiguities',), "7af3fec03ccc", None), # #346's alternation. Four corpus names, `family` and # `given` together: the fold moves both roles at once, so a # widening taking one alone would change the roles here @@ -2602,6 +2630,13 @@ def _claim(rule: dict) -> _Claim: _Claim(6, ('_ambiguities',), "9ee2a07d96c8", None), "feat(#491) an all-suffix input reports suffix-or-name for the word it made the name": _Claim(2, ('_ambiguities',), "e0756e2e1cd4", None), + # The join clause's own rule, added by the #518 review round: + # hoisting the clause out from under O5's field-deciding + # guards reached two corpus names, both a peeled title in + # front of a joined unit standing last. Literal-anchored on + # the two, so _MUST_NOT_MATCH carries the boundary probes. + "feat(#491) a joined unit carrying title vocabulary reports title-or-name": + _Claim(2, ('_ambiguities',), "7af3fec03ccc", None), # #346's alternation. Four corpus names, `family` and # `given` together: the fold moves both roles at once, so a # widening taking one alone would change the roles here @@ -2676,6 +2711,13 @@ def _claim(rule: dict) -> _Claim: _Claim(6, ('_ambiguities',), "9ee2a07d96c8", None), "feat(#491) an all-suffix input reports suffix-or-name for the word it made the name": _Claim(2, ('_ambiguities',), "e0756e2e1cd4", None), + # The join clause's own rule, added by the #518 review round: + # hoisting the clause out from under O5's field-deciding + # guards reached two corpus names, both a peeled title in + # front of a joined unit standing last. Literal-anchored on + # the two, so _MUST_NOT_MATCH carries the boundary probes. + "feat(#491) a joined unit carrying title vocabulary reports title-or-name": + _Claim(2, ('_ambiguities',), "7af3fec03ccc", None), # #346's alternation. Four corpus names, `family` and # `given` together: the fold moves both roles at once, so a # widening taking one alone would change the roles here diff --git a/tools/differential/expected_since_2.0.0.toml b/tools/differential/expected_since_2.0.0.toml index cc784fa4..10435a29 100644 --- a/tools/differential/expected_since_2.0.0.toml +++ b/tools/differential/expected_since_2.0.0.toml @@ -168,6 +168,22 @@ issue = "feat(#491) an all-suffix input reports suffix-or-name for the word it m name_regex = "(?i)^(?:QC MP|Rinpoche)$" fields = ["_ambiguities"] +[[change]] +issue = "feat(#491) a joined unit carrying title vocabulary reports title-or-name" +# H4's join clause -- the one name unit is a join (P3) and one of the +# words it joins is title vocabulary, so the doubt is whether that +# word is a title rather than which field the unit takes. These two +# arrive because the #518 review round moved the clause out from under +# O5's field-deciding guards: both peel a title in front ('Attorney', +# 'Deputy') and leave the join standing as the last piece, which a +# title cannot be, so H3 never chains it. A list of two names, not a +# copy of TITLES: a member copying the wordlist would reach 'Prince of +# Wales' -- a join LED by a title word, which H3 does chain -- and +# every title-plus-surname name besides. `_ambiguities` alone: the +# reading is unchanged, only the silence about it. +name_regex = "(?i)^(?:Attorney General of Minnesota|Deputy Secretary of State)$" +fields = ["_ambiguities"] + # #346: swami, guru, baba and lama moved from the TITLES-only block # into GIVEN_NAME_TITLES on 2026-09-06. rules.md#H1's Accepted clause # -- "a given-name title plus one name word leaves the family empty" diff --git a/tools/differential/expected_since_2.1.0.toml b/tools/differential/expected_since_2.1.0.toml index 6af933c4..9c507578 100644 --- a/tools/differential/expected_since_2.1.0.toml +++ b/tools/differential/expected_since_2.1.0.toml @@ -188,6 +188,22 @@ issue = "feat(#491) an all-suffix input reports suffix-or-name for the word it m name_regex = "(?i)^(?:QC MP|Rinpoche)$" fields = ["_ambiguities"] +[[change]] +issue = "feat(#491) a joined unit carrying title vocabulary reports title-or-name" +# H4's join clause -- the one name unit is a join (P3) and one of the +# words it joins is title vocabulary, so the doubt is whether that +# word is a title rather than which field the unit takes. These two +# arrive because the #518 review round moved the clause out from under +# O5's field-deciding guards: both peel a title in front ('Attorney', +# 'Deputy') and leave the join standing as the last piece, which a +# title cannot be, so H3 never chains it. A list of two names, not a +# copy of TITLES: a member copying the wordlist would reach 'Prince of +# Wales' -- a join LED by a title word, which H3 does chain -- and +# every title-plus-surname name besides. `_ambiguities` alone: the +# reading is unchanged, only the silence about it. +name_regex = "(?i)^(?:Attorney General of Minnesota|Deputy Secretary of State)$" +fields = ["_ambiguities"] + # The four CJK names of #436/#437's class, one rule each. Not one # alternation: an alternation holding a script-classified member is # claimed by the honorific pin in tests/v2/test_ledger_guards.py, diff --git a/tools/differential/expected_since_2.2.0.toml b/tools/differential/expected_since_2.2.0.toml index 49612904..a063492d 100644 --- a/tools/differential/expected_since_2.2.0.toml +++ b/tools/differential/expected_since_2.2.0.toml @@ -177,6 +177,22 @@ issue = "feat(#491) an all-suffix input reports suffix-or-name for the word it m name_regex = "(?i)^(?:QC MP|Rinpoche)$" fields = ["_ambiguities"] +[[change]] +issue = "feat(#491) a joined unit carrying title vocabulary reports title-or-name" +# H4's join clause -- the one name unit is a join (P3) and one of the +# words it joins is title vocabulary, so the doubt is whether that +# word is a title rather than which field the unit takes. These two +# arrive because the #518 review round moved the clause out from under +# O5's field-deciding guards: both peel a title in front ('Attorney', +# 'Deputy') and leave the join standing as the last piece, which a +# title cannot be, so H3 never chains it. A list of two names, not a +# copy of TITLES: a member copying the wordlist would reach 'Prince of +# Wales' -- a join LED by a title word, which H3 does chain -- and +# every title-plus-surname name besides. `_ambiguities` alone: the +# reading is unchanged, only the silence about it. +name_regex = "(?i)^(?:Attorney General of Minnesota|Deputy Secretary of State)$" +fields = ["_ambiguities"] + # The four CJK names of #436/#437's class, one rule each. Not one # alternation: an alternation holding a script-classified member is # claimed by the honorific pin in tests/v2/test_ledger_guards.py, From 495a9bd435735e962228463ab4eec0aad4019031 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Tue, 8 Sep 2026 02:51:55 -0700 Subject: [PATCH 5/5] refactor(assign): one field-undecided predicate, one title-vocabulary branch Cleanup after review, behavior identical -- every corpus name reads the same roles and reports the same kinds, details and tokens under all three orders (measured against the previous tree). The bare-suffix flag is the peel count; the two field-deciding emitters share one predicate instead of repeating its clauses; the two title-vocabulary branches are one branch choosing its detail on the unit's length; head and token are computed once; _WORD_ALREADY_CLAIMED derives from M4's _NEVER_FLIPPED, now in a shared home, so the pair cannot drift; the test order list is shared; the emitter comments are the rule quotes plus what is true only of this code. The hoist is KEPT: segment 1 is assigned before segment 0's positional read, so a title standing after the family comma carries a Role.TITLE by the time the emitters run, and a token-role scan replaces both the `titled` keyword and the `n == 0` clause beside it. Co-Authored-By: Claude Fable 5.1 --- docs/design/decisions.md | 5 +- nameparser/_pipeline/_assign.py | 279 ++++++++++++---------------- nameparser/_pipeline/_post_rules.py | 8 +- nameparser/_pipeline/_state.py | 10 + tests/v2/pipeline/test_assign.py | 22 ++- 5 files changed, 141 insertions(+), 183 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index 32c6d96d..dba003b2 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -347,11 +347,12 @@ Declined (rc1 arc; the full argument is AGENTS.md's gotcha): Five CJK-bearing names DO report, and none of them is a script rule deciding. Three are a script rule DECLINING: マイケル (W4 gives katakana no order of its own, a wholly-katakana name being read as a transcription), 王·Smith (T3's interpunct suppresses the script_orders lookup whole) and 田中、太郎 (the comma sits inside the token, so the text resolves to no single script). Two are a glued honorific peeled off a Latin stem — Andersonさん and Anderson선생님 — where what is left is Latin and has no script order at all. Recorded because "no CJK name reports" was the expectation going in, and it is wrong in a way that is right: where the script rules decline, what is left IS the one-word convention. Four conditions were drafted into the guard and none of them shipped, but they were dropped for TWO different reasons, and a first draft of this entry flattened both into "inert". Each was measured the same way: restore that one clause on the O5 branch of a `git archive` copy of this commit, reparse the deduped `corpus*.jsonl` glob, and count which of the 27 lose the report. Two are genuinely INERT, silencing ZERO names (measured 2026-09-08, and again after the 2026-09-08 review round below). `len(state.segments) == 1` is inert OVER THE CORPUS rather than by construction, and a first draft of this line gave the construction reason, which is false: a family comma that names a family does read segment 0 wholly as the family and never positionally, but a comma followed by NO name word hands segment 0 back to the positional read, and `Smith V, Dr.` reaches this very site that way. What silences it is not the clause but the title in segment 1 — after the review round below, which is what tells the read that a title stands there. `not flagged`: a split credential decides the field no more than a whole one does. The general shape is the one this file keeps recording — a guard written from the cases that prompted it carries terms the code already excludes — and the instrument that finds them is mutation, not reading. The other two were dropped DELIBERATELY, and the same mutation prices each. `not suffix_pieces` silences NINE of the 27 — `Smith Jr.`, `'Smitty' Jones Jr.`, `Carod i`, `Donald mc`, `Jack M.A.`, `John V`, `Mohamad X`, `Andersonさん`, `Anderson선생님` — and `not has_nickname` silences ONE, `'Smitty' Jones Jr.`. Both were dropped on one principle: a suffix or a nickname standing beside the lone name word decides nothing about that word's FIELD, which is the only question this report is about. `Smith Jr.` and `'Smitty' Jones Jr.` are S2's peel taking the suffix and the convention placing what is left, so they report. A first draft justified the nickname clause with "N3 returns before this site", and that is false as written: `_assign.py`'s N3 branch tests `len(pieces) == 1 and len(rest) == 1 and has_nickname`, so it returns only where the segment holds ONE piece, and `'Smitty' Jones Jr.` holds two and reaches this line. What N3 takes is the nickname-plus-one-word shape, not every name a nickname stands in. - Not reported, deliberately: `abdul`, where bound-given vocabulary claimed the word; `de`, where a lone particle's reading is P4's; and `J.`, where the initial's shape is the claim. Those are M4's `_NEVER_FLIPPED` pair plus the particle, read off the tags classify already recorded rather than off a predicate of this emitter's own. Also not reported: `Sir John`, `'Smitty' Jones` and `abd née Jones`, which O5's example block lists because the convention is what H1, N3 and M4 each left behind on them. The rule's SCOPE and the report's scope differ there, and the difference is stated in the rule rather than left to be discovered: the report is narrowed to an input nothing else stands in, because a report on a name where a title, a nickname or a maiden name is visible in the output tells the caller less than the output already does. + Not reported, deliberately: `abdul`, where bound-given vocabulary claimed the word; `de`, where a lone particle's reading is P4's; and `J.`, where the initial's shape is the claim. Those are M4's `_NEVER_FLIPPED` pair plus the particle, read off the tags classify already recorded rather than off a predicate of this emitter's own — literally so since 2026-09-08: `_NEVER_FLIPPED` moved to `_state` beside `WorkToken.tags` (post_rules imports _assign, so _assign cannot reach the other way) and `_WORD_ALREADY_CLAIMED` is that frozenset unioned with `particle`, so the two sets cannot drift. Also not reported: `Sir John`, `'Smitty' Jones` and `abd née Jones`, which O5's example block lists because the convention is what H1, N3 and M4 each left behind on them. The rule's SCOPE and the report's scope differ there, and the difference is stated in the rule rather than left to be discovered: the report is narrowed to an input nothing else stands in, because a report on a name where a title, a nickname or a maiden name is visible in the output tells the caller less than the output already does. The report is order-independent, which is not the same as the FIELD being order-independent: `Andrew` reports under the default order with `detail` naming given and under FAMILY_FIRST with `detail` naming family. Measured over the whole corpus under the default order, FAMILY_FIRST and FAMILY_FIRST_GIVEN_LAST, the set of names reporting this kind is the same 27 in all three; the only names whose reported kinds move with the order at all are four pre-existing `particle-or-given` reports. Nothing moves but the report. Every role is identical before and after — measured by parsing all 1123 corpus names under all three orders against a `git archive` of the parent commit, 3369 parses, zero role differences — and the three 2.x ledgers classify the 27 names on `_ambiguities` alone; `_ambiguities` cannot enter a diff below baseline 2.0, so `expected_since_1.4.0.toml` is untouched and its 368 intentional diffs are unchanged. SIX ledger rules rather than one, because an alternation carrying a script-classified member is claimed by the honorific pin in tests/v2/test_ledger_guards.py, which would demand it be a hand copy of GLUED_HONORIFICS: the twenty-two Latin-and-Arabic names ride one alternation, declared in `_NOT_A_VOCABULARY_COPY`, and the five CJK-bearing names take a literal-anchored rule each, which is also where their arguments differ. The three ledger copies are NOT byte-identical, which the drafting expected them to be: in `expected_since_2.0.0.toml` alone the two glued-honorific rules declare `["_ambiguities", "given", "suffix"]`, because at that baseline #308's peel has not shipped and its diff arrives in the same rule as this report, and the gate's over-declared check refuses a rule declaring a role no diff it explains moves. - 2026-09-08 #518 review round (the entry above ran into a second day; its own measurements are dated 2026-09-08 under a 2026-09-07 header) — a title standing after a family comma now silences the report. `John V, Dr.` is the shape: a comma with no name word after it hands segment 0 back to the positional read, the leading-title peel counts only what stands in segment 0, so `n` is 0 and the report named `given` for a word H1 then wrote to `family` — the caller was told the convention chose a field the output does not show. `_assign_main` takes a keyword-only `titled`, which the family-comma call fills from the gate's own reading (a False in it is a title, not a suffix), and the field-deciding emitters require `not titled`. Measured on this branch: `John V, Dr.`, `Smith V, Prince` and `Smith V, Dr.` keep every role and lose the `given-or-family` report, keeping the roman numeral's; `John V, Sir` does too, and it is the one worth naming — a given-name title leaves the word in `given`, which is the field the report would have named, so the report and the reading agreeing is what a silenced clause looks like when the rule that decided it happens to agree. Zero corpus names move: no corpus name reaches the site under a comma at all, which is also why the `len(state.segments) == 1` clause above measures inert. + Consolidated the same day, 2026-09-08, and BEHAVIOR IDENTICAL. Segment 1 is now assigned BEFORE segment 0's positional read, so the title standing after the comma already carries a `Role.TITLE` when that read reaches its emitters; the keyword is gone and `n == 0`, `not titled`, the maiden scan and `not resolved.by_script` are one `field_undecided` predicate that both field-deciding emitters read. The token scan subsumes `n == 0` because nothing before assign sets `Role.TITLE` and assign sets it in exactly two places, the leading-title peel and the post-comma reading. Measured against a `git archive` of the previous commit: every distinct name in the deduped `corpus*.jsonl` glob plus the shapes named in this entry and in the #491 entries — 1136 names, 5680 parses — reads the same seven roles and reports the same kinds, details and token texts under the default order, FAMILY_FIRST and FAMILY_FIRST_GIVEN_LAST. One shape constraint came out of the measurement rather than the reading: the two token scans sit UNDER the peel count rather than beside it, because run on every parse they cost 10 calls/name and put the facade 457 over the 456 ceiling of its band (decisions.md#parse-cost). ### O4 — positional assignment and declared order @@ -405,7 +406,7 @@ Open: [#316](https://github.com/derek73/python-nameparser/issues/316) what a tra - 2026-09-07 #491 — the reading is unchanged and the silence is what ends. Handed a string the title peel eats down to one last word which is itself title vocabulary, the parser reads that word as the name; decisions.md#v1-xfail-triage recorded it in the fourth of its NOT FIXED entries — "this is a name parser, not a title parser" — and said in the same breath that what is actually wrong is the guess being silent. It now reports `title-or-name`, with `detail` naming the word that was made into the name. Six corpus names gain it: the Queen's Bench string, `Lord Chancellor`, `Dr. King`, `The Rt Hon`, `His Holiness` and `His Holiness the Dalai Lama`. Three read contract-tier and three radar, and the tier split is an artifact of the documentation rather than of the shape — the three contract ones are contract because this bundle made them rules.md examples, which puts them in corpus_rules.jsonl. `Dr. King` is the one worth arguing about and it is deliberate: `king` is in the titles vocabulary for the addressing forms, which the triage entry above decided and did not reopen, so `Dr. King` IS an input whose last standing word is title vocabulary and the rule claims it. Reporting there is honest rather than noisy — the reading came from a convention, not from anything the input says — and a caller who wants only the exotic cases has `detail` to filter on. ONE emitter, and it is at assign's lone-name-word site rather than at H1's retag, which is where the drafting put it. H1 is not the site: under a declared family-first order the assignment places the word in the family directly and H1 never runs, so an emitter there would report under one order and not the other for a reading that is the same either way. Measured under the default order, FAMILY_FIRST and FAMILY_FIRST_GIVEN_LAST, the six report identically. That placement is also why the `detail` names no field, unlike O5's: under the default order H1 retags the word after assign, so a field named at the emitter would be the one the word was placed in and not the one it ends in — and the fork the kind reports is title-versus-name, which no field answers either way. - What is silent, all measured 2026-09-08. A lone title word: `Dr.`, `Sir`, `King` and the chained `Prince of Wales` are a title run with nothing behind it, the peel takes the whole string, no word is left standing to be read as a name, and nothing was chosen — mechanisms.md#AMBIGUITY-AT-THE-DECISION-SITE's "a branch that runs but changes nothing is not a decision". A title with an ordinary word behind it: `Dr. Smith` and `King Charles` leave a word standing, but not a title-vocabulary one, so H1 alone explains them. And a title followed by post-nominal vocabulary: `Dr King Jr` and `Dr. King MD` peel `Dr King` and `Dr. King` WHOLE, leaving a credential that the bare-suffix carve-out makes the name — a different convention, and its report is scoped to `n == 0`, so a title in front of the run takes the input out of it and leaves the reading H1's. The peeled titles are never tested for anything: H2 makes an unlisted abbreviation a title by SHAPE, and `Xyz. Smith` is not this input. What the rule turns on is the word left standing. + What is silent, all measured 2026-09-08. A lone title word: `Dr.`, `Sir`, `King` and the chained `Prince of Wales` are a title run with nothing behind it, the peel takes the whole string, no word is left standing to be read as a name, and nothing was chosen — mechanisms.md#AMBIGUITY-AT-THE-DECISION-SITE's "a branch that runs but changes nothing is not a decision". A title with an ordinary word behind it: `Dr. Smith` and `King Charles` leave a word standing, but not a title-vocabulary one, so H1 alone explains them. And a title followed by post-nominal vocabulary: `Dr King Jr` and `Dr. King MD` peel `Dr King` and `Dr. King` WHOLE, leaving a credential that the bare-suffix carve-out makes the name — a different convention, and its report is scoped to inputs no title stands in (spelled `n == 0` when this was written, one clause of the `field_undecided` predicate since the 2026-09-08 consolidation recorded under O5), so a title in front of the run takes the input out of it and leaves the reading H1's. The peeled titles are never tested for anything: H2 makes an unlisted abbreviation a title by SHAPE, and `Xyz. Smith` is not this input. What the rule turns on is the word left standing. A refinement of Derek's, made after the population was measured and widening the rule past the all-titles shape it was drafted for: a lone name word that is a JOIN (P3) carrying title vocabulary reports `title-or-name` too, the fork there being whether the title word inside the unit is a title at all rather than which field the unit takes. `John of Prince` and `Smith and Prince` are the measured inputs; no corpus name reaches that branch, because a join LED by a title word is chained into a title run by H3 (`Prince of Wales`), so the two are pinned as case rows rather than as rules.md examples. It sits on O5's branch and takes precedence there, which is why O5's statement says a title silences THIS kind and not every report at the site. The suffix half is the same argument on the other vocabulary and needed no new kind. An input whose every word is post-nominal vocabulary reads its first word as a name — assign's "everything suffix-shaped after titles: first one is the name" carve-out — and that is the doubt SUFFIX_OR_NAME already names, so it reports that. `Rinpoche` and `QC MP` are the corpus names; `PhD`, `MBA` and `III` are the same shape. `Jr.` alone is NOT the shape at all: H2's opening-abbreviation rule reads it as a title before the suffix vocabulary is consulted, which is that rule's stated precedence and is recorded here because the expectation going in was that `Jr.` alone read as a name. The guard carries two exclusions of its own. A maiden name beside the credential says the input is not post-nominal vocabulary and nothing else, so `abd née Jones` is out for the reason M4 keeps it out of O5's report. And the report is scoped to names no script order placed, so a lone glued CJK honorific — さん, 씨, 선생님 — reports nothing: that is the same shape read through the glued-honorific rules (W2, #271/#308) and the script's own order, and whether those readings should report is left to the arc that revisits them rather than settled here. An asymmetry on the boundary between this rule and O5, noted and deliberately not fixed. `MA` and `Ma` alone report `given-or-family`; `PhD` alone reports `suffix-or-name`. Both are a bare credential with nothing beside it, and what separates them is which gate reads them: `ma` is an AMBIGUOUS acronym, so S2's gate declines to peel it and the word stands as the one name word, which is O5's branch; `phd` is unambiguous, so it peels to suffix, leaves no name piece, and reaches the bare-suffix carve-out, which is this rule's. Two conventions, two kinds, and the reading each name gets is the same either way — the caller sees a report in both cases and only the kind differs. Fixing it would mean one of the two gates changing what it reads, which moves fields for a bundle that moves none. diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index cbf3f3ff..536612d3 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -58,7 +58,7 @@ segment_suffix_reading, ) from nameparser._pipeline._state import ( - ParseState, PendingAmbiguity, Structure, WorkToken, + ParseState, PendingAmbiguity, Structure, WorkToken, _NEVER_FLIPPED, ) from nameparser._policy import Policy, Script from nameparser._types import AmbiguityKind, Role @@ -70,14 +70,15 @@ def _set_roles(tokens: list[WorkToken], piece: tuple[int, ...], #: Tags that say the word's own reading was claimed before position -#: could speak, so O5's convention decided nothing. Two are M4's -#: `_NEVER_FLIPPED` pair, for M4's reasons -- a bound given-name word is -#: vocabulary claiming the word as a given name, `initial` is the shape -#: claim -- and `particle` is here because a lone particle's reading is -#: P4's. None is a predicate this emitter owns: they are read off the -#: tags classify already recorded (mechanisms.md#TWO-LAYER-ASSIGN). -_WORD_ALREADY_CLAIMED = frozenset({ - "particle", "vocab:bound-given", "initial"}) +#: could speak, so O5's convention decided nothing. Built from M4's +#: `_NEVER_FLIPPED` pair rather than respelling it: the two sets answer +#: different questions -- may M4 retag the word, and did anything +#: decide the field -- and share the pair because a word vocabulary +#: claimed as a given name, or wrote as an initial, answers both. The +#: third, `particle`, is here alone because a lone particle's reading +#: is P4's. None is a predicate this emitter owns: they are read off +#: the tags classify already recorded (mechanisms.md#TWO-LAYER-ASSIGN). +_WORD_ALREADY_CLAIMED = _NEVER_FLIPPED | frozenset({"particle"}) # rules.md#H2: "an abbreviation opening the part of the name that @@ -213,19 +214,9 @@ def _name_positions(order: tuple[Role, Role, Role], def _assign_main(seg_idx: int, state: ParseState, tokens: list[WorkToken], ambiguities: list[PendingAmbiguity], - *, titled: bool = False, ) -> tuple[Role, Role, Role] | None: """Returns the order the positional read used, for ParseState.order - -- None on every path that returns before resolving one. - - `titled` is the one fact about ANOTHER segment this read needs: on - the family-comma path a title stands after the comma ('John V, - Dr.'), the caller sends segment 0 here to be read positionally, - and the leading-title peel below counts nothing -- so `n` cannot - see the title and O5's report would name a field H1 then rewrites. - Only the field-deciding emitters read it; H4's two halves ask - whether a WORD is a title, which a title elsewhere does not - answer.""" + -- None on every path that returns before resolving one.""" pieces = state.pieces[seg_idx] ptags = state.piece_tags[seg_idx] has_nickname = any(t.role is Role.NICKNAME for t in tokens) @@ -272,10 +263,8 @@ def _assign_main(seg_idx: int, state: ParseState, f"letter there would be a middle initial", peeled.numeral)) name_pieces, suffix_pieces = rest[:peeled.names], rest[peeled.names:] - bare_suffix = False - if not name_pieces and suffix_pieces: + if peeled.names == 0: # everything suffix-shaped after titles: first one is the name - bare_suffix = True name_pieces, suffix_pieces = suffix_pieces[:1], suffix_pieces[1:] # AFTER both peels, and load-bearing: the script test sees the NAME # pieces only, so a Latin title or suffix ('Dr. 毛 泽东', '毛 泽东, @@ -289,137 +278,97 @@ def _assign_main(seg_idx: int, state: ParseState, _set_roles(tokens, pieces[piece_idx], roles[pos]) for piece_idx in suffix_pieces: _set_roles(tokens, pieces[piece_idx], Role.SUFFIX) - # rules.md#H4: "an input whose every word is post-nominal - # vocabulary reads its first word as a name and reports - # `suffix-or-name`" (history: decisions.md#H4) -- reported at the - # carve-out above that applies it - # (mechanisms.md#AMBIGUITY-AT-THE-DECISION-SITE), and only the - # word made into a name reports. `n == 0` is what keeps a titled - # run out -- 'Dr King Jr' and 'MD DDS' peel a title first, and - # after a title the reading is H1's rather than this convention's; - # `titled` is the same exclusion for the title the peel could not - # see, standing after a family comma. A maiden name beside the - # credential says the input is not post-nominal vocabulary and - # nothing else, so 'abd née Jones' is out for the same reason M4 - # keeps it out of O5's report below. `resolved.by_script` is a - # SCOPE line rather than a claim that something else decided: a - # lone CJK honorific ('さん', '씨', '선생님') is the same shape read - # through the glued-honorific rules and the script's own order - # (W2, #271/#308), and whether those readings should report is - # left to the arc that revisits them rather than settled here. The - # role comes off the token for the reason stated at the particle - # emitter below. - if (bare_suffix and n == 0 and not titled and not resolved.by_script - and not any(t.role is Role.MAIDEN for t in tokens)): + # Both emitters below turn on the PEEL's count, so neither can + # reach a name that kept two name pieces, and the two scans are + # nested under the count rather than run beside it: they cost the + # call budget on every parse otherwise (decisions.md#parse-cost). + if peeled.names <= 1: head = pieces[name_pieces[0]] token = tokens[head[0]] assert token.role is not None - text = " ".join(tokens[i].text for i in head) - ambiguities.append(PendingAmbiguity( - AmbiguityKind.SUFFIX_OR_NAME, - f"{text!r} is post-nominal vocabulary with no name word " - f"beside it; read as a {token.role.value} name rather than " - f"a post-nominal, nothing else being left to be the name", - tuple(head))) - # The site that places a lone name word, and so the site that - # reports both conventions which turn on one - # (mechanisms.md#AMBIGUITY-AT-THE-DECISION-SITE): H4's two halves - # below, then O5's. The outer guard holds only what silences ALL - # THREE -- a script whose order convention settles the reading - # (W4 decided it, which O5's rule states without naming W4), read - # as `resolved.by_script` rather than as a comparison against - # name_order, a declared family-first order agreeing with a Han - # name's entry being agreement and not authorship. Everything - # else on O5's carve-out list decides the FIELD and not whether - # the word is a title, so it sits on O5's own branch: a leading - # title (H1's reading then), a maiden name (M4's), the word's own - # claim, and `titled` for the title a family comma put where the - # peel cannot count it. A suffix beside the word is NOT such a - # shape -- 'Smith Jr.' and "'Smitty' Jones Jr." are the convention - # placing a lone name word, which is why S2's peel does not - # silence it. The count comes off the PEEL rather than off - # name_pieces, which is what leaves the bare-suffix carve-out - # above (peeled.names == 0, the first post-nominal read as the - # name for want of anything else) out of all three branches: that - # reading is a different convention, and reporting it is H4's - # suffix half above. The role comes off the token for the reason - # stated at the particle emitter below. - if peeled.names == 1 and not resolved.by_script: - head = pieces[name_pieces[0]] - text = " ".join(tokens[i].text for i in head) - token = tokens[head[0]] - assert token.role is not None - # rules.md#H4: "an input whose only remaining name word after - # the title peel is itself title vocabulary reads that word as - # the name by convention and reports `title-or-name`" - # (history: decisions.md#H4). ONE site, ahead of O5's leg - # rather than inside it, which is what makes the report - # order-independent: under the default order H1 retags this - # word from given to family AFTERWARDS, and under a declared - # family-first order it is placed in the family here and H1 - # never runs -- the same reading, so the same report. A LONE - # title word never reaches this line at all ('Dr.', 'Prince of - # Wales'): the leading-title peel takes the whole name and - # `rest` is empty. The peeled titles are not tested: H2 makes - # an unlisted abbreviation a title by SHAPE, and 'Xyz. Smith' - # is not this input -- what the rule turns on is the word left - # standing as the name. - if len(head) == 1 and "vocab:title" in token.tags: - ambiguities.append(PendingAmbiguity( - AmbiguityKind.TITLE_OR_NAME, - f"{text!r} is title vocabulary and the only name word " - f"the title peel left standing; read as the name by " - f"convention rather than as more title", - tuple(head))) - # rules.md#H4's join clause, stated at rules.md#O5 as its - # exception: the one name word is a JOIN (P3) and one of the - # words it joins is title vocabulary, so the doubt is not - # which field the unit takes but whether that word is a title - # at all -- 'John of Prince', 'Smith and Prince', 'Dr. Smith - # and Prince', 'van and Prince'. A join LEADING the name with - # a title word does not reach here: H3 chains it into a title - # run ('Prince of Wales'), the same silence as a lone 'Dr.'. - # Behind a peeled title it does, the join being the last piece - # and a title needing a following one -- 'Attorney General of - # Minnesota' and 'Deputy Secretary of State' are the two - # corpus names that arrive that way, and 'General' or - # 'Secretary' being a title is exactly this fork. - # `len(head) > 1` would be redundant beside the `any`: - # a one-token head carrying the tag took the branch above. - # Beside that branch and not under O5's leg, because every - # clause on that leg decides the FIELD -- a title peeled in - # front, a maiden marker, a claimed word -- and none of them - # answers whether the word inside the join is a title. - elif any("vocab:title" in tokens[i].tags for i in head): + # Both conventions here turn on a lone name word, so both + # report at the site that places one + # (mechanisms.md#AMBIGUITY-AT-THE-DECISION-SITE), and one + # predicate carries what says nothing else decided the FIELD: + # a title (segment 0's own peel, or one a family comma left in + # the next segment -- either is a Role.TITLE by now, and + # either makes the reading H1's), a maiden name (M4's), or a + # script order convention. `resolved.by_script` rather than a + # comparison against name_order: a declared family-first order + # agreeing with a Han name's entry is agreement, not + # authorship, and for a lone CJK honorific ('さん', '씨') it + # scopes W2's reading out rather than excusing it (#271/#308). + titled = any(t.role is Role.TITLE for t in tokens) + field_undecided = ( + not resolved.by_script and not titled + and not any(t.role is Role.MAIDEN for t in tokens)) + # rules.md#H4: "an input whose every word is post-nominal + # vocabulary reads its first word as a name and reports + # `suffix-or-name`" (history: decisions.md#H4) -- only the + # word made into a name reports, and no name piece survived + # the peel, so the carve-out above made the first post-nominal + # the name. The role comes off the token for the reason stated + # at the particle emitter below. + if peeled.names == 0 and field_undecided: + text = " ".join(tokens[i].text for i in head) ambiguities.append(PendingAmbiguity( - AmbiguityKind.TITLE_OR_NAME, - f"{text!r} is the only name unit and joins title " - f"vocabulary to a name word; read as a " - f"{token.role.value} name by convention", + AmbiguityKind.SUFFIX_OR_NAME, + f"{text!r} is post-nominal vocabulary with no name word " + f"beside it; read as a {token.role.value} name rather than " + f"a post-nominal, nothing else being left to be the name", tuple(head))) - # rules.md#O5: "a name of one name word that nothing else has - # decided reads that word as the given name under the default - # given-first order, and as the family name under a declared - # family-first one" (history: decisions.md#O5). The clauses - # are that rule's carve-out list, each naming the rule that - # decided the field instead: H1 (`n`, and `titled` for the - # title standing after a family comma), M4 (the maiden role), - # and the word's own claim. A2's content test is the last: a - # piece with no alphanumeric character is no name word, and - # the name it sits in assembles empty, so a convention report - # there would describe a reading nobody got -- parse("(") - # keeps its unbalanced-delimiter report and gains nothing. - elif (n == 0 and not titled - and not any(t.role is Role.MAIDEN for t in tokens) - and not any(_WORD_ALREADY_CLAIMED & tokens[i].tags + # One name piece off the peel: the convention placed a lone + # name word. A suffix beside it is not a decision -- 'Smith + # Jr.' and "'Smitty' Jones Jr." ARE this convention -- and the + # count comes off the peel rather than off name_pieces, which + # is what keeps the carve-out above out of these branches. + # Title vocabulary in the unit makes the doubt H4's (is the + # WORD a title), anything else O5's (which FIELD it took), so + # only the second reads `field_undecided`. + if peeled.names == 1 and not resolved.by_script: + # rules.md#H4: "an input whose only remaining name word + # after the title peel is itself title vocabulary reads + # that word as the name by convention and reports + # `title-or-name`" (history: decisions.md#H4), and H4's + # join clause, stated at rules.md#O5 as its exception -- + # one branch, its detail naming a field only in the join + # shape ('John of Prince', 'Attorney General of + # Minnesota'), where the unit is more than the title word. + # A LONE title word never reaches here ('Dr.', 'Prince of + # Wales'): the leading-title peel took the whole name. + if any("vocab:title" in tokens[i].tags for i in head): + text = " ".join(tokens[i].text for i in head) + ambiguities.append(PendingAmbiguity( + AmbiguityKind.TITLE_OR_NAME, + f"{text!r} is title vocabulary and the only name word " + f"the title peel left standing; read as the name by " + f"convention rather than as more title" + if len(head) == 1 else + f"{text!r} is the only name unit and joins title " + f"vocabulary to a name word; read as a " + f"{token.role.value} name by convention", + tuple(head))) + # rules.md#O5: "a name of one name word that nothing else has + # decided reads that word as the given name under the default + # given-first order, and as the family name under a declared + # family-first one" (history: decisions.md#O5). Past + # `field_undecided`, two clauses of O5's own: the word's + # own reading may have claimed it, and A2's content test + # -- a piece with no alphanumeric character is no name + # word, so parse("(") keeps its unbalanced-delimiter + # report and gains nothing here. + elif (field_undecided + and all(tokens[i].tags.isdisjoint(_WORD_ALREADY_CLAIMED) for i in head) - and any(c.isalnum() for c in text)): - ambiguities.append(PendingAmbiguity( - AmbiguityKind.GIVEN_OR_FAMILY, - f"{text!r} is the only name word and nothing else " - f"decides it; read as a {token.role.value} name by " - f"convention, which follows the read order", - tuple(head))) + and any(c.isalnum() for i in head + for c in tokens[i].text)): + text = " ".join(tokens[i].text for i in head) + ambiguities.append(PendingAmbiguity( + AmbiguityKind.GIVEN_OR_FAMILY, + f"{text!r} is the only name word and nothing else " + f"decides it; read as a {token.role.value} name by " + f"convention, which follows the read order", + tuple(head))) for piece in peeled.picks: # every pick is in rest, so the loops above just gave it a role token = tokens[piece[0]] @@ -539,23 +488,13 @@ def assign(state: ParseState) -> ParseState: reading = segment_suffix_reading( state.pieces[1], state.piece_tags[1], tokens, state.policy.lenient_comma_suffixes) - if reading is not None and sum( - 1 for k, piece in enumerate(fam_pieces) - if not is_suffix_piece(piece, fam_tags[k], tokens)) > 1: - # `titled`: the gate's own reading says which pieces after - # the comma are suffixes, so a False in it is a TITLE - # ('John V, Dr.'). Segment 0's leading-title peel cannot - # count that title -- it stands in the other segment -- - # and H1 reads it, so the field-deciding reports have to - # be told (#449 review round). - order = _assign_main(0, state, tokens, ambiguities, - titled=not all(reading)) - else: - for k, piece in enumerate(fam_pieces): - if k > 0 and is_suffix_piece(piece, fam_tags[k], tokens): - _set_roles(tokens, piece, Role.SUFFIX) - else: - _set_roles(tokens, piece, Role.FAMILY) + # Segment 1 is read FIRST, ahead of either branch below. It + # consumes `reading`, piece tags and text only -- nothing + # segment 0's read writes -- and running it first is what puts + # a TITLE role on the title standing after the comma ('John V, + # Dr.') before _assign_main scans for one, so the positional + # read below sees that title the way it sees its own peeled + # ones (#449 review round; it replaced a `titled` keyword). if len(state.segments) > 1: pieces = state.pieces[1] ptags = state.piece_tags[1] @@ -614,6 +553,16 @@ def assign(state: ParseState) -> ParseState: _set_roles(tokens, pieces[m], Role.SUFFIX) else: _set_roles(tokens, pieces[m], Role.MIDDLE) + if reading is not None and sum( + 1 for k, piece in enumerate(fam_pieces) + if not is_suffix_piece(piece, fam_tags[k], tokens)) > 1: + order = _assign_main(0, state, tokens, ambiguities) + else: + for k, piece in enumerate(fam_pieces): + if k > 0 and is_suffix_piece(piece, fam_tags[k], tokens): + _set_roles(tokens, piece, Role.SUFFIX) + else: + _set_roles(tokens, piece, Role.FAMILY) tail = 2 # segments past the structure's name segments are wholly suffixes for seg_idx in range(tail, len(state.segments)): diff --git a/nameparser/_pipeline/_post_rules.py b/nameparser/_pipeline/_post_rules.py index 8efa051b..42d417b7 100644 --- a/nameparser/_pipeline/_post_rules.py +++ b/nameparser/_pipeline/_post_rules.py @@ -26,7 +26,8 @@ from nameparser._lexicon import _title_key from nameparser._pipeline._assign import _name_positions from nameparser._pipeline._state import ( - ParseState, PendingAmbiguity, Structure, WorkToken, comma_bucket, + ParseState, PendingAmbiguity, Structure, WorkToken, _NEVER_FLIPPED, + comma_bucket, ) from nameparser._pipeline._vocab import delimiter_cores from nameparser._policy import PatronymicRule @@ -51,11 +52,6 @@ _NAME_ROLES = (Role.GIVEN, Role.MIDDLE, Role.FAMILY) -#: M4's two carve-outs, as the tags classify recorded them: a bound -#: given-name word is vocabulary claiming the word as a given name, -#: and `initial` is the shape claim. Neither is a predicate M4 owns. -_NEVER_FLIPPED = frozenset({"vocab:bound-given", "initial"}) - #: The roles that are transparent to a run of post-nominals (R1's #: entry pass below). These three roles render into fields other than #: the name and the suffix, so a run of diff --git a/nameparser/_pipeline/_state.py b/nameparser/_pipeline/_state.py index d7da34f3..98054a95 100644 --- a/nameparser/_pipeline/_state.py +++ b/nameparser/_pipeline/_state.py @@ -55,6 +55,16 @@ class WorkToken: role: Role | None = None +#: M4's two carve-outs, as the tags classify recorded them: a bound +#: given-name word is vocabulary claiming the word as a given name, +#: and `initial` is the shape claim. Neither is a predicate M4 owns. +#: Shared here beside WorkToken.tags for the reason COMMA_CHARS is: +#: assign's `_WORD_ALREADY_CLAIMED` is built from this pair, and the +#: two stages must not drift (post_rules imports _assign, so _assign +#: cannot reach the other way). +_NEVER_FLIPPED = frozenset({"vocab:bound-given", "initial"}) + + class Structure(Enum): """segment's comma-structure decision.""" diff --git a/tests/v2/pipeline/test_assign.py b/tests/v2/pipeline/test_assign.py index c55d5e9e..010402c0 100644 --- a/tests/v2/pipeline/test_assign.py +++ b/tests/v2/pipeline/test_assign.py @@ -14,6 +14,16 @@ ) from nameparser._types import AmbiguityKind, Role +#: The three read orders and the role a lone name word takes under +#: each, shared by every parametrized case below that asks the same +#: question of all three: the default (None, given-first), and the two +#: declared family-first orders. +_ORDERS = [ + (None, "given"), + (Policy(name_order=FAMILY_FIRST), "family"), + (Policy(name_order=FAMILY_FIRST_GIVEN_LAST), "family"), +] + _LEX = Lexicon( titles=frozenset({"dr", "mr", "mrs", "sir", "sr"}), given_name_titles=frozenset({"sir"}), @@ -83,11 +93,7 @@ def test_leading_ambiguous_particle_reads_as_given_with_ambiguity() -> None: assert not _assigned("John Smith").ambiguities -@pytest.mark.parametrize("policy,role", [ - (None, "given"), - (Policy(name_order=FAMILY_FIRST), "family"), - (Policy(name_order=FAMILY_FIRST_GIVEN_LAST), "family"), -]) +@pytest.mark.parametrize("policy,role", _ORDERS) def test_leading_particle_detail_names_the_role_it_took( policy: Policy | None, role: str) -> None: # The fork is the same under every order -- particle or name -- @@ -104,11 +110,7 @@ def test_leading_particle_detail_names_the_role_it_took( f"read as a {role} name") -@pytest.mark.parametrize("policy,role", [ - (None, "given"), - (Policy(name_order=FAMILY_FIRST), "family"), - (Policy(name_order=FAMILY_FIRST_GIVEN_LAST), "family"), -]) +@pytest.mark.parametrize("policy,role", _ORDERS) @pytest.mark.parametrize("text,kind,detail", [ ("Andrew", AmbiguityKind.GIVEN_OR_FAMILY, "'Andrew' is the only name word and nothing else decides it; "