From 58872049ffae6e262e0586496e5da0d2a5bbee4a Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Tue, 28 Jul 2026 00:19:27 -0700 Subject: [PATCH 01/20] Add the Script enum and the shared single_script classifier (#271) Co-Authored-By: Claude Fable 5 --- nameparser/__init__.py | 3 ++- nameparser/_pipeline/_vocab.py | 35 ++++++++++++++++++++++++++++++++- nameparser/_policy.py | 18 +++++++++++++++++ tests/v2/pipeline/test_vocab.py | 31 ++++++++++++++++++++++++++++- tests/v2/test_layering.py | 2 +- tests/v2/test_policy.py | 6 +++++- 6 files changed, 90 insertions(+), 5 deletions(-) diff --git a/nameparser/__init__.py b/nameparser/__init__.py index 621f1aec..2401a6da 100644 --- a/nameparser/__init__.py +++ b/nameparser/__init__.py @@ -18,6 +18,7 @@ PatronymicRule, Policy, PolicyPatch, + Script, ) from nameparser._types import ( STABLE_TAGS, @@ -35,7 +36,7 @@ # v2 core "Span", "Role", "Token", "Ambiguity", "AmbiguityKind", "ParsedName", "STABLE_TAGS", - "Lexicon", "Policy", "PolicyPatch", "PatronymicRule", "UNSET", + "Lexicon", "Policy", "PolicyPatch", "PatronymicRule", "Script", "UNSET", "GIVEN_FIRST", "FAMILY_FIRST", "FAMILY_FIRST_GIVEN_LAST", "DEFAULT_NICKNAME_DELIMITERS", "Locale", "Parser", "parse", "parser_for", diff --git a/nameparser/_pipeline/_vocab.py b/nameparser/_pipeline/_vocab.py index 699da052..ce601555 100644 --- a/nameparser/_pipeline/_vocab.py +++ b/nameparser/_pipeline/_vocab.py @@ -4,13 +4,14 @@ predicates live with their stage. All take normalized-or-raw text explicitly -- no state. -Layering: imports _lexicon and _types only. +Layering: imports _lexicon, _types, and _policy only. """ from __future__ import annotations import re from nameparser._lexicon import Lexicon, _normalize +from nameparser._policy import Script # Ported verbatim from v1 (nameparser/config/regexes.py "initial") minus # its empty-string alternative -- WorkToken text is never empty. Kept in @@ -28,6 +29,28 @@ PH = re.compile(r"^ph\.?$", re.IGNORECASE) D = re.compile(r"^d\.?$", re.IGNORECASE) +# Codepoint ranges per Script (#271): integer ranges following the +# _EMOJI_RANGES precedent in _tokenize.py -- the per-char test needs +# no regex. HAN: the URO plus Extension A, the compatibility block, +# and the supplementary-plane block (Ext B-I + CJK Compat Ideographs +# Supplement, 0x20000-0x323AF) -- rare surnames are the biggest real +# source of supplementary-plane hanzi in personal names (e.g. 𠮷田's +# 𠮷, U+20BB7), so leaving them out silently mis-orders those names; +# unassigned gaps inside the span are harmless, since no real name +# contains an unassigned codepoint. HANGUL: precomposed syllables +# only -- modern Korean text never writes names as bare jamo. Kana is +# DELIBERATELY absent: a kana token identifies Japanese, whose +# conventions are #272's segmenter, not this table's. The ranges +# below must stay mutually disjoint: single_script returns the FIRST +# covering entry (dict iteration order), so an overlapping future +# script (e.g. a ja entry that also covers Han) would make the result +# order-dependent instead of well-defined. +_SCRIPT_RANGES: dict[Script, tuple[tuple[int, int], ...]] = { + Script.HAN: ((0x3400, 0x4DBF), (0x4E00, 0x9FFF), (0xF900, 0xFAFF), + (0x20000, 0x323AF)), + Script.HANGUL: ((0xAC00, 0xD7A3),), +} + def is_initial(text: str) -> bool: """'A.' / 'j.' / bare capital -- v1's is_an_initial.""" @@ -124,3 +147,13 @@ def period_joined_vocab(text: str, lexicon: Lexicon) -> str | None: return "suffix" return None + +def single_script(text: str) -> Script | None: + """The one Script whose ranges cover EVERY char of `text`, else + None (mixed-script text has no well-defined convention to apply; + the caller falls back to the positional default).""" + for script, ranges in _SCRIPT_RANGES.items(): + if all(any(lo <= ord(c) <= hi for lo, hi in ranges) + for c in text): + return script + return None diff --git a/nameparser/_policy.py b/nameparser/_policy.py index e750798a..1cddd414 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -29,6 +29,24 @@ class PatronymicRule(StrEnum): TURKIC = "turkic" +class Script(StrEnum): + """Writing systems the parser can key SCRIPT-CONDITIONAL behavior + on: per-script name order (``Policy.script_orders``) and + unspaced-name segmentation (``Policy.segment_scripts``). The rule + that admits these (amendment 2026-07-27): script-conditional + behavior only where the script itself determines the convention -- + Latin-script input is never affected. Character tables live in + nameparser/_pipeline/_vocab.py, not here.""" + + #: Chinese Hanzi -- and Japanese Kanji: a pure-Han string cannot + #: say which language it is, which is fine for ORDER (both write + #: family-first natively) and exactly why Han SEGMENTATION is + #: opt-in (locales.ZH; Japanese is #272's pluggable segmenter). + HAN = "han" + #: Korean Hangul (precomposed syllables). Unambiguously Korean. + HANGUL = "hangul" + + # Order-spec constants (#270). Each reads as its contents because roles # are named given/family, not first/last. diff --git a/tests/v2/pipeline/test_vocab.py b/tests/v2/pipeline/test_vocab.py index a8e21326..e00202e1 100644 --- a/tests/v2/pipeline/test_vocab.py +++ b/tests/v2/pipeline/test_vocab.py @@ -1,5 +1,8 @@ from nameparser._lexicon import Lexicon -from nameparser._pipeline._vocab import is_initial, is_suffix_lenient, is_suffix_strict +from nameparser._pipeline._vocab import ( + is_initial, is_suffix_lenient, is_suffix_strict, single_script, +) +from nameparser._policy import Script _LEX = Lexicon( suffix_acronyms=frozenset({"phd", "ma"}), @@ -38,3 +41,29 @@ def test_strict_excludes_bare_ambiguous_even_when_in_acronyms() -> None: # mirrors the real data shape: ambiguous is a SUBSET of acronyms assert not is_suffix_strict("Ma", _LEX) assert is_suffix_strict("M.A.", _LEX) + + +def test_single_script_requires_every_char_in_one_script() -> None: + assert single_script("毛泽东") is Script.HAN + assert single_script("諸葛") is Script.HAN # traditional + assert single_script("김민준") is Script.HANGUL + assert single_script("Smith") is None + assert single_script("毛zedong") is None # mixed + assert single_script("毛김") is None # mixed CJK + assert single_script("イチロー") is None # kana: not HAN + + +def test_single_script_range_edges() -> None: + # one char at each declared bound, and a neighbour outside + assert single_script("㐀") is Script.HAN # Ext A first + assert single_script("䶿") is Script.HAN # Ext A last + assert single_script("䷀") is None # hexagram, not Han + # U+F900 is a COMPATIBILITY ideograph; it renders identically to + # the URO 豈 (U+8C48) it decomposes to, so spell it as an escape + # or this assertion silently tests the URO range instead + assert single_script("\uf900") is Script.HAN + assert single_script("\U00020bb7") is Script.HAN # Ext B (𠮷) + assert single_script("가") is Script.HANGUL + assert single_script("힣") is Script.HANGUL # last ASSIGNED syllable + assert single_script("ힰ") is None # jungseong, not a syllable + assert single_script("ㄱ") is None # bare jamo diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index 7ddcf910..9eeedb46 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -165,7 +165,7 @@ def test_lexicon_never_imports_config_package_root_or_parser() -> None: def test_public_exports() -> None: expected = { "Span", "Role", "Token", "Ambiguity", "AmbiguityKind", "ParsedName", - "Lexicon", "Policy", "PolicyPatch", "PatronymicRule", "UNSET", + "Lexicon", "Policy", "PolicyPatch", "PatronymicRule", "Script", "UNSET", "GIVEN_FIRST", "FAMILY_FIRST", "FAMILY_FIRST_GIVEN_LAST", "DEFAULT_NICKNAME_DELIMITERS", "Locale", "Parser", "parse", "parser_for", diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index 01a8554a..c50d887a 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -4,7 +4,7 @@ from nameparser._policy import ( FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST, GIVEN_FIRST, - PatronymicRule, Policy, PolicyPatch, UNSET, apply_patch, + PatronymicRule, Policy, PolicyPatch, Script, UNSET, apply_patch, ) from nameparser._types import Role @@ -59,6 +59,10 @@ def test_name_order_rejects_plain_string_tuples() -> None: Policy(name_order=("given", "middle", "family")) # type: ignore[arg-type] +def test_script_values_are_the_public_names() -> None: + assert Script.HAN == "han" and Script.HANGUL == "hangul" + + def test_patronymic_rules_coerce_and_reject() -> None: p = Policy(patronymic_rules=frozenset({"east-slavic"})) # type: ignore[arg-type] assert p.patronymic_rules == frozenset({PatronymicRule.EAST_SLAVIC}) From cef68f30690228fb09418cb0f1404f2754f6213b Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Tue, 28 Jul 2026 00:43:28 -0700 Subject: [PATCH 02/20] Read wholly-CJK names family-first by default via Policy.script_orders (#271) Co-Authored-By: Claude Fable 5 --- nameparser/__init__.py | 3 +- nameparser/_pipeline/_assign.py | 44 ++++++- nameparser/_pipeline/_vocab.py | 2 + nameparser/_policy.py | 207 ++++++++++++++++++++++++++----- tests/v2/cases.py | 13 +- tests/v2/pipeline/test_assign.py | 35 +++++- tests/v2/pipeline/test_vocab.py | 4 + tests/v2/test_facade.py | 12 ++ tests/v2/test_facade_cases.py | 4 + tests/v2/test_layering.py | 2 +- tests/v2/test_parser.py | 72 +++++++++++ tests/v2/test_policy.py | 74 ++++++++++- 12 files changed, 429 insertions(+), 43 deletions(-) diff --git a/nameparser/__init__.py b/nameparser/__init__.py index 2401a6da..a433a76e 100644 --- a/nameparser/__init__.py +++ b/nameparser/__init__.py @@ -11,6 +11,7 @@ from nameparser._parser import Parser, parse, parser_for from nameparser._policy import ( DEFAULT_NICKNAME_DELIMITERS, + DEFAULT_SCRIPT_ORDERS, FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST, GIVEN_FIRST, @@ -38,6 +39,6 @@ "STABLE_TAGS", "Lexicon", "Policy", "PolicyPatch", "PatronymicRule", "Script", "UNSET", "GIVEN_FIRST", "FAMILY_FIRST", "FAMILY_FIRST_GIVEN_LAST", - "DEFAULT_NICKNAME_DELIMITERS", "Locale", + "DEFAULT_NICKNAME_DELIMITERS", "DEFAULT_SCRIPT_ORDERS", "Locale", "Parser", "parse", "parser_for", ] diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index 266e7d54..c4ce09bb 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -2,8 +2,10 @@ Consumes: pieces + piece_tags (grouped), segments, structure, tokens. Produces: tokens with roles set on every main-stream token. -Reads: Policy.name_order (#270); token/piece tags; Lexicon only through -tags already applied by classify (plus the leading-title period rule). +Reads: Policy.name_order (#270) and Policy.script_orders (#271, which +overrides it when every name piece is written wholly in one script); +token/piece tags; Lexicon only through tags already applied by classify +(plus the leading-title period rule). Ports v1's assignment loops. NO_COMMA (per name_order): leading title pieces chain while no given-position name has been seen @@ -27,13 +29,14 @@ import dataclasses import re -from nameparser._pipeline._vocab import is_suffix_lenient +from nameparser._pipeline._vocab import is_suffix_lenient, single_script from nameparser._pipeline._group import ( _is_suffix_piece, _is_title_piece, ) from nameparser._pipeline._state import ( ParseState, PendingAmbiguity, Structure, WorkToken, ) +from nameparser._policy import Policy from nameparser._types import AmbiguityKind, Role # Ported verbatim from v1 (nameparser/config/regexes.py @@ -74,6 +77,30 @@ def _peel_leading_titles(pieces: tuple[tuple[int, ...], ...], return n +def _effective_order(policy: Policy, + pieces: list[tuple[int, ...]], + tokens: list[WorkToken]) -> tuple[Role, Role, Role]: + """script_orders resolution (#271): when every name piece is + written wholly in ONE script that has an entry, that script's + order governs the positional read; anything else -- Latin, mixed + scripts, no entry -- falls back to name_order. Piece-level, after + title/suffix peeling: 'Dr. 毛泽东' is a wholly-Han NAME under a + Latin title.""" + if not policy.script_orders: + return policy.name_order + # ONE script for the whole name, not "the entries all agree": two + # scripts that both read family-first still fall back, because a + # Han+Hangul name is not written in either tradition. + found = {single_script("".join(tokens[i].text for i in piece)) + for piece in pieces} + if len(found) == 1: + script = found.pop() # None (no single script) matches + for entry_script, order in policy.script_orders: # no entry + if entry_script is script: + return order + return policy.name_order + + def _name_positions(order: tuple[Role, Role, Role], count: int) -> list[Role]: """Roles for `count` name pieces (titles/suffixes already peeled), @@ -175,7 +202,12 @@ def _assign_main(seg_idx: int, state: ParseState, if not name_pieces and suffix_pieces: # everything suffix-shaped after titles: first one is the name name_pieces, suffix_pieces = suffix_pieces[:1], suffix_pieces[1:] - roles = _name_positions(state.policy.name_order, len(name_pieces)) + # AFTER both peels, and load-bearing: the script test sees the NAME + # pieces only, so a Latin title or suffix ('Dr. 毛 泽东', '毛 泽东, + # PhD') cannot make a wholly-CJK name look mixed-script. + order = _effective_order(state.policy, + [pieces[i] for i in name_pieces], tokens) + roles = _name_positions(order, len(name_pieces)) for pos, piece_idx in enumerate(name_pieces): _set_roles(tokens, pieces[piece_idx], roles[pos]) for piece_idx in suffix_pieces: @@ -219,7 +251,9 @@ def assign(state: ParseState) -> ParseState: else: # FAMILY_COMMA # PARTICLE_OR_GIVEN is deliberately not emitted here: after a # comma the family is already fixed, so a leading given-position - # particle is not meaningfully ambiguous. + # particle is not meaningfully ambiguous. script_orders is not + # consulted here for the parallel reason -- the comma already + # fixed the family, so there is no positional read to override. # v1: "lastname part may have suffixes in it" -- the first # piece is always the family even if suffix-shaped; any later # strict-suffix piece goes to SUFFIX per piece ('Smith Jr., diff --git a/nameparser/_pipeline/_vocab.py b/nameparser/_pipeline/_vocab.py index ce601555..abbd72bc 100644 --- a/nameparser/_pipeline/_vocab.py +++ b/nameparser/_pipeline/_vocab.py @@ -152,6 +152,8 @@ def single_script(text: str) -> Script | None: """The one Script whose ranges cover EVERY char of `text`, else None (mixed-script text has no well-defined convention to apply; the caller falls back to the positional default).""" + if not text: + return None # all() is vacuously true; "" belongs to no script for script, ranges in _SCRIPT_RANGES.items(): if all(any(lo <= ord(c) <= hi for lo, hi in ranges) for c in text): diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 1cddd414..fd12a98c 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -100,6 +100,19 @@ def _order_repr(value: tuple[Role, ...]) -> str: "parser_for(locales.RU) / locales.TR_AZ)" ) +#: Policy.script_orders' default: wholly-Han and wholly-Hangul names +#: read family-first. Public and named so opting out or extending +#: reads against a documented value (the DEFAULT_NICKNAME_DELIMITERS +#: precedent). The HAN entry is safe WITHOUT knowing Chinese from +#: Japanese: both write family-first in native script -- the +#: languages differ, the convention doesn't. Canonical form: sorted +#: (Script, order) pairs, matching the field's storage. +DEFAULT_SCRIPT_ORDERS: tuple[ + tuple[Script, tuple[Role, Role, Role]], ...] = ( + (Script.HAN, FAMILY_FIRST), + (Script.HANGUL, FAMILY_FIRST), +) + #: Policy.nickname_delimiters' default. Public and named so #: customizations read as set math against a documented value -- e.g. #: ``DEFAULT_NICKNAME_DELIMITERS | {("｟", "｠")}`` -- instead of a @@ -121,13 +134,16 @@ def _order_repr(value: tuple[Role, ...]) -> str: }) -def _reject_bare_string_order(value: object) -> None: +def _reject_bare_string_order(value: object, field_name: str) -> None: # tuple("gmf") would be ("g", "m", "f") -- catch the bare string # with the same TypeError every other iterable field raises. # Single-sourced: called from Policy AND PolicyPatch __post_init__. + # field_name is REQUIRED, with no name_order default: script_orders' + # values obey the same rule (via _validated_order), and a defaulted + # caller that forgot to pass it would silently name the wrong field. if isinstance(value, str): raise TypeError( - f"name_order must be an iterable of three Roles, " + f"{field_name} must be an iterable of three Roles, " f"not a bare string: {value!r}" ) # A {Role: position} dict iterates to the right three Roles in the @@ -138,12 +154,12 @@ def _reject_bare_string_order(value: object) -> None: # out of it is how the PolicyPatch hole happened. if isinstance(value, Mapping): raise TypeError( - f"name_order must be an iterable of three Roles, not a " + f"{field_name} must be an iterable of three Roles, not a " f"mapping: {value!r}" ) if isinstance(value, (bytes, bytearray, memoryview)): raise TypeError( - f"name_order must be an iterable of three Roles, not " + f"{field_name} must be an iterable of three Roles, not " f"{type(value).__name__} -- decode first, e.g. " f"raw.decode('utf-8')" ) @@ -201,6 +217,121 @@ def _require_iterable(value: Iterable[Any], field_name: str) -> Iterable[Any]: ) from None +def _validated_order(value: Iterable[Any], + field_name: str) -> tuple[Role, Role, Role]: + """name_order's element/permutation check, single-sourced so + script_orders values obey the identical rule (only the three + exported orders have implemented assignment semantics).""" + _reject_bare_string_order(value, field_name) + order = tuple(_require_iterable(value, field_name)) + # Sole rejection point for plain-string tuples: Role is a StrEnum, + # so the named-order membership check below compares EQUAL for + # ("given", "middle", "family") -- do not remove this loop as + # redundant. + for element in order: + if not isinstance(element, Role): + raise TypeError( + f"{field_name} elements must be Role members, " + f"got {element!r}" + ) + # Only the three exported orders have implemented assignment + # semantics; the unnamed permutations would silently misassign. + # Pre-2.0 strictness is free -- relaxing later is compatible. + if order not in (GIVEN_FIRST, FAMILY_FIRST, + FAMILY_FIRST_GIVEN_LAST): + raise ValueError( + f"{field_name} must be one of the exported orders, got " + f"{order!r}; use GIVEN_FIRST, FAMILY_FIRST, or " + f"FAMILY_FIRST_GIVEN_LAST" + ) + return order # type: ignore[return-value] # length checked above + + +def _validated_script(key: object) -> Script: + """Coerce one Script key, with the message every script-keyed field + shares. Single-sourced deliberately: script_orders and the + script-keyed fields that follow it must not each grow their own + wording for the same lookup.""" + try: + # Enum lookup accepts ANY value at runtime and answers a + # non-member with ValueError -- which is the contract here, and + # the taxonomy's rule for a failed enum lookup whatever the + # input type was (stdlib EnumType precedent). + return Script(key) # type: ignore[arg-type] + except ValueError: + valid = ", ".join(v.value for v in Script) + raise ValueError( + f"unknown script {key!r}; valid scripts: {valid}" + ) from None + + +def _validated_script_orders( + value: object) -> tuple[tuple[Script, tuple[Role, Role, Role]], ...]: + """Policy.script_orders' whole check, from raw input to canonical + storage. script_orders is the one MAPPING-shaped field, so the + guards read inverted from every other one here: a Mapping is what + the caller SHOULD pass, and a bare string is the shape that would + otherwise iterate into plausible-looking garbage.""" + if isinstance(value, str): + raise TypeError( + f"script_orders must be a mapping of Script to order, " + f"not a bare string: {value!r}" + ) + if isinstance(value, (bytes, bytearray, memoryview)): + raise TypeError( + f"script_orders must be a mapping of Script to order, " + f"not {type(value).__name__} -- decode first, " + f"e.g. raw.decode('utf-8')" + ) + raw = value.items() if isinstance(value, Mapping) else value + try: + entries: tuple[Any, ...] = tuple(raw) # type: ignore[arg-type] + except TypeError: + raise TypeError( + f"script_orders must be a mapping of Script to order, " + f"got {value!r}" + ) from None + canonical: dict[Script, tuple[Role, Role, Role]] = {} + for entry in entries: + try: + key, order = entry + except (TypeError, ValueError): + raise TypeError( + f"script_orders entries must be (Script, order) " + f"pairs, got {entry!r}" + ) from None + canonical[_validated_script(key)] = _validated_order( + order, "script_orders") + # Sorted pairs, not a dict: Policy is hashable, and two + # differently-written but equivalent tables must converge (the + # capitalization_exceptions precedent). + return tuple(sorted(canonical.items())) + + +def _canonical_script_pair(pair: Iterable[Any]) -> tuple[Any, ...]: + """Hashability-only canonicalization of one PolicyPatch + script_orders entry: tuple-ize the pair AND the order value inside + it. Shallow was not enough -- a {Script: [Role, ...]} patch stored + the list, and hash() then raised far from the construction site, + the same failure name_order's canonicalization exists to prevent. + A malformed entry is still tuple-ized (that is the hashability + floor); its CONTENTS are left exactly as written so Policy quotes + the caller's own value when it raises at apply time.""" + out = tuple(pair) + if len(out) != 2: + return out # not a (Script, order) pair + key, value = out + # str/bytes are iterable, so tuple() would shred exactly the two + # values whose deferred errors ("not a bare string", the decode + # hint) need to quote what the caller wrote. + if isinstance(value, (str, bytes, bytearray, memoryview)): + return out + try: + return (key, tuple(value)) + except TypeError: + return out # non-iterable order value + + @dataclass(frozen=True, slots=True) class Policy: """The behavior switches a parser runs with: name order, @@ -220,6 +351,16 @@ class Policy: #: given or family names). A comma that only sets off suffixes #: ("John Smith, Jr.") leaves name_order governing the name part. name_order: tuple[Role, Role, Role] = GIVEN_FIRST + #: Per-script overrides of name_order (#271), consulted when every + #: name piece is written wholly in one script: {Script: order} + #: (constructor accepts a mapping; stored as sorted pairs). The + #: default reads wholly-Han/Hangul names family-first -- see + #: :data:`~nameparser.DEFAULT_SCRIPT_ORDERS`. Opt out with + #: ``script_orders={}``. Latin-script and mixed-script input is + #: never affected. Like name_order, ignored where a comma already + #: decides the family name. + script_orders: tuple[tuple[Script, tuple[Role, Role, Role]], ...] = ( + DEFAULT_SCRIPT_ORDERS) #: Opt-in detectors that reorder patronymic-shaped names #: (EAST_SLAVIC, TURKIC); usually set via a locale pack. patronymic_rules: frozenset[PatronymicRule] = frozenset() @@ -266,29 +407,12 @@ class Policy: __setstate__ = _guarded_setstate def __post_init__(self) -> None: - _reject_bare_string_order(self.name_order) - order = tuple(_require_iterable(self.name_order, "name_order")) - # Sole rejection point for plain-string tuples: Role is a StrEnum, - # so the named-order membership check below compares EQUAL for - # ("given", "middle", "family") -- do not remove this loop as - # redundant. - for element in order: - if not isinstance(element, Role): - raise TypeError( - f"name_order elements must be Role members, " - f"got {element!r}" - ) - # Only the three exported orders have implemented assignment - # semantics; the unnamed permutations would silently misassign. - # Pre-2.0 strictness is free -- relaxing later is compatible. - if order not in (GIVEN_FIRST, FAMILY_FIRST, - FAMILY_FIRST_GIVEN_LAST): - raise ValueError( - f"name_order must be one of the exported orders, got " - f"{order!r}; use GIVEN_FIRST, FAMILY_FIRST, or " - f"FAMILY_FIRST_GIVEN_LAST" - ) - object.__setattr__(self, "name_order", order) + object.__setattr__( + self, "name_order", _validated_order(self.name_order, + "name_order")) + object.__setattr__( + self, "script_orders", + _validated_script_orders(self.script_orders)) _reject_str_and_mapping(self.patronymic_rules, "patronymic_rules") # Probe with iter() rather than wrapping tuple(): non-iterables # (True especially -- v1's patronymic_name_order was a bool flag, @@ -423,6 +547,12 @@ class PolicyPatch: """ name_order: tuple[Role, Role, Role] | _Unset = UNSET + #: Composes as a SCALAR (override, not merge) -- deliberate: nothing + #: shipped patches it today, so the simpler rule is the one to + #: defend; revisit if a pack ever needs to add one script's entry + #: without restating the rest. + script_orders: tuple[ + tuple[Script, tuple[Role, Role, Role]], ...] | _Unset = UNSET patronymic_rules: frozenset[PatronymicRule] | _Unset = field( default=UNSET, metadata=_UNION) middle_as_family: bool | _Unset = UNSET @@ -447,8 +577,29 @@ def __post_init__(self) -> None: # coerce a list at apply time, but the patch itself (and any # Locale holding it) must already be hashable. if self.name_order is not UNSET: - _reject_bare_string_order(self.name_order) + _reject_bare_string_order(self.name_order, "name_order") object.__setattr__(self, "name_order", tuple(self.name_order)) + # Same reason for script_orders, one level deeper: a patch built + # from a {Script: order} dict (or a list of pairs) must already + # be hashable, since a Locale holds it -- and hashable all the + # way down, hence _canonical_script_pair. Validation still + # belongs to Policy at apply time, so a shape tuple() cannot + # digest is left exactly as written for it to report. + # The str case must be excluded HERE rather than delegated to + # _canonical_script_pair: a string's elements are themselves + # tuple-izable, so no TypeError ever fires to signal "leave this + # alone" -- "han" would shred to (("h",), ("a",), ("n",)) and + # Policy's bare-string message would have nothing left to quote. + if self.script_orders is not UNSET and not isinstance( + self.script_orders, str): + raw = self.script_orders + pairs = raw.items() if isinstance(raw, Mapping) else raw + try: + object.__setattr__( + self, "script_orders", + tuple(_canonical_script_pair(p) for p in pairs)) + except TypeError: + pass # Policy validates (and raises properly) at apply for f in dataclasses.fields(self): if f.metadata.get("compose") != "union": continue diff --git a/tests/v2/cases.py b/tests/v2/cases.py index cc2d53f6..8756dead 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -208,13 +208,16 @@ def __post_init__(self) -> None: {"given": "Anna", "family": "Larsson", "nickname": "Ann"}, classification="feat(#273)"), Case("cjk_corner_bracket_nickname", '山田「タロ」太郎', - {"given": "山田", "family": "太郎", "nickname": "タロ"}, - classification="feat(#273)", + {"family": "山田", "given": "太郎", "nickname": "タロ"}, + classification="feat(#273) + fix(#271)", notes="extraction also splits the unspaced remainder -- the " - "masked region acts as a token boundary"), + "masked region acts as a token boundary. Both name " + "pieces are then wholly Han, so script_orders reads " + "them family-first; the nickname's kana is outside the " + "name pieces and does not enter that test"), Case("cjk_white_corner_bracket_nickname", '田中『ハナ』花子', - {"given": "田中", "family": "花子", "nickname": "ハナ"}, - classification="feat(#273)"), + {"family": "田中", "given": "花子", "nickname": "ハナ"}, + classification="feat(#273) + fix(#271)"), Case("fullwidth_paren_nickname", 'John （Jack） Kennedy', {"given": "John", "family": "Kennedy", "nickname": "Jack"}, classification="feat(#273)"), diff --git a/tests/v2/pipeline/test_assign.py b/tests/v2/pipeline/test_assign.py index b506b5a2..c0508907 100644 --- a/tests/v2/pipeline/test_assign.py +++ b/tests/v2/pipeline/test_assign.py @@ -7,7 +7,9 @@ from nameparser._pipeline._segment import segment from nameparser._pipeline._state import ParseState from nameparser._pipeline._tokenize import tokenize -from nameparser._policy import FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST, Policy +from nameparser._policy import ( + FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST, Policy, Script, +) from nameparser._types import AmbiguityKind, Role _LEX = Lexicon( @@ -169,3 +171,34 @@ def test_family_first_given_last_order() -> None: assert _by_role(out, Role.FAMILY) == "Nguyen" assert _by_role(out, Role.GIVEN) == "Van Anh Thu" assert _by_role(out, Role.MIDDLE) == "" + + +def test_script_order_applies_when_every_piece_is_one_script() -> None: + out = _assigned("毛 泽东") + assert _by_role(out, Role.FAMILY) == "毛" + assert _by_role(out, Role.GIVEN) == "泽东" + + +def test_script_order_declines_on_a_mixed_piece_set() -> None: + # {None, HAN}: one Latin piece is enough to put the name back on + # the positional default, even though the other piece is Han. + out = _assigned("毛 Smith") + assert _by_role(out, Role.GIVEN) == "毛" + assert _by_role(out, Role.FAMILY) == "Smith" + + +def test_script_order_declines_when_no_piece_has_a_script() -> None: + # all-None: the ordinary Latin path, unreachable by the table. + out = _assigned("John Smith") + assert _by_role(out, Role.GIVEN) == "John" + assert _by_role(out, Role.FAMILY) == "Smith" + + +def test_script_with_no_table_entry_falls_back() -> None: + # A single, well-defined script the table simply does not list: + # resolution must fall through to name_order rather than pick an + # arbitrary entry. + hangul_only = Policy(script_orders={Script.HANGUL: FAMILY_FIRST}) # type: ignore[arg-type] + out = _assigned("毛 泽东", hangul_only) + assert _by_role(out, Role.GIVEN) == "毛" + assert _by_role(out, Role.FAMILY) == "泽东" diff --git a/tests/v2/pipeline/test_vocab.py b/tests/v2/pipeline/test_vocab.py index e00202e1..bee3cb64 100644 --- a/tests/v2/pipeline/test_vocab.py +++ b/tests/v2/pipeline/test_vocab.py @@ -67,3 +67,7 @@ def test_single_script_range_edges() -> None: assert single_script("힣") is Script.HANGUL # last ASSIGNED syllable assert single_script("ힰ") is None # jungseong, not a syllable assert single_script("ㄱ") is None # bare jamo + + +def test_single_script_empty_string_is_no_script() -> None: + assert single_script("") is None diff --git a/tests/v2/test_facade.py b/tests/v2/test_facade.py index aae2c511..7ca9ac9b 100644 --- a/tests/v2/test_facade.py +++ b/tests/v2/test_facade.py @@ -559,3 +559,15 @@ def test_cold_import_order_config_first() -> None: capture_output=True, text=True) assert proc.returncode == 0, (first, proc.stderr) assert proc.stdout.strip() == "Smith" + + +def test_facade_reads_wholly_han_names_family_first() -> None: + # classified fix(#271): script_orders reaches the v1 surface. + # Constants has no opt-out knob (v1 surface is frozen); the + # migration path is v2 Policy(script_orders={}). + n = HumanName("毛 泽东") + assert (n.last, n.first) == ("毛", "泽东") + # unsplit (Han segmentation is locales.ZH, v2-only), but the lone + # wholly-Han token now takes the family role, not first + assert HumanName("毛泽东").last == "毛泽东" + assert HumanName("毛泽东").first == "" diff --git a/tests/v2/test_facade_cases.py b/tests/v2/test_facade_cases.py index 90932553..b57d19db 100644 --- a/tests/v2/test_facade_cases.py +++ b/tests/v2/test_facade_cases.py @@ -30,6 +30,10 @@ def _constants_for(case: Case) -> Constants | None: ) unexpressible = ( policy.name_order != default.name_order + # script_orders has no v1 Constants spelling at all (the v1 + # surface is frozen), so a row opting out must SKIP here rather + # than fail against the facade's inherited default. + or policy.script_orders != default.script_orders or policy.lenient_comma_suffixes != default.lenient_comma_suffixes or policy.strip_emoji != default.strip_emoji or policy.strip_bidi != default.strip_bidi diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index 9eeedb46..c740db5a 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -167,7 +167,7 @@ def test_public_exports() -> None: "Span", "Role", "Token", "Ambiguity", "AmbiguityKind", "ParsedName", "Lexicon", "Policy", "PolicyPatch", "PatronymicRule", "Script", "UNSET", "GIVEN_FIRST", "FAMILY_FIRST", "FAMILY_FIRST_GIVEN_LAST", - "DEFAULT_NICKNAME_DELIMITERS", "Locale", + "DEFAULT_NICKNAME_DELIMITERS", "DEFAULT_SCRIPT_ORDERS", "Locale", "Parser", "parse", "parser_for", } assert expected <= set(nameparser.__all__) diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index 152cdaa5..2e08e2bc 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -436,3 +436,75 @@ def test_revise_forces_the_named_role_on_every_harvested_token() -> None: r = p.revise(p.parse("John Smith"), family="Dr. Vega Jr.") assert r.family == "Dr. Vega Jr." assert all(t.role is Role.FAMILY for t in r.tokens_for(Role.FAMILY)) + + +def test_wholly_cjk_names_read_family_first_by_default() -> None: + # the 2026-07-27 amendment: script determines the convention, so + # no pack is needed -- release-log-classified fix (#271) + n = parse("毛 泽东") + assert (n.family, n.given) == ("毛", "泽东") + n = parse("김 민준") + assert (n.family, n.given) == ("김", "민준") + # a lone wholly-CJK token takes the script order's first role + assert parse("毛泽东").family == "毛泽东" # unspaced: split arrives + # with Task 4's stage + + +def test_latin_names_are_untouched_by_script_orders() -> None: + n = parse("John Smith") + assert (n.given, n.family) == ("John", "Smith") + # mixed-script names fall back to name_order too + n = parse("John 王") + assert (n.given, n.family) == ("John", "王") + + +def test_script_order_survives_latin_titles_and_suffixes() -> None: + # The script test runs on the NAME pieces, after both peels, so a + # Latin title or post-nominal cannot make the name look mixed. + n = parse("Dr. 毛 泽东") + assert (n.title, n.family, n.given) == ("Dr.", "毛", "泽东") + n = parse("毛 泽东, PhD") + assert (n.family, n.given, n.suffix) == ("毛", "泽东", "PhD") + + +def test_a_comma_still_decides_the_family_name_for_cjk() -> None: + # The family-comma structure fixes the family before any positional + # read, so the table is never consulted -- same rule name_order has. + n = parse("泽东, 毛") + assert (n.family, n.given) == ("泽东", "毛") + + +def test_three_cjk_pieces_take_the_script_order_middles() -> None: + n = parse("毛 泽东 泽民") + assert (n.family, n.given, n.middle) == ("毛", "泽东", "泽民") + + +def test_two_cjk_scripts_fall_back_even_though_both_read_family_first() -> None: + # The rule is ONE script for the whole name, not "the entries + # agree": a Han+Hangul name is written in neither tradition, so it + # takes the positional default. + n = parse("毛 김") + assert (n.given, n.family) == ("毛", "김") + + +def test_a_hyphen_in_a_name_piece_declines_the_script_order() -> None: + # Documenting the conservative direction, not proposing it: ANY + # non-CJK character in a name piece (here the hyphen) puts that + # piece in no script, so the piece set has two members and + # script_orders declines in favour of the positional default. + n = parse("毛 泽东-泽民") + assert (n.given, n.family) == ("毛", "泽东-泽民") + + +def test_script_orders_opt_out_restores_positional_reading() -> None: + p = Parser(policy=Policy(script_orders={})) # type: ignore[arg-type] + n = p.parse("毛 泽东") + assert (n.given, n.family) == ("毛", "泽东") + + +def test_script_order_beats_explicit_global_name_order() -> None: + # the script entry is the more specific rule; opting out means + # script_orders={}, not a different name_order + p = Parser(policy=Policy(name_order=FAMILY_FIRST_GIVEN_LAST)) + n = p.parse("김 민준") + assert (n.family, n.given) == ("김", "민준") diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index c50d887a..31f2d23c 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -3,8 +3,9 @@ import pytest from nameparser._policy import ( - FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST, GIVEN_FIRST, - PatronymicRule, Policy, PolicyPatch, Script, UNSET, apply_patch, + DEFAULT_SCRIPT_ORDERS, FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST, + GIVEN_FIRST, PatronymicRule, Policy, PolicyPatch, Script, UNSET, + apply_patch, ) from nameparser._types import Role @@ -274,6 +275,52 @@ def test_policy_patch_canonicalizes_scalar_name_order() -> None: assert isinstance(hash(p), int) +def test_policy_patch_canonicalizes_script_orders_all_the_way_down() -> None: + # Same failure as name_order above, one level deeper: tuple-izing + # only the (Script, order) pair left the ORDER a list, so the patch + # -- and any Locale holding it -- was still unhashable. + p = PolicyPatch(script_orders={Script.HAN: [Role.FAMILY, Role.GIVEN, # type: ignore[arg-type] + Role.MIDDLE]}) + assert p.script_orders == ((Script.HAN, FAMILY_FIRST),) + assert isinstance(hash(p), int) + + +def test_apply_patch_overrides_script_orders_as_a_scalar() -> None: + # script_orders composes as a SCALAR: a patch REPLACES the table + # rather than merging into it, so an empty one is how a pack turns + # the default off. An UNSET patch must leave the default alone -- + # the distinction a union field would blur. + cleared = apply_patch(Policy(), PolicyPatch(script_orders={})) # type: ignore[arg-type] + assert cleared.script_orders == () + untouched = apply_patch(Policy(), PolicyPatch()) + assert untouched.script_orders == DEFAULT_SCRIPT_ORDERS + replaced = apply_patch( + Policy(), PolicyPatch(script_orders={Script.HANGUL: GIVEN_FIRST})) # type: ignore[arg-type] + assert replaced.script_orders == ((Script.HANGUL, GIVEN_FIRST),) + + +def test_apply_patch_revalidates_deferred_script_orders() -> None: + # The value half defers exactly like name_order's, and the error + # must name script_orders -- not name_order, whose message the + # check is shared with. + bad = PolicyPatch(script_orders={Script.HAN: (Role.MIDDLE, Role.GIVEN, # type: ignore[arg-type] + Role.FAMILY)}) + with pytest.raises(ValueError, match="script_orders must be one of"): + apply_patch(Policy(), bad) + + +def test_policy_patch_defers_malformed_script_orders_verbatim() -> None: + # The canonicalization must not eat the evidence: a bare string + # would shred to character tuples (its elements are themselves + # tuple-izable, so nothing raises to stop it) and bytes to ints, + # leaving Policy's curated messages nothing of the caller's to + # quote. Both shapes must survive the patch and raise at APPLY. + with pytest.raises(TypeError, match="bare string"): + apply_patch(Policy(), PolicyPatch(script_orders="han")) # type: ignore[arg-type] + with pytest.raises(TypeError, match="decode first"): + apply_patch(Policy(), PolicyPatch(script_orders=b"han")) # type: ignore[arg-type] + + def test_apply_patch_revalidates_deferred_values() -> None: # PolicyPatch documents lazy validation: invalid values sit latent in # the patch and must fail when applied, not silently flow into Policy. @@ -450,3 +497,26 @@ def test_patched_validates_patch_values_at_apply_time() -> None: def test_patched_rejects_non_patch() -> None: with pytest.raises(TypeError, match="takes a PolicyPatch"): Policy().patched({"strip_emoji": False}) # type: ignore[arg-type] + + +def test_script_orders_default_and_canonical_storage() -> None: + p = Policy() + assert p.script_orders == DEFAULT_SCRIPT_ORDERS + # constructor tolerates a Mapping; storage is the sorted pair + # tuple (hashability, the capitalization_exceptions precedent -- + # which is also why the mapping spelling needs the ignore: the + # field is annotated with what it STORES) + q = Policy(script_orders={Script.HANGUL: FAMILY_FIRST, # type: ignore[arg-type] + Script.HAN: FAMILY_FIRST}) + assert q == p and hash(q) == hash(p) + assert Policy(script_orders={}).script_orders == () # type: ignore[arg-type] + + +def test_script_orders_validates_keys_and_values() -> None: + with pytest.raises(ValueError, match="han, hangul"): + Policy(script_orders={"klingon": FAMILY_FIRST}) # type: ignore[arg-type] + with pytest.raises(ValueError, match="exported orders"): + Policy(script_orders={Script.HAN: (Role.MIDDLE, Role.GIVEN, # type: ignore[arg-type] + Role.FAMILY)}) + with pytest.raises(TypeError, match="bare string"): + Policy(script_orders="han") # type: ignore[arg-type] From 79533ebdf69dcfbb48da34839b182a270eaa87a1 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Tue, 28 Jul 2026 01:09:24 -0700 Subject: [PATCH 03/20] Add Lexicon.surnames and Policy.segment_scripts (#271) Co-Authored-By: Claude Fable 5 --- nameparser/_lexicon.py | 11 +++++++++- nameparser/_policy.py | 38 +++++++++++++++++++++++++++++++++ tests/v2/test_lexicon.py | 8 +++++++ tests/v2/test_policy.py | 45 +++++++++++++++++++++++++++++++++++++--- 4 files changed, 98 insertions(+), 4 deletions(-) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index cde2c75a..7fd00314 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -23,7 +23,7 @@ _VOCAB_FIELDS = ( "titles", "given_name_titles", "suffix_acronyms", "suffix_words", "suffix_acronyms_ambiguous", "particles", "particles_ambiguous", - "conjunctions", "bound_given_names", "maiden_markers", + "conjunctions", "bound_given_names", "maiden_markers", "surnames", ) #: (marker, base, why) triples. Each marker narrows how entries of its @@ -327,6 +327,15 @@ class Lexicon: #: field ("née", "geb.", "roz.", ...). Full default list: #: :data:`~nameparser.config.maiden_markers.MAIDEN_MARKERS`. maiden_markers: frozenset[str] = frozenset() + #: Family names for the unspaced-name segmentation stage (#271), + #: matched longest-first against the start of the FIRST token + #: written wholly in a script :attr:`Policy.segment_scripts + #: ` activates. The default + #: carries the Korean census list + #: (:data:`~nameparser.config.surnames.KOREAN_SURNAMES`, wired in + #: a later task); Chinese surnames ship in locales.ZH because Han + #: segmentation is opt-in. + surnames: frozenset[str] = frozenset() #: Lowercase word -> exact-cased replacement used by capitalized() #: ("phd" -> "Ph.D."). Pair-valued: change it with #: dataclasses.replace(), not add()/remove(); read it as a mapping diff --git a/nameparser/_policy.py b/nameparser/_policy.py index fd12a98c..2ccbc2fc 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -285,6 +285,7 @@ def _validated_script_orders( ) raw = value.items() if isinstance(value, Mapping) else value try: + # a non-iterable raises TypeError, caught below entries: tuple[Any, ...] = tuple(raw) # type: ignore[arg-type] except TypeError: raise TypeError( @@ -308,6 +309,26 @@ def _validated_script_orders( return tuple(sorted(canonical.items())) +def _validated_segment_scripts(value: object) -> frozenset[Script]: + """segment_scripts' check: an iterable of Script members (or + their string values), coerced via _validated_script so the + unknown-script wording stays single-sourced.""" + _reject_str_and_mapping(value, "segment_scripts") + # Probe with iter() only (mirrors the patronymic_rules probe in + # Policy.__post_init__): a genuine non-iterable is caught here and + # gets the curated message below, while an exception raised INSIDE + # a caller's generator during consumption (the frozenset() below) + # must propagate untouched, not be rewritten as "not an iterable". + try: + script_iter = _require_iterable(value, "segment_scripts") # type: ignore[arg-type] + except TypeError: + raise TypeError( + f"segment_scripts must be an iterable of Script members, " + f"got {value!r}" + ) from None + return frozenset(_validated_script(s) for s in script_iter) + + def _canonical_script_pair(pair: Iterable[Any]) -> tuple[Any, ...]: """Hashability-only canonicalization of one PolicyPatch script_orders entry: tuple-ize the pair AND the order value inside @@ -361,6 +382,18 @@ class Policy: #: decides the family name. script_orders: tuple[tuple[Script, tuple[Role, Role, Role]], ...] = ( DEFAULT_SCRIPT_ORDERS) + #: Scripts for which the unspaced-name segmentation stage is + #: active (#271): the first token written wholly in an activated + #: script is split by longest surname match against + #: :attr:`Lexicon.surnames `. + #: Default: {Script.HANGUL} -- hangul is unambiguously Korean and + #: Korean surnames are a closed default-shipped set. Han is NOT + #: default: a zh surname list corrupts Japanese names (高橋一郎 + #: must not split as 高+橋一郎), so it's opt-in via locales.ZH. + #: Opt out with ``segment_scripts=()``; note a PolicyPatch unions + #: rather than replaces, so a pack can only add scripts, never + #: disable one. + segment_scripts: frozenset[Script] = frozenset({Script.HANGUL}) #: Opt-in detectors that reorder patronymic-shaped names #: (EAST_SLAVIC, TURKIC); usually set via a locale pack. patronymic_rules: frozenset[PatronymicRule] = frozenset() @@ -413,6 +446,9 @@ def __post_init__(self) -> None: object.__setattr__( self, "script_orders", _validated_script_orders(self.script_orders)) + object.__setattr__( + self, "segment_scripts", + _validated_segment_scripts(self.segment_scripts)) _reject_str_and_mapping(self.patronymic_rules, "patronymic_rules") # Probe with iter() rather than wrapping tuple(): non-iterables # (True especially -- v1's patronymic_name_order was a bool flag, @@ -553,6 +589,8 @@ class PolicyPatch: #: without restating the rest. script_orders: tuple[ tuple[Script, tuple[Role, Role, Role]], ...] | _Unset = UNSET + segment_scripts: frozenset[Script] | _Unset = field( + default=UNSET, metadata=_UNION) patronymic_rules: frozenset[PatronymicRule] | _Unset = field( default=UNSET, metadata=_UNION) middle_as_family: bool | _Unset = UNSET diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index 19eb60f5..89f14217 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -556,3 +556,11 @@ def test_remove_of_a_stored_dead_entry_is_silent() -> None: with warnings.catch_warnings(): warnings.simplefilter("error") dirty.remove(titles=["zqx zqy"]) + + +def test_surnames_is_a_vocab_field() -> None: + lex = Lexicon.empty().add(surnames={"毛", "欧阳"}) + assert lex.surnames == frozenset({"毛", "欧阳"}) + assert lex.remove(surnames={"毛"}).surnames == frozenset({"欧阳"}) + merged = Lexicon(surnames=frozenset({"김"})) | Lexicon(surnames=frozenset({"박"})) + assert merged.surnames == frozenset({"김", "박"}) diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index 31f2d23c..3504fcec 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -341,7 +341,7 @@ def test_all_set_valued_patch_fields_declare_union_composition() -> None: } assert union_fields == { "patronymic_rules", "nickname_delimiters", - "maiden_delimiters", "extra_suffix_delimiters", + "maiden_delimiters", "extra_suffix_delimiters", "segment_scripts", } @@ -389,7 +389,7 @@ def test_name_order_rejects_bare_string() -> None: # ships, and its frozenset() coercion destroys the evidence before # Policy could catch it at apply time. _GUARDED = ["nickname_delimiters", "maiden_delimiters", - "extra_suffix_delimiters", "patronymic_rules"] + "extra_suffix_delimiters", "patronymic_rules", "segment_scripts"] @pytest.mark.parametrize("cls", [Policy, PolicyPatch]) @@ -420,6 +420,7 @@ def test_collection_fields_reject_a_bare_string( ("maiden_delimiters", [("[", "]")], frozenset({("[", "]")})), ("extra_suffix_delimiters", ["/"], frozenset({"/"})), ("patronymic_rules", ["turkic"], frozenset({PatronymicRule.TURKIC})), + ("segment_scripts", [Script.HAN], frozenset({Script.HAN})), ]) def test_legitimate_iterables_are_accepted_and_stored_intact( cls: type, field: str, items: list, expected: frozenset) -> None: @@ -457,7 +458,8 @@ def test_a_pair_mapping_is_accepted_through_items(cls: type) -> None: @pytest.mark.parametrize("value", [b" - ", bytearray(b" - "), memoryview(b" - ")]) @pytest.mark.parametrize( "field", ["extra_suffix_delimiters", "nickname_delimiters", - "maiden_delimiters", "patronymic_rules", "name_order"]) + "maiden_delimiters", "patronymic_rules", "name_order", + "segment_scripts"]) def test_every_field_rejects_a_buffer_with_a_decode_hint( cls: type, field: str, value: object) -> None: # Same cryptic int message Lexicon had, and name_order was the one @@ -520,3 +522,40 @@ def test_script_orders_validates_keys_and_values() -> None: Role.FAMILY)}) with pytest.raises(TypeError, match="bare string"): Policy(script_orders="han") # type: ignore[arg-type] + + +def test_segment_scripts_defaults_to_hangul_and_coerces() -> None: + # hangul segmentation is default-on (amendment 2026-07-27): + # hangul is unambiguously Korean and the surname set is closed + assert Policy().segment_scripts == frozenset({Script.HANGUL}) + p = Policy(segment_scripts=["han", Script.HANGUL]) # type: ignore[arg-type] + assert p.segment_scripts == frozenset({Script.HAN, Script.HANGUL}) + assert Policy(segment_scripts=()).segment_scripts == frozenset() # type: ignore[arg-type] + + +def test_segment_scripts_rejects_unknown_script() -> None: + # bare-string/mapping rejection is covered by the _GUARDED family + # sweep (test_collection_fields_reject_a_bare_string / + # test_collection_fields_reject_a_mapping); this pins the one + # check unique to segment_scripts -- the unknown-script message. + with pytest.raises(ValueError, match="han, hangul"): + Policy(segment_scripts={"klingon"}) # type: ignore[arg-type] + + +def test_segment_scripts_rejects_non_iterable_with_curated_message() -> None: + # parallel to patronymic_rules: name the expected shape rather than + # surfacing _require_iterable's generic "must be an iterable" text + with pytest.raises(TypeError, + match="segment_scripts must be an iterable of " + "Script members, got True"): + Policy(segment_scripts=True) # type: ignore[arg-type] + + +def test_segment_scripts_patch_unions() -> None: + patched = apply_patch( + Policy(), PolicyPatch(segment_scripts={Script.HAN})) # type: ignore[arg-type] + assert patched.segment_scripts == frozenset( + {Script.HAN, Script.HANGUL}) + stringly = apply_patch( + Policy(), PolicyPatch(segment_scripts={"han"})) # type: ignore[arg-type] + assert all(isinstance(s, Script) for s in stringly.segment_scripts) From 6a8c94d1cd80ae2c34dd8fdc813e396e9913214a Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Tue, 28 Jul 2026 09:59:01 -0700 Subject: [PATCH 04/20] Propagate caller-generator errors untouched in script-field validation (#271) Co-Authored-By: Claude Fable 5 --- nameparser/_policy.py | 85 ++++++++++++++++++++++++++++++++++++++--- tests/v2/test_policy.py | 85 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 164 insertions(+), 6 deletions(-) diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 2ccbc2fc..d8eee4e1 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -285,13 +285,19 @@ def _validated_script_orders( ) raw = value.items() if isinstance(value, Mapping) else value try: - # a non-iterable raises TypeError, caught below - entries: tuple[Any, ...] = tuple(raw) # type: ignore[arg-type] + # Probe with iter() only (mirrors the patronymic_rules probe in + # Policy.__post_init__, and _validated_segment_scripts below): + # a genuine non-iterable is caught here and gets the curated + # message below, while an exception raised INSIDE a caller's + # generator during consumption (the tuple() below) must + # propagate untouched, not be rewritten as "not a mapping". + raw_iter = _require_iterable(raw, "script_orders") # type: ignore[arg-type] except TypeError: raise TypeError( f"script_orders must be a mapping of Script to order, " f"got {value!r}" ) from None + entries: tuple[Any, ...] = tuple(raw_iter) canonical: dict[Script, tuple[Role, Role, Role]] = {} for entry in entries: try: @@ -631,13 +637,80 @@ def __post_init__(self) -> None: if self.script_orders is not UNSET and not isinstance( self.script_orders, str): raw = self.script_orders + # Mapping.items() is always iterable, so it needs no probe; + # only the non-Mapping branch can fail the iter() check below. pairs = raw.items() if isinstance(raw, Mapping) else raw + # Four outcomes share this block, and only one of them + # should propagate immediately: + # 1. raw itself isn't iterable at all (script_orders=5) -- + # caught by the iter() probe. Shape problem, defers to + # apply time, where Policy's own guard raises the + # curated message ("Policy validates ... at apply"). + # 2. an already-yielded PAIR has a bad shape (a bare int + # from a shredded bytes buffer, e.g. b"han" iterating + # to 104) -- caught around _canonical_script_pair + # below. Also a shape problem: abort the whole + # conversion and leave script_orders as a RE-ITERABLE + # value (see case 4) Policy's curated message (e.g. the + # decode-first bytes hint, or the bad-pair message) can + # still quote at apply time. + # 3. the caller's OWN exception, raised by their + # generator while it is being consumed -- this is not + # a shape problem this guard exists to catch, and + # deferral is impossible anyway: by the time + # apply_patch reaches Policy.__post_init__, the same + # generator is already exhausted, so catching this + # here would lose the error for good rather than + # merely delay it. Consuming a one-shot iterator + # therefore runs UNGUARDED (case 4), so this + # propagates on its own terms. + # 4. a ONE-SHOT iterator (iter(x) is x -- generators, + # map/zip/filter objects, ...) cannot be safely + # re-iterated once consumed. Deferring case 2 by + # leaving it as originally written -- fine for + # re-iterable inputs (bytes, list, dict, tuple) -- would + # leave Policy re-validating an EXHAUSTED generator at + # apply time: iterating it again silently yields + # nothing, storing () ("opt out of per-script + # ordering") and dropping the Han/Hangul family-first + # defaults with NO error anywhere. So a one-shot input + # is eagerly materialized into a tuple first (case 3's + # unguarded consumption), and case 2's deferral stores + # THAT re-iterable tuple instead of the original, + # exhausted generator. try: - object.__setattr__( - self, "script_orders", - tuple(_canonical_script_pair(p) for p in pairs)) + pairs_iter = iter(pairs) except TypeError: - pass # Policy validates (and raises properly) at apply + pairs_iter = None # non-iterable: deferred, case 1 + if pairs_iter is not None: + # mypy sees pairs' declared type (tuple[...] | ItemsView) + # as never identical to iter(pairs)'s Iterator type, but + # at runtime pairs can be anything iterable a caller + # wrote (a generator especially) -- the whole point of + # this check. + one_shot = pairs_iter is pairs # type: ignore[comparison-overlap] + items: Iterable[Any] = tuple(pairs_iter) if one_shot else pairs + canonical = [] + deferred = False + for item in items: + try: + canonical.append(_canonical_script_pair(item)) + except TypeError: + deferred = True # bad pair shape: case 2 + break + if not deferred: + object.__setattr__( + self, "script_orders", tuple(canonical)) + elif one_shot: + # items is already the materialized (re-iterable) + # tuple built above -- store it so Policy can + # re-examine and quote it at apply time, instead of + # the original, now-exhausted generator. + object.__setattr__(self, "script_orders", items) + # else: re-iterable input (bytes/list/dict/tuple/...) is + # left exactly as originally written; Policy can freely + # re-iterate it at apply time and quote the caller's + # value there -- unchanged from the prior behavior. for f in dataclasses.fields(self): if f.metadata.get("compose") != "union": continue diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index 3504fcec..80d97da1 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -1,4 +1,5 @@ import dataclasses +from collections.abc import Iterator import pytest @@ -559,3 +560,87 @@ def test_segment_scripts_patch_unions() -> None: stringly = apply_patch( Policy(), PolicyPatch(segment_scripts={"han"})) # type: ignore[arg-type] assert all(isinstance(s, Script) for s in stringly.segment_scripts) + + +# One item per field, used to build a bad_iter() below that yields a +# legitimate first entry, then raises -- forcing consumption past the +# iter()-probe stage so a rewrite-vs-propagate bug can only be caught +# by actually running the generator. +_PROPAGATION_FIELD_VALUES = { + "segment_scripts": Script.HAN, + "patronymic_rules": PatronymicRule.TURKIC, + "script_orders": (Script.HAN, FAMILY_FIRST), + "nickname_delimiters": ("<", ">"), + "maiden_delimiters": ("[", "]"), + "extra_suffix_delimiters": "/", +} + + +@pytest.mark.parametrize("cls,field", [ + (Policy, "segment_scripts"), + (Policy, "patronymic_rules"), + (Policy, "script_orders"), + (Policy, "nickname_delimiters"), + (Policy, "maiden_delimiters"), + (Policy, "extra_suffix_delimiters"), + (PolicyPatch, "segment_scripts"), + (PolicyPatch, "patronymic_rules"), + (PolicyPatch, "script_orders"), + (PolicyPatch, "nickname_delimiters"), + (PolicyPatch, "maiden_delimiters"), + (PolicyPatch, "extra_suffix_delimiters"), +]) +def test_caller_generator_errors_propagate_untouched( + cls: type, field: str) -> None: + """A TypeError raised INSIDE a caller's generator, after it has + already yielded a legitimate value, must reach the caller with its + own message -- not be rewritten as "not an iterable" by whichever + guard happens to consume the generator to check its shape. + + Probe-only-in-try is what makes this true: iter() succeeds (so the + guard doesn't fire), and only the SEPARATE, unguarded consumption + step sees the generator's own exception. PolicyPatch.script_orders + is the scalar (non-union) field with its own canonicalization + block, distinct from the shared union loop the other fields go + through below -- it needs the identical probe/consume split, since + deferring a caller-generator error to apply time is impossible + once the generator is already exhausted (verified: the error would + otherwise be lost for good, not merely delayed).""" + def bad_iter() -> Iterator[object]: + yield _PROPAGATION_FIELD_VALUES[field] + raise TypeError("boom") + + with pytest.raises(TypeError, match="boom"): + cls(**{field: bad_iter()}) + + +def test_policy_patch_script_orders_non_iterable_still_defers_to_apply() -> None: + # The deferral contract this fix must NOT break: a genuinely + # non-iterable script_orders value is not a caller-generator error + # -- it fails the iter() probe itself, so PolicyPatch construction + # stays lazy (and hashable) and Policy's own guard raises the + # curated message once the patch is actually applied. + patch = PolicyPatch(script_orders=5) # type: ignore[arg-type] + assert isinstance(hash(patch), int) + with pytest.raises(TypeError, match="script_orders must be a mapping"): + apply_patch(Policy(), patch) + + +def test_policy_patch_one_shot_bad_tail_defers_without_silent_drop() -> None: + # The silent-drop repro: a ONE-SHOT generator whose first item is a + # good pair and whose second is a bad shape (not caller-generator + # code raising -- just a malformed entry). PolicyPatch construction + # must not raise (this is a shape problem, deferred to apply time, + # like the bytes case), but the exhausted-generator hole meant + # apply_patch used to silently store () -- "opt out of per-script + # ordering" -- with NO error anywhere, dropping the Han/Hangul + # family-first defaults. The list form of the identical input + # already raised correctly; only the one-shot form leaked. + gen = (x for x in [(Script.HAN, FAMILY_FIRST), 5]) + patch = PolicyPatch(script_orders=gen) # type: ignore[arg-type] + # Materialized into a re-iterable tuple, not left as the exhausted + # generator -- Policy can still quote the caller's bad entry (5). + assert patch.script_orders == ((Script.HAN, FAMILY_FIRST), 5) + with pytest.raises(TypeError, + match=r"script_orders entries must be .* got 5"): + apply_patch(Policy(), patch) From 548cd0f10a8cdc5b06e34300392c6219af25d159 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Tue, 28 Jul 2026 10:20:16 -0700 Subject: [PATCH 05/20] Add the script_segment stage: longest-match CJK surname splitting (#271) Unspaced CJK names give tokenize no separator to find, so the new stage inserts the missing boundary by vocabulary: the first token written wholly in an activated script is matched longest-first against Lexicon.surnames and split into two sub-slices of the original. It runs after segment, on the comma doctrine that script-conditional behavior is ignored where a comma already decides the family: FAMILY_COMMA opts out whole, and the remaining structures split within the name segment only, remapping the segment runs past the insertion. Inert by default -- the default surname vocabulary is still empty. Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/__init__.py | 7 +- nameparser/_pipeline/_script_segment.py | 133 ++++++++++++++++ nameparser/_pipeline/_state.py | 23 +-- nameparser/_types.py | 5 + tests/v2/pipeline/test_script_segment.py | 189 +++++++++++++++++++++++ tests/v2/pipeline/test_state.py | 7 + tests/v2/test_contracts.py | 9 +- tests/v2/test_layering.py | 4 +- 8 files changed, 363 insertions(+), 14 deletions(-) create mode 100644 nameparser/_pipeline/_script_segment.py create mode 100644 tests/v2/pipeline/test_script_segment.py diff --git a/nameparser/_pipeline/__init__.py b/nameparser/_pipeline/__init__.py index f69c2188..9c91ba08 100644 --- a/nameparser/_pipeline/__init__.py +++ b/nameparser/_pipeline/__init__.py @@ -16,14 +16,15 @@ from nameparser._pipeline._extract import extract_delimited from nameparser._pipeline._group import group from nameparser._pipeline._post_rules import post_rules +from nameparser._pipeline._script_segment import script_segment from nameparser._pipeline._segment import segment from nameparser._pipeline._state import ParseState from nameparser._pipeline._tokenize import tokenize -#: The full seven-stage fold. +#: The full eight-stage fold. STAGES: tuple[Callable[[ParseState], ParseState], ...] = ( - extract_delimited, tokenize, segment, classify, group, assign, - post_rules, + extract_delimited, tokenize, segment, script_segment, classify, + group, assign, post_rules, ) diff --git a/nameparser/_pipeline/_script_segment.py b/nameparser/_pipeline/_script_segment.py new file mode 100644 index 00000000..a8cd961d --- /dev/null +++ b/nameparser/_pipeline/_script_segment.py @@ -0,0 +1,133 @@ +"""Stage: script_segment (#271). + +Consumes: tokens, segments, structure. +Produces: tokens (the first activated-script token of the name +segment split in two), segments (index runs remapped past the +insertion), ambiguities (indices likewise remapped, plus a +SEGMENTATION report when more than one split was +vocabulary-supported). +Reads: Policy.segment_scripts, Lexicon.surnames. + +Unspaced CJK names give tokenize no separator to find, so this stage +inserts the missing token boundary by vocabulary: the first token +written wholly in an activated script is matched longest-first against +Lexicon.surnames, and a hit splits it in two. Compound-before-single +("欧阳明" is 欧阳 + 明, though 欧 is itself a surname) falls out of +longest-first. The split makes two sub-slices of the one token, +rewriting nothing -- spans still index the original exactly, so the +anti-#100 invariant holds by construction. + +Placed AFTER segment, on the comma doctrine that script-conditional +behavior is ignored where a comma already decides the family (the +rule script_orders follows): under FAMILY_COMMA the pre-comma text IS +the family by declaration, and splitting it would invent a boundary +the writer explicitly did not draw -- "남궁민수, 지훈" must render +family "남궁민수", not "남궁 민수". The post-comma side is given-name +text, no surname site either, so that structure opts out whole. +NO_COMMA and SUFFIX_COMMA still split, within segments[0] (the name +part) only. Running after segment costs the index remaps below; +running BEFORE it would have made the comma structure itself depend +on the split -- segment's suffix-comma rule needs more than one word +before the comma, so a pre-split "김민준, Jr." would have changed +structure on vocabulary alone. As written it stays FAMILY_COMMA, +which is why the SUFFIX_COMMA path needs a second word ("Dr 김민준, +Jr.") to be reachable at all. + +Activation is per script because the AMBIGUITY is per script +(amendment 2026-07-27): HANGUL is on by default (hangul is +unambiguously Korean, and its surname set is closed and +default-shipped), while HAN is opt-in via locales.ZH -- a Chinese +surname list corrupts Japanese names ("高橋一郎" must not split as +高 + 橋一郎), which is #272's pluggable segmenter, not this table's. +Only the FIRST activated-script token is considered, match or no +match: family-first traditions put the surname at the front of the +name, and a match deeper in the token stream would be a given name or +an ordinary word, not a surname site. +""" +from __future__ import annotations + +import dataclasses + +from nameparser._pipeline._state import ( + ParseState, PendingAmbiguity, Structure, +) +from nameparser._pipeline._vocab import single_script +from nameparser._types import AmbiguityKind, Span + + +def _remap(run: tuple[int, ...], split_at: int) -> tuple[int, ...]: + """One segment run after token `split_at` split in two: the second + half joins the run immediately after it, and every later index + shifts.""" + out: list[int] = [] + for j in run: + if j == split_at: + out.extend((split_at, split_at + 1)) + else: + out.append(j + 1 if j > split_at else j) + return tuple(out) + + +def script_segment(state: ParseState) -> ParseState: + scripts = state.policy.segment_scripts + surnames = state.lexicon.surnames + if not scripts or not surnames or not state.segments: + return state + if state.structure is Structure.FAMILY_COMMA: + return state # the comma already drew the boundary + # segments[0] is the NAME part under both remaining structures + # (everything, under NO_COMMA); later segments are suffixes. Its + # members are main-stream token indices by construction, so + # extracted nickname/maiden content is unreachable from here -- + # no input can produce it, so no test pins it. + i = next((i for i in state.segments[0] + if single_script(state.tokens[i].text) in scripts), None) + if i is None: + return state + token = state.tokens[i] + text = token.text + # A token that IS a surname never splits: a bare "남궁" must not + # become 남 + 궁 just because the single-syllable surname also + # matches -- there is nothing to split off, and a lone token's + # role is the order resolution's call. + if text in surnames: + return state + # Longest-first (compound-before-single falls out of it), capped + # so the remainder is never empty. Direct membership, no + # _normalize: the script gate admits only CJK text, which the + # storage fold stores unchanged. + cap = min(max(map(len, surnames)), len(text) - 1) + matches = [length for length in range(cap, 0, -1) + if text[:length] in surnames] + if not matches: + return state # the first activated-script token decides + take = matches[0] + cut = token.span.start + take + head = dataclasses.replace(token, text=text[:take], + span=Span(token.span.start, cut)) + tail = dataclasses.replace(token, text=text[take:], + span=Span(cut, token.span.end)) + tokens = state.tokens[:i] + (head, tail) + state.tokens[i + 1:] + # Every index the earlier stages recorded is now stale past the + # split point: the segment runs group's own iteration rests on, + # and the ambiguities from extract_delimited (resolved to indices + # by tokenize) and segment. An ambiguity ON the split token keeps + # pointing at the head. + segments = tuple(_remap(run, i) for run in state.segments) + ambiguities = tuple( + dataclasses.replace(a, indices=tuple( + j + 1 if j > i else j for j in a.indices)) + for a in state.ambiguities) + if len(matches) > 1: + # more than one vocabulary-supported split: longest-first + # DECIDED a fork, and the deciding stage records it. A + # single-match split chose nothing, so it stays silent. + alt = matches[1] + ambiguities += (PendingAmbiguity( + AmbiguityKind.SEGMENTATION, + f"{text!r} splits as {text[:take]!r} + {text[take:]!r} " + f"on the longest surname; {text[:alt]!r} + " + f"{text[alt:]!r} also reads", + (i, i + 1)),) + return dataclasses.replace(state, tokens=tokens, segments=segments, + ambiguities=ambiguities) diff --git a/nameparser/_pipeline/_state.py b/nameparser/_pipeline/_state.py index 3feeacb0..dcba3111 100644 --- a/nameparser/_pipeline/_state.py +++ b/nameparser/_pipeline/_state.py @@ -66,15 +66,20 @@ class ParseState: """Carried through the stage fold. Frozen; stages return copies via dataclasses.replace. Fields are filled progressively: extract_delimited -> extracted/masked; tokenize -> tokens (span- - sorted)/comma_offsets; segment -> segments/structure; classify -> - token tags; group -> pieces/piece_tags/dropped AND maiden token - roles; assign/post_rules -> the remaining token roles. Ambiguities - are recorded by every stage that DECIDES one -- extract (resolved to - a token index by tokenize), segment, classify, group, and assign -- - since a fork whose branches are taken in different stages needs an - emitter in each. Post-group, segments may retain indices of dropped - tokens -- assign iterates pieces, never segments. This ownership map - is pinned by tests/v2/pipeline/test_state.py.""" + sorted)/comma_offsets; segment -> segments/structure; + script_segment -> tokens and segments again (the one stage that + changes the token COUNT: an unspaced CJK token splits in two, + still as sub-slices of the original, and every later index in the + segment runs shifts); classify -> token tags; group -> + pieces/piece_tags/dropped AND maiden token roles; + assign/post_rules -> the remaining token roles. Ambiguities are + recorded by every stage that DECIDES one -- extract (resolved to a + token index by tokenize), segment, script_segment, classify, + group, and assign -- since a fork whose branches are taken in + different stages needs an emitter in each. Post-group, segments + may retain indices of dropped tokens -- assign iterates pieces, + never segments. This ownership map is pinned by + tests/v2/pipeline/test_state.py.""" original: str lexicon: Lexicon diff --git a/nameparser/_types.py b/nameparser/_types.py index ff2346a1..2354e702 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -280,6 +280,11 @@ class AmbiguityKind(StrEnum): #: More comma-separated segments than any recognized name shape; #: the parse is best-effort over the extra segments. COMMA_STRUCTURE = "comma-structure" + #: An unspaced CJK token had more than one vocabulary-supported + #: surname split ("欧阳明": 欧阳 + 明 was taken, 欧 + 阳明 also + #: matched); the longest surname won, and ``detail`` names both + #: readings. Points at the two tokens the split produced. (#271) + SEGMENTATION = "segmentation" @dataclass(frozen=True, slots=True) diff --git a/tests/v2/pipeline/test_script_segment.py b/tests/v2/pipeline/test_script_segment.py new file mode 100644 index 00000000..22ad2d6e --- /dev/null +++ b/tests/v2/pipeline/test_script_segment.py @@ -0,0 +1,189 @@ +"""Stage: script_segment (#271) -- unspaced CJK surname splitting.""" +import dataclasses + +from nameparser import Parser +from nameparser._lexicon import Lexicon +from nameparser._pipeline._script_segment import script_segment +from nameparser._pipeline._segment import segment +from nameparser._pipeline._state import ( + ParseState, PendingAmbiguity, Structure, +) +from nameparser._pipeline._tokenize import tokenize +from nameparser._policy import Policy, Script +from nameparser._types import AmbiguityKind + +_HAN = Policy(segment_scripts=frozenset({Script.HAN})) +_HANGUL = Policy() # HANGUL is the default activation +# 欧 AND 欧阳 both present on purpose: compound-before-single must +# pick 欧阳 (the issue's own acceptance example); same for 남/남궁. +# "jr" so the comma cases can reach SUFFIX_COMMA. +_LEX = Lexicon(surnames=frozenset({"毛", "欧", "欧阳", "김", "남", "남궁"}), + suffix_words=frozenset({"jr"})) + + +def _run(text: str, policy: Policy = _HAN, + lexicon: Lexicon = _LEX) -> ParseState: + state = ParseState(original=text, lexicon=lexicon, policy=policy) + return script_segment(segment(tokenize(state))) + + +def _texts(state: ParseState) -> list[str]: + return [t.text for t in state.tokens] + + +def test_splits_leading_surname_and_spans_index_the_original() -> None: + state = _run("毛泽东") + assert _texts(state) == ["毛", "泽东"] + assert [(t.span.start, t.span.end) for t in state.tokens] == [ + (0, 1), (1, 3)] + assert all(state.original[t.span.start:t.span.end] == t.text + for t in state.tokens) + + +def test_compound_before_single() -> None: + assert _texts(_run("欧阳明")) == ["欧阳", "明"] + assert _texts(_run("남궁민수", policy=_HANGUL)) == ["남궁", "민수"] + + +def test_hangul_splits_under_the_default_policy() -> None: + assert _texts(_run("김민준", policy=_HANGUL)) == ["김", "민준"] + + +def test_han_needs_the_opt_in() -> None: + # default policy activates HANGUL only: Han tokens stay whole + assert _texts(_run("毛泽东", policy=_HANGUL)) == ["毛泽东"] + + +def test_inert_without_surname_vocabulary() -> None: + assert _texts(_run("毛泽东", lexicon=Lexicon.empty())) == ["毛泽东"] + + +def test_inert_with_segmentation_disabled() -> None: + off = Policy(segment_scripts=()) # type: ignore[arg-type] + assert _texts(_run("김민준", policy=off)) == ["김민준"] + + +def test_whole_token_surname_does_not_split() -> None: + # a bare surname has nothing to split off; a lone token's role is + # the order resolution's call, not this stage's. The trap: 欧 and + # 남 are ALSO listed, so without the whole-token guard these + # would split 欧+阳 / 남+궁 by the shorter prefix + assert _texts(_run("欧阳")) == ["欧阳"] + assert _texts(_run("남궁", policy=_HANGUL)) == ["남궁"] + + +def test_no_match_leaves_the_token_alone() -> None: + assert _texts(_run("阿明")) == ["阿明"] + + +def test_only_the_first_script_token_is_considered() -> None: + assert _texts(_run("毛泽东 毛泽东")) == ["毛", "泽东", "毛泽东"] + + +def test_leading_non_script_token_is_skipped_over() -> None: + assert _texts(_run("Dr 毛泽东")) == ["Dr", "毛", "泽东"] + + +def test_mixed_script_token_is_not_split() -> None: + assert _texts(_run("毛zedong")) == ["毛zedong"] + + +def test_first_script_token_decides_even_on_no_match() -> None: + # no fallthrough: the front of the name is the only surname site + assert _texts(_run("阿明 毛泽东")) == ["阿明", "毛泽东"] + + +def test_compound_fork_emits_segmentation_ambiguity() -> None: + # both 欧阳+明 and 欧+阳明 are vocabulary-supported: the stage + # decided a fork, and the pipeline contract is that deciding + # stages record it (indices = the two result tokens) + out = _run("欧阳明") + assert [a.kind for a in out.ambiguities] == [ + AmbiguityKind.SEGMENTATION] + assert out.ambiguities[0].indices == (0, 1) + # the documented promise: detail names BOTH readings + detail = out.ambiguities[0].detail + assert "欧阳" in detail and "明" in detail + assert "阳明" in detail + ko = _run("남궁민수", policy=_HANGUL) + assert [a.kind for a in ko.ambiguities] == [ + AmbiguityKind.SEGMENTATION] + + +def test_single_possible_split_emits_no_ambiguity() -> None: + # only 毛 matches: no fork, no noise + assert _run("毛泽东").ambiguities == () + assert _run("김민준", policy=_HANGUL).ambiguities == () + + +def test_family_comma_is_inert() -> None: + # the comma declared the family: splitting it would invent a + # boundary the writer did not draw ("남궁 민수" with a space) + out = _run("남궁민수, 지훈", policy=_HANGUL) + assert out.structure is Structure.FAMILY_COMMA + assert _texts(out) == ["남궁민수", "지훈"] + assert out.ambiguities == () + + +def test_family_comma_given_side_untouched() -> None: + assert _texts(_run("박, 남궁민수", policy=_HANGUL)) == [ + "박", "남궁민수"] + + +def test_one_word_before_the_comma_is_never_suffix_comma() -> None: + # an unspaced CJK name is ONE word, and segment's suffix-comma + # rule needs >1 word before the comma -- so this reads as + # FAMILY_COMMA and the opt-out above covers it too + out = _run("김민준, Jr.", policy=_HANGUL) + assert out.structure is Structure.FAMILY_COMMA + assert _texts(out) == ["김민준", "Jr."] + + +def test_suffix_comma_name_part_still_splits() -> None: + out = _run("Dr 김민준, Jr.", policy=_HANGUL) + assert out.structure is Structure.SUFFIX_COMMA + assert _texts(out) == ["Dr", "김", "민준", "Jr."] + + +def test_segments_remap_after_the_split() -> None: + out = _run("Dr 김민준, Jr.", policy=_HANGUL) + # name segment gains the second half in place; the suffix + # segment's index follows the token it names + assert out.segments == ((0, 1, 2), (3,)) + assert [[out.tokens[i].text for i in run] for run in out.segments] \ + == [["Dr", "김", "민준"], ["Jr."]] + + +def test_ambiguity_indices_shift_past_the_split() -> None: + state = segment(tokenize(ParseState( + original="毛泽东 X", lexicon=_LEX, policy=_HAN))) + state = dataclasses.replace(state, ambiguities=( + PendingAmbiguity(AmbiguityKind.UNBALANCED_DELIMITER, + "synthetic", indices=(0,)), + PendingAmbiguity(AmbiguityKind.UNBALANCED_DELIMITER, + "synthetic", indices=(1,)), + )) + out = script_segment(state) + # index 0 (the split token itself) stays; index 1 moves right + assert out.ambiguities[0].indices == (0,) + assert out.ambiguities[1].indices == (2,) + + +def test_the_real_fold_splits_and_keeps_the_stage_after_segment() -> None: + # the stage-order contract end to end: a set-comparison ownership + # test cannot catch a reorder, but this can -- placed before + # segment, the split would feed segment two words where the writer + # wrote one and turn the family-comma parse below into a + # suffix-comma one + p = Parser(lexicon=_LEX, policy=_HANGUL) + plain = p.parse("김민준") + assert (plain.family, plain.given) == ("김", "민준") + declared = p.parse("남궁민수, 지훈") + assert (declared.family, declared.given) == ("남궁민수", "지훈") + + +def test_delimited_only_input_reaches_the_empty_segments_guard() -> None: + # extraction masks the whole string, so segment produces no runs + # at all -- the guard the stage-level helper above cannot reach + nick = Parser(lexicon=_LEX, policy=_HANGUL).parse("(민준)") + assert nick.nickname == "민준" diff --git a/tests/v2/pipeline/test_state.py b/tests/v2/pipeline/test_state.py index 3b3775de..a02908df 100644 --- a/tests/v2/pipeline/test_state.py +++ b/tests/v2/pipeline/test_state.py @@ -51,6 +51,13 @@ def test_stage_field_ownership() -> None: # a character offset that tokenize resolves to a token index "tokenize": {"tokens", "comma_offsets", "ambiguities"}, "segment": {"segments", "structure", "ambiguities"}, + # script_segment splits one unspaced CJK token in two, so it + # rewrites tokens and shifts every later index the earlier + # stages recorded -- the segment runs included. It is + # deliberately absent from token_ownership below, whose + # token-count assert is the one contract this stage is exempt + # from. structure it only READS (the FAMILY_COMMA opt-out). + "script_segment": {"tokens", "segments", "ambiguities"}, # classify also emits SUFFIX_OR_NICKNAME: the delimiter escape # that decides it lives in extract_delimited, which has no token # index to point at, so the report is raised here instead diff --git a/tests/v2/test_contracts.py b/tests/v2/test_contracts.py index 0cd24164..cf1eb8f1 100644 --- a/tests/v2/test_contracts.py +++ b/tests/v2/test_contracts.py @@ -15,12 +15,19 @@ AmbiguityKind.SUFFIX_OR_NAME: "John Smith MA", # no emitter yet -- arrives with locale-pack order detection (2.x) AmbiguityKind.ORDER: None, + # the emitter exists (script_segment), but nothing the DEFAULT + # parser sees can reach it: the default surname vocabulary is + # still empty, so no input splits at all. A trigger lands with + # the Korean census list. + AmbiguityKind.SEGMENTATION: None, } @pytest.mark.parametrize("kind", [ + # the per-entry comment above says WHY each None is None (no + # emitter yet, or an emitter no default parse can reach) pytest.param(k, marks=pytest.mark.xfail( - strict=True, reason=f"{k.value}: emitter not yet implemented")) + strict=True, reason=f"{k.value}: no trigger registered yet")) if k in _AMBIGUITY_TRIGGERS and _AMBIGUITY_TRIGGERS[k] is None else k for k in AmbiguityKind ]) diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index c740db5a..8739bf11 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -14,7 +14,8 @@ _MUST_EXIST = {"_types.py", "_lexicon.py", "_policy.py", "_locale.py", "_render.py", "_pipeline/_state.py", "_pipeline/__init__.py", "_pipeline/_extract.py", "_pipeline/_tokenize.py", - "_pipeline/_vocab.py", "_pipeline/_segment.py", + "_pipeline/_vocab.py", "_pipeline/_script_segment.py", + "_pipeline/_segment.py", "_pipeline/_classify.py", "_pipeline/_group.py", "_pipeline/_assign.py", "_pipeline/_post_rules.py", "_pipeline/_assemble.py", "_parser.py", "_facade.py", @@ -59,6 +60,7 @@ "_pipeline/_vocab.py": _PIPELINE_STAGE_ALLOWED, "_pipeline/_extract.py": _PIPELINE_STAGE_ALLOWED, "_pipeline/_tokenize.py": _PIPELINE_STAGE_ALLOWED, + "_pipeline/_script_segment.py": _PIPELINE_STAGE_ALLOWED, "_pipeline/_segment.py": _PIPELINE_STAGE_ALLOWED, "_pipeline/_classify.py": _PIPELINE_STAGE_ALLOWED, "_pipeline/_group.py": _PIPELINE_STAGE_ALLOWED, From 204d8a0328239e182984893dc63a29dd8726af40 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Tue, 28 Jul 2026 10:57:02 -0700 Subject: [PATCH 06/20] Ship the Korean census surnames as default vocabulary (#271) Co-Authored-By: Claude Fable 5 --- nameparser/_config_shim.py | 7 +++++ nameparser/_lexicon.py | 9 ++++-- nameparser/_pipeline/_vocab.py | 5 ++++ nameparser/config/surnames.py | 49 +++++++++++++++++++++++++++++++++ tests/v2/pipeline/test_vocab.py | 13 ++++++++- tests/v2/test_contracts.py | 37 +++++++++++++++++-------- tests/v2/test_facade.py | 7 +++++ tests/v2/test_lexicon.py | 10 +++++++ tests/v2/test_parser.py | 9 ++++++ 9 files changed, 130 insertions(+), 16 deletions(-) create mode 100644 nameparser/config/surnames.py diff --git a/nameparser/_config_shim.py b/nameparser/_config_shim.py index f0a2ca85..331809ca 100644 --- a/nameparser/_config_shim.py +++ b/nameparser/_config_shim.py @@ -986,6 +986,7 @@ def _build_snapshot(self) -> tuple[Lexicon, Policy, _RenderDefaults]: removal path. """ from nameparser.config.maiden_markers import MAIDEN_MARKERS + from nameparser.config.surnames import KOREAN_SURNAMES acronyms = frozenset(self.suffix_acronyms) particles = frozenset(self.prefixes) bound = frozenset(self.bound_first_names) @@ -1061,6 +1062,12 @@ def _build_snapshot(self) -> tuple[Lexicon, Policy, _RenderDefaults]: # v1 Constants has no manager for these (#274 is 2.0 # behavior); the data module is the only source maiden_markers=frozenset(MAIDEN_MARKERS), + # likewise no v1 manager: the unspaced-name segmentation + # vocabulary is 2.0 behavior (#271), so it rides in the + # snapshot only -- v1's Constants surface stays frozen. + # Unwrapped where maiden_markers above is wrapped: this + # module is born frozen (#293), so no wrap + surnames=KOREAN_SURNAMES, # TupleManager is dict[str, object] (v1 parity: values were # never statically str-typed); every real entry is a str, # same assumption _DelimiterManager's sentinel lookup makes diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 7fd00314..7190e738 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -332,9 +332,8 @@ class Lexicon: #: written wholly in a script :attr:`Policy.segment_scripts #: ` activates. The default #: carries the Korean census list - #: (:data:`~nameparser.config.surnames.KOREAN_SURNAMES`, wired in - #: a later task); Chinese surnames ship in locales.ZH because Han - #: segmentation is opt-in. + #: (:data:`~nameparser.config.surnames.KOREAN_SURNAMES`); Chinese + #: surnames ship in locales.ZH because Han segmentation is opt-in. surnames: frozenset[str] = frozenset() #: Lowercase word -> exact-cased replacement used by capitalized() #: ("phd" -> "Ph.D."). Pair-valued: change it with @@ -580,6 +579,7 @@ def _default_lexicon() -> Lexicon: from nameparser.config.suffixes import ( SUFFIX_ACRONYMS, SUFFIX_ACRONYMS_AMBIGUOUS, SUFFIX_NOT_ACRONYMS, ) + from nameparser.config.surnames import KOREAN_SURNAMES from nameparser.config.titles import FIRST_NAME_TITLES, TITLES # v1 data modules export plain `set[str]`; wrap each at this call site @@ -599,6 +599,9 @@ def _default_lexicon() -> Lexicon: conjunctions=frozenset(CONJUNCTIONS), bound_given_names=frozenset(BOUND_FIRST_NAMES), maiden_markers=frozenset(MAIDEN_MARKERS), + # surnames.py is born frozen (#293) -- no call-site wrap needed, + # unlike the v1 modules above (their wraps drop when #293 lands) + surnames=KOREAN_SURNAMES, # pass canonical pair-tuples so this strictly-typed call site never # feeds a Mapping to the tuple-annotated field; __post_init__ # still tolerates a Mapping at runtime for interactive use diff --git a/nameparser/_pipeline/_vocab.py b/nameparser/_pipeline/_vocab.py index abbd72bc..075105e1 100644 --- a/nameparser/_pipeline/_vocab.py +++ b/nameparser/_pipeline/_vocab.py @@ -154,6 +154,11 @@ def single_script(text: str) -> Script | None: the caller falls back to the positional default).""" if not text: return None # all() is vacuously true; "" belongs to no script + if text.isascii(): + # every _SCRIPT_RANGES entry is non-ASCII (lowest today is + # U+3400): skip the range sweep for the overwhelmingly common + # Latin token (the _tokenize._ignorable ASCII-floor precedent) + return None for script, ranges in _SCRIPT_RANGES.items(): if all(any(lo <= ord(c) <= hi for lo, hi in ranges) for c in text): diff --git a/nameparser/config/surnames.py b/nameparser/config/surnames.py new file mode 100644 index 00000000..b9e0130c --- /dev/null +++ b/nameparser/config/surnames.py @@ -0,0 +1,49 @@ +from nameparser.config._invariants import assert_normalized + +# Born a frozenset (#293's convention: new modules have no users to +# bridge, and a mutable module constant would silently desync the +# cached ``Lexicon.default()`` from the shim's per-construction +# copies). +# +# Single-syllable surnames in census rank order, 10 per row; the cut is +# the top ~94 -- append new entries at the tail, do not alphabetize +# (rank order is the only in-file record of where the coverage floor +# sits). +KOREAN_SURNAMES = frozenset({ + "김", "이", "박", "최", "정", "강", "조", "윤", "장", "임", + "한", "오", "서", "신", "권", "황", "안", "송", "전", "홍", + "유", "고", "문", "양", "손", "배", "백", "허", "남", "심", + "노", "하", "곽", "성", "차", "주", "우", "구", "민", "류", + "나", "진", "지", "엄", "채", "원", "천", "방", "공", "현", + "함", "변", "염", "여", "추", "도", "소", "석", "선", "설", + "마", "길", "연", "위", "표", "명", "기", "반", "왕", "금", + "옥", "육", "인", "맹", "제", "모", "탁", "국", "은", "편", + "용", "예", "경", "봉", "사", "부", "가", "복", "태", "목", + "형", "계", "피", "두", + # the two-syllable surnames in current use (census-complete); + # longest-first matching splits 남궁민수 as 남궁+민수, not 남+궁민수 + "남궁", "황보", "제갈", "사공", "선우", "서문", "독고", "동방", + "망절", +}) +""" +Korean surnames (#271), used by the 2.0 API's unspaced-name +segmentation (``Lexicon.default().surnames``): a hangul token like +"김민준" splits into surname + given name by longest match. This ships +as DEFAULT vocabulary because it is self-selecting -- a hangul entry +can only ever match hangul text -- and hangul text is unambiguously +Korean. Chinese surnames deliberately live in ``nameparser.locales.zh`` +instead (Han segmentation is opt-in; a zh list corrupts Japanese kanji +names). + +Source: the 2015 South Korean census surname tables -- the most common +single-syllable surnames (Kim/Lee/Park alone cover ~45% of the +population) plus the two-syllable surnames in current use. A coverage +floor, not the complete census roster: extend with +``Lexicon.default().add(surnames={...})``. + +Consumed by the 2.0 parser's default lexicon. The 1.x parser does not +read this module. +""" + + +assert_normalized("KOREAN_SURNAMES", KOREAN_SURNAMES) diff --git a/tests/v2/pipeline/test_vocab.py b/tests/v2/pipeline/test_vocab.py index bee3cb64..82ef6292 100644 --- a/tests/v2/pipeline/test_vocab.py +++ b/tests/v2/pipeline/test_vocab.py @@ -1,6 +1,7 @@ from nameparser._lexicon import Lexicon from nameparser._pipeline._vocab import ( - is_initial, is_suffix_lenient, is_suffix_strict, single_script, + _SCRIPT_RANGES, is_initial, is_suffix_lenient, is_suffix_strict, + single_script, ) from nameparser._policy import Script @@ -71,3 +72,13 @@ def test_single_script_range_edges() -> None: def test_single_script_empty_string_is_no_script() -> None: assert single_script("") is None + + +def test_no_script_range_reaches_ascii() -> None: + # single_script short-circuits on an ASCII token instead of + # sweeping the ranges (the hot path: every Latin name). The + # classifications above stay pinned either way; what the shortcut + # rests on is this floor, so a future script entry that dips below + # it fails here rather than silently going unclassified. + assert all(lo >= 0x80 + for ranges in _SCRIPT_RANGES.values() for lo, _ in ranges) diff --git a/tests/v2/test_contracts.py b/tests/v2/test_contracts.py index cf1eb8f1..e2ecc401 100644 --- a/tests/v2/test_contracts.py +++ b/tests/v2/test_contracts.py @@ -15,11 +15,9 @@ AmbiguityKind.SUFFIX_OR_NAME: "John Smith MA", # no emitter yet -- arrives with locale-pack order detection (2.x) AmbiguityKind.ORDER: None, - # the emitter exists (script_segment), but nothing the DEFAULT - # parser sees can reach it: the default surname vocabulary is - # still empty, so no input splits at all. A trigger lands with - # the Korean census list. - AmbiguityKind.SEGMENTATION: None, + # 남 and 남궁 are both shipped surnames, so longest-first picks + # between two vocabulary-supported splits + AmbiguityKind.SEGMENTATION: "남궁민수", } @@ -78,14 +76,29 @@ def test_every_stable_tag_has_a_trigger(tag: str) -> None: def test_every_guarded_config_module_is_imported() -> None: """Import-time asserts only run if the module is imported. - Five of the seven guarded config modules are pulled in by ``import - nameparser``, but ``maiden_markers`` is lazy (imported inside - ``_snapshot``/``_default_lexicon``), so its guard fired only - incidentally, whenever some other test happened to build a default + ``maiden_markers`` and ``surnames`` are lazy (imported inside + ``_snapshot``/``_default_lexicon``), so their guards fire only + incidentally, whenever some other test happens to build a default Lexicon. Importing them all here makes every guard unconditional. + + The roster is DERIVED from the source tree, not listed: a + hand-written tuple fails open, silently skipping the next guarded + module nobody remembers to add. """ import importlib - - for name in ("titles", "suffixes", "prefixes", "bound_first_names", - "conjunctions", "maiden_markers", "capitalization"): + import pathlib + + import nameparser.config + + config_dir = pathlib.Path(nameparser.config.__file__).parent + # private modules excluded: _invariants DEFINES the guard (the + # substring hits its own def) and holds no vocabulary of its own + guarded = sorted( + p.stem for p in config_dir.glob("*.py") + if not p.stem.startswith("_") + and "assert_normalized(" in p.read_text(encoding="utf-8")) + assert guarded, ( + f"no guarded config module found in {config_dir} -- the " + f"derivation broke, and an empty roster asserts nothing") + for name in guarded: importlib.import_module(f"nameparser.config.{name}") diff --git a/tests/v2/test_facade.py b/tests/v2/test_facade.py index 7ca9ac9b..7f50c7a1 100644 --- a/tests/v2/test_facade.py +++ b/tests/v2/test_facade.py @@ -571,3 +571,10 @@ def test_facade_reads_wholly_han_names_family_first() -> None: # wholly-Han token now takes the family role, not first assert HumanName("毛泽东").last == "毛泽东" assert HumanName("毛泽东").first == "" + + +def test_facade_parses_unspaced_korean_by_default() -> None: + # the default-behavior fix flows through the v1 facade (its + # lexicon mirrors Lexicon.default() via the shim snapshot) + n = HumanName("김민준") + assert (n.last, n.first) == ("김", "민준") diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index 89f14217..d2b13af7 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -564,3 +564,13 @@ def test_surnames_is_a_vocab_field() -> None: assert lex.remove(surnames={"毛"}).surnames == frozenset({"欧阳"}) merged = Lexicon(surnames=frozenset({"김"})) | Lexicon(surnames=frozenset({"박"})) assert merged.surnames == frozenset({"김", "박"}) + + +def test_default_lexicon_ships_korean_surnames_only() -> None: + lex = Lexicon.default() + assert "김" in lex.surnames and "남궁" in lex.surnames + # Han surnames are locales.ZH's cargo (segmentation opt-in), so + # the DEFAULT set is wholly hangul -- structural check, not + # content-pinning + assert all(all(0xAC00 <= ord(c) <= 0xD7A3 for c in s) + and 1 <= len(s) <= 2 for s in lex.surnames) diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index 2e08e2bc..4539e9bc 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -508,3 +508,12 @@ def test_script_order_beats_explicit_global_name_order() -> None: p = Parser(policy=Policy(name_order=FAMILY_FIRST_GIVEN_LAST)) n = p.parse("김 민준") assert (n.family, n.given) == ("김", "민준") + + +def test_unspaced_korean_names_parse_by_default() -> None: + # the whole point of shipping the census list as default + # vocabulary (#271): no pack, no config + n = parse("김민준") + assert (n.family, n.given) == ("김", "민준") + n = parse("남궁민수") # two-syllable surname beats single 남 + assert (n.family, n.given) == ("남궁", "민수") From 4ece41833c9f50a526748a73a435cc0faeb77b85 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Tue, 28 Jul 2026 11:19:07 -0700 Subject: [PATCH 07/20] Add the ZH locale pack: Han segmentation opt-in + surname data (#271) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Han ORDER is already default (script_orders reads a wholly-Han name family-first); Han SEGMENTATION is the one behavior that must stay opt-in, because a pure-Han string cannot say whether it is Chinese or Japanese and a zh surname list corrupts Japanese names. locales.ZH is that opt-in: PolicyPatch(segment_scripts={Script.HAN}) plus the vocabulary to segment with -- 2020 census top-100 singles, their traditional variant forms, and the Hundred Family Surnames compounds still in use. It sets no order field at all, so applying it can only ADD: Korean segmentation keeps working through a zh parser. Two entries deviate from census rank order and say so in the data comment: 阎 sits off-rank beside its reading-pair 闫, and 萧 is appended because the mainland census records that surname under the merged 肖 while both simplified spellings circulate. DEVIATES declares any name containing a Han character. That over-declares (a Han name needs an unspaced surname match to actually change), which is the gate's safe direction. Two deferred items ride along. script_segment's per-parse max(map(len, surnames)) becomes a per-vocabulary lru_cache -- the pack roughly triples the surname set for its users, and Lexicon is frozen and slotted, so it cannot carry the cache itself. And the property layer's vocabulary pool gains CJK entries, with _names_using offering each drawn surname concatenated with a given name: script_segment splits an unspaced token, so a space-joined name could never reach it (measured: the stage fired on none of the 250 config-fuzz examples). Co-Authored-By: Claude Fable 5 --- docs/locales.rst | 2 +- nameparser/_pipeline/_script_segment.py | 19 ++- nameparser/locales/__init__.py | 1 + nameparser/locales/zh.py | 110 +++++++++++++++++ tests/v2/test_locales.py | 156 +++++++++++++++++++++--- tests/v2/test_properties.py | 23 +++- 6 files changed, 294 insertions(+), 17 deletions(-) create mode 100644 nameparser/locales/zh.py diff --git a/docs/locales.rst b/docs/locales.rst index 08152c51..354121f7 100644 --- a/docs/locales.rst +++ b/docs/locales.rst @@ -91,7 +91,7 @@ takes: .. doctest:: >>> locales.available() - ('ru', 'tr_az') + ('ru', 'tr_az', 'zh') >>> locales.get("ru") is locales.RU True diff --git a/nameparser/_pipeline/_script_segment.py b/nameparser/_pipeline/_script_segment.py index a8cd961d..8c3ac502 100644 --- a/nameparser/_pipeline/_script_segment.py +++ b/nameparser/_pipeline/_script_segment.py @@ -47,6 +47,7 @@ from __future__ import annotations import dataclasses +import functools from nameparser._pipeline._state import ( ParseState, PendingAmbiguity, Structure, @@ -68,6 +69,22 @@ def _remap(run: tuple[int, ...], split_at: int) -> tuple[int, ...]: return tuple(out) +@functools.lru_cache(maxsize=8) +def _longest_entry(surnames: frozenset[str]) -> int: + """The longest surname in a vocabulary, cached per-vocabulary + rather than recomputed per parse (Lexicon is frozen and slotted, + so it cannot carry a cached_property of its own). The frozenset is + hashable and a process holds only a handful of distinct + vocabularies -- the default one, plus one per constructed pack + parser -- so maxsize=8 bounds pathological many-lexicon churn + without ever evicting in normal use. + + Callers must pass a NON-EMPTY vocabulary: max() of an empty set + raises, and the stage's own emptiness guard (`not surnames`) runs + first, so the only call site cannot reach it.""" + return max(map(len, surnames)) + + def script_segment(state: ParseState) -> ParseState: scripts = state.policy.segment_scripts surnames = state.lexicon.surnames @@ -96,7 +113,7 @@ def script_segment(state: ParseState) -> ParseState: # so the remainder is never empty. Direct membership, no # _normalize: the script gate admits only CJK text, which the # storage fold stores unchanged. - cap = min(max(map(len, surnames)), len(text) - 1) + cap = min(_longest_entry(surnames), len(text) - 1) matches = [length for length in range(cap, 0, -1) if text[:length] in surnames] if not matches: diff --git a/nameparser/locales/__init__.py b/nameparser/locales/__init__.py index 7b9a99d5..40982b12 100644 --- a/nameparser/locales/__init__.py +++ b/nameparser/locales/__init__.py @@ -20,6 +20,7 @@ _REGISTRY = { "RU": ("nameparser.locales.ru", "RU"), "TR_AZ": ("nameparser.locales.tr_az", "TR_AZ"), + "ZH": ("nameparser.locales.zh", "ZH"), } diff --git a/nameparser/locales/zh.py b/nameparser/locales/zh.py new file mode 100644 index 00000000..00e1f23e --- /dev/null +++ b/nameparser/locales/zh.py @@ -0,0 +1,110 @@ +"""The Chinese locale pack (#271): activates Han segmentation and +ships the surname vocabulary for it. Deliberately does NOT set any +order -- Policy.script_orders already reads wholly-Han names +family-first by default (amendment 2026-07-27); segmentation is the +one Han behavior that must stay opt-in, because a pure-Han string +cannot say whether it is Chinese or Japanese (林 is Lin or Hayashi) +and a zh surname list corrupts Japanese names (高橋一郎 would split +高 + 橋一郎). Applying this pack IS the "my data is Chinese" +declaration. Japanese is #272's pluggable segmenter. + +Data sources: single-character surnames are the top-100 of the 2020 +PRC census; traditional-script variant forms are included for names +written in Taiwan/Hong Kong/overseas orthography; two-character +compounds are the Hundred Family Surnames compounds still in modern +use. A coverage floor, not a census roster: extend by giving +parser_for a base whose lexicon already carries the additions -- +``parser_for(locales.ZH, base=Parser(lexicon=Lexicon.default().add( +surnames={...})))``. (Adding to the finished parser's lexicon instead +is a silent no-op: Lexicon is frozen, so ``.add`` returns a NEW +lexicon that no parser is holding.) + +Declared deviations (spec §2 authoring requirement 3): the pack adds +vocabulary and one union policy field, both self-selecting by script, +so it can only change names containing Han characters -- DEVIATES +below declares exactly that (over-declaring within the script: a Han +name needs an unspaced surname match to actually change, but +per-character scanning is the safe direction). +""" +from __future__ import annotations + +from nameparser._lexicon import Lexicon +from nameparser._locale import Locale +from nameparser._policy import PolicyPatch, Script + +# 2020 census top-100 plus the annotated additions below, simplified +# forms, census rank order, 10 per row -- append new entries at the +# tail, do not alphabetize (rank order is the only in-file record of +# where the coverage floor sits). Two entries are NOT census rows and +# are called out so the rank-order record stays honest: +# +# * 阎 sits off-rank beside 闫 rather than at the tail, because the two +# are a reading pair, not a rank neighbour: 闫 is the census row and +# 阎 is the distinct surname sharing its reading (Yan). Keeping them +# adjacent is worth the one break in rank order. +# * 萧 is at the tail: the mainland census records this surname under +# the merged 肖 (rank 33 above), but 萧 stays a distinct simplified +# spelling in current use and both circulate. The traditional form +# 蕭 ships either way, below. +_SINGLE = ( + "王", "李", "张", "刘", "陈", "杨", "黄", "赵", "吴", "周", + "徐", "孙", "马", "朱", "胡", "郭", "何", "林", "高", "罗", + "郑", "梁", "谢", "宋", "唐", "许", "韩", "邓", "冯", "曹", + "彭", "曾", "肖", "田", "董", "潘", "袁", "蔡", "蒋", "余", + "于", "杜", "叶", "程", "魏", "苏", "吕", "丁", "任", "卢", + "姚", "沈", "钟", "姜", "崔", "谭", "陆", "范", "汪", "廖", + "石", "金", "韦", "贾", "夏", "傅", "方", "邹", "熊", "白", + "孟", "秦", "邱", "侯", "江", "尹", "薛", "闫", "阎", "段", + "雷", "龙", "黎", "史", "陶", "贺", "毛", "郝", "顾", "龚", + "邵", "万", "覃", "武", "钱", "戴", "严", "莫", "孔", "向", + "常", "萧", +) +# Traditional-script forms of the above where the glyph differs. 蕭 is +# the traditional spelling of both 萧 and the merged 肖 (see the note +# above); every other row is a plain 1:1 variant. +_SINGLE_TRADITIONAL = ( + "張", "劉", "陳", "楊", "黃", "趙", "吳", "孫", "馬", "羅", + "鄭", "謝", "許", "韓", "鄧", "馮", "蕭", "葉", "蘇", "呂", + "盧", "鍾", "譚", "陸", "韋", "賈", "鄒", "閆", "閻", "龍", + "賀", "顧", "龔", "萬", "錢", "嚴", "蔣", +) +# Hundred Family Surnames compounds still in modern use, simplified. +_COMPOUND = ( + "欧阳", "司马", "诸葛", "上官", "夏侯", "皇甫", "尉迟", "公孙", + "长孙", "慕容", "司徒", "司空", "令狐", "宇文", "东方", "独孤", + "南宫", "西门", "澹台", "淳于", "单于", "申屠", "公羊", "仲孙", + "轩辕", "呼延", "端木", "百里", "东郭", "闻人", "拓跋", "万俟", + "夹谷", "太史", +) +# Traditional forms of the compounds where any glyph differs. +_COMPOUND_TRADITIONAL = ( + "歐陽", "司馬", "諸葛", "尉遲", "公孫", "長孫", "東方", "獨孤", + "南宮", "西門", "澹臺", "單于", "仲孫", "軒轅", "東郭", "聞人", + "萬俟", "夾谷", +) + +# Private: ZH.lexicon.surnames is the supported way to read this, and +# publishing a name later is compatible where un-publishing is not. +_SURNAMES = frozenset( + _SINGLE + _SINGLE_TRADITIONAL + _COMPOUND + _COMPOUND_TRADITIONAL) + +ZH = Locale( + code="zh", + lexicon=Lexicon(surnames=_SURNAMES), + policy=PolicyPatch(segment_scripts=frozenset({Script.HAN})), +) + +# Han codepoint spans, kept in sync BY HAND with +# _pipeline/_vocab.py's _SCRIPT_RANGES[Script.HAN] (layering forbids +# a pack importing the pipeline; the sync test in +# tests/v2/test_locales.py pins the equality). +_HAN_RANGES = ((0x3400, 0x4DBF), (0x4E00, 0x9FFF), (0xF900, 0xFAFF), + (0x20000, 0x323AF)) + + +def DEVIATES(name: str) -> bool: + """True when this pack may parse `name` differently from the + default parser (the declared-deviation predicate the + non-interference gate consumes).""" + return any(any(lo <= ord(c) <= hi for lo, hi in _HAN_RANGES) + for c in name) diff --git a/tests/v2/test_locales.py b/tests/v2/test_locales.py index 9fdb05e1..a26f3425 100644 --- a/tests/v2/test_locales.py +++ b/tests/v2/test_locales.py @@ -1,5 +1,5 @@ -"""The locale pack layer (locales spec §2-3): lazy access, the two -2.0.0 packs, composition, and the non-interference gate.""" +"""The locale pack layer (locales spec §2-3): lazy access, the shipped +packs, composition, and the non-interference gate.""" import functools import importlib import re @@ -10,9 +10,12 @@ from nameparser import Locale, Parser, locales, parse, parser_for from nameparser._lexicon import _VOCAB_FIELDS, Lexicon -from nameparser._policy import PatronymicRule, Policy +from nameparser._pipeline._vocab import _SCRIPT_RANGES +from nameparser._policy import UNSET, PatronymicRule, Policy, Script +from nameparser._types import AmbiguityKind from nameparser.locales import ru as _ru from nameparser.locales import tr_az as _tr_az +from nameparser.locales import zh as _zh from .conftest import differential_corpus @@ -55,8 +58,8 @@ def test_locales_module_attribute_access() -> None: def test_locales_get_and_available() -> None: assert locales.get("ru") is locales.RU - assert set(locales.available()) == {"ru", "tr_az"} - with pytest.raises(KeyError, match="ru, tr_az"): + assert set(locales.available()) == {"ru", "tr_az", "zh"} + with pytest.raises(KeyError, match="ru, tr_az, zh"): locales.get("xx") @@ -67,6 +70,89 @@ def test_tr_az_pack_contents() -> None: assert locales.TR_AZ.lexicon == Lexicon.empty() +def test_zh_pack_contents() -> None: + assert locales.ZH.code == "zh" + # the pack activates Han segmentation and ships vocabulary -- + # and does NOT touch order: script_orders already reads Han + # family-first by default (amendment 2026-07-27) + assert locales.ZH.policy.segment_scripts == frozenset({Script.HAN}) + assert locales.ZH.policy.name_order is UNSET + assert locales.ZH.policy.script_orders is UNSET + assert "王" in locales.ZH.lexicon.surnames + assert "欧阳" in locales.ZH.lexicon.surnames # compound + assert "歐陽" in locales.ZH.lexicon.surnames # traditional + + +def test_zh_surname_entries_are_wellformed() -> None: + # structural integrity, not content-pinning: 1-2 chars each, + # every char in the stage's own Han ranges (catches a stray + # Latin char, a pasted full name, or a hangul entry) + ranges = _SCRIPT_RANGES[Script.HAN] + for entry in locales.ZH.lexicon.surnames: + assert 1 <= len(entry) <= 2, entry + assert all(any(lo <= ord(c) <= hi for lo, hi in ranges) + for c in entry), entry + + +def test_zh_parses_unspaced_names() -> None: + p = _PACKED["zh"] + n = p.parse("毛泽东") + assert (n.family, n.given) == ("毛", "泽东") + n = p.parse("欧阳修") # compound + 1-char given + assert (n.family, n.given) == ("欧阳", "修") + n = p.parse("司马相如") # compound + 2-char given + assert (n.family, n.given) == ("司马", "相如") + # the 肖/萧 census merger ships both simplified spellings + n = p.parse("萧红") + assert (n.family, n.given) == ("萧", "红") + + +def test_zh_longest_match_prefers_compound_over_single_surname() -> None: + # The longest-first tie-break, exercised by the PACK's own data + # rather than by the synthetic lexicon the stage tests use. Most + # compounds here begin with a character that is not itself a listed + # surname (欧, 司, 诸 are all outside the top-100), so they would + # split the same way under shortest-first and prove nothing. 夏侯 is + # one of only three compounds that would not -- 夏 is rank 65 -- so + # this is the pair that shows the shipped data actually reaches the + # tie-break. + p = _PACKED["zh"] + n = p.parse("夏侯惇") + assert (n.family, n.given) == ("夏侯", "惇") + n = p.parse("夏雨") # the single-surname foil + assert (n.family, n.given) == ("夏", "雨") + # ...and the ambiguity contract on that same data: a decided fork + # is REPORTED (first Han coverage of the SEGMENTATION emitter -- + # the stage tests reach it through a synthetic lexicon, and the + # default lexicon can only reach it through hangul), while the + # foil, where only one split was ever possible, stays silent. + assert [a.kind for a in p.parse("夏侯惇").ambiguities] == [ + AmbiguityKind.SEGMENTATION] + assert p.parse("夏雨").ambiguities == () + + +# 毛泽东/欧阳修/夏侯惇/萧红 appear in _ROTATORS["zh"] below as well. +# Not deduplicated on purpose: the rotator tests assert only THAT the +# packed parse differs from the default and that DEVIATES declares it +# -- they never look at a field value, so the readings above are +# pinned nowhere else. + + +def test_zh_composes_with_korean_defaults() -> None: + # the pack only ADDS (vocabulary + one union field): Korean + # segmentation keeps working through a zh parser + n = _PACKED["zh"].parse("김민준") + assert (n.family, n.given) == ("김", "민준") + + +def test_zh_han_ranges_stay_in_sync_with_vocab() -> None: + # zh.py hand-copies the Han spans from _pipeline/_vocab.py for its + # DEVIATES predicate (layering forbids a pack importing the + # pipeline -- see the module docstring); pin the equality here, the + # codepoint-range twin of the marker-regex sync test above. + assert _zh._HAN_RANGES == _SCRIPT_RANGES[Script.HAN] + + def test_pack_vocabulary_entries_are_single_words() -> None: # The shipped-vocabulary guard (test_lexicon.py's # test_default_lexicon_builds_warning_free), extended to the OTHER @@ -79,10 +165,10 @@ def test_pack_vocabulary_entries_are_single_words() -> None: # this file may already have forced the import, which would make a # warning-capture version of this test pass even on a regression. # Iterates every registered pack, not just ru/tr_az by name, so a - # future pack is covered automatically. The field loop is dormant - # until a pack actually ships vocabulary -- both current packs build - # on Lexicon.empty() -- so the registry-non-empty assert below is - # what keeps the test from passing vacuously today. + # future pack is covered automatically. zh is the pack that gives + # the field loop something to chew on (ru/tr_az build on + # Lexicon.empty()); the registry-non-empty assert below keeps the + # test from passing vacuously if that ever stops being true. assert locales.available() for code in locales.available(): lexicon = locales.get(code).lexicon @@ -290,6 +376,34 @@ def _default_corpus() -> list[str]: "Токаев Касым Кемел ұлы", # ұлы "Жээнбеков Сооронбай Шарип уулу", # уулу ] +# zh has no marker regexes to rotate through: its rotators cover the +# SEGMENTATION shapes instead (single/compound surname, simplified/ +# traditional, 1- and 2-character given names). No spaced entry -- +# spaced Han already parses family-first by default, so a spaced name +# would not deviate from the default parser at all. +_ROTATORS["zh"] = [ + "毛泽东", # 1-char surname, unspaced + "张伟", # 2-char name: 1-char surname + 1-char given + "欧阳修", # compound surname (欧 is NOT itself listed) + "夏侯惇", # compound beats single: 夏 IS listed too + "司马相如", # compound + 2-char given + "王力宏", # 1-char surname + 2-char given + "諸葛亮", # traditional-script compound + "萧红", # the 肖/萧 merger's second simplified spelling +] + + +def _marker_regexes(module: ModuleType) -> list[re.Pattern]: + return [v for v in vars(module).values() if isinstance(v, re.Pattern)] + + +# Packs whose DEVIATES surface is defined by marker REGEXES -- derived, +# never listed, same rule as the registry itself: a pack qualifies by +# having any module-level re.Pattern, so zh (which declares by Han +# codepoint range, pinned by its own sync test) drops out on its own +# and a future marker-regex pack cannot silently miss branch coverage. +_MARKER_REGEX_PACKS = tuple(code for code in sorted(_PACKS) + if _marker_regexes(_PACKS[code])) def test_registry_is_the_pack_contract() -> None: @@ -302,8 +416,23 @@ def test_registry_is_the_pack_contract() -> None: for code, module in _PACKS.items(): assert callable(getattr(module, "DEVIATES", None)), ( f"pack {code!r} does not declare DEVIATES") + # _MARKER_REGEX_PACKS selects on module-level patterns, so a + # pack that declares by regex but keeps its patterns inside a + # function would drop out of the branch-coverage sweep in + # silence. Importing `re` is the tell. The tradeoff is + # deliberate: a range-declaring pack that imports re for some + # unrelated reason trips this loudly, which beats a silent + # coverage gap -- and the fix is to expose the pattern. + if "re" in vars(module): + assert _marker_regexes(module), ( + f"pack {code!r} imports re but exposes no module-level " + f"pattern") assert set(_ROTATORS) == set(_PACKS), ( "every pack needs a rotator list (and only registered packs)") + # the derivation is code, not a list, so an empty result would make + # the branch-coverage test vanish into zero parametrizations rather + # than fail. Assert it found something. + assert _MARKER_REGEX_PACKS, "no pack declares marker regexes" def _alternation_branches(pattern: str) -> list[str]: @@ -317,7 +446,7 @@ def _alternation_branches(pattern: str) -> list[str]: return inner[1:-1].split("|") -@pytest.mark.parametrize("code", sorted(_PACKS)) +@pytest.mark.parametrize("code", _MARKER_REGEX_PACKS) def test_rotators_cover_every_marker_branch(code: str) -> None: # The rotator lists' per-row branch comments are a human convention; # this enforces them mechanically: every alternation branch of every @@ -328,10 +457,9 @@ def test_rotators_cover_every_marker_branch(code: str) -> None: # pattern's own shape: '^(...)$' = whole-token marker, '(...)$' = # token ending. tokens = [tok for name in _ROTATORS[code] for tok in name.split()] - patterns = [v for v in vars(_PACKS[code]).values() - if isinstance(v, re.Pattern)] - assert patterns, f"pack {code!r} defines no marker regexes" - for regex in patterns: + # non-empty by construction: having marker regexes is what put this + # pack in _MARKER_REGEX_PACKS in the first place + for regex in _marker_regexes(_PACKS[code]): for branch in _alternation_branches(regex.pattern): anchored = regex.pattern.startswith("^") branch_re = re.compile( diff --git a/tests/v2/test_properties.py b/tests/v2/test_properties.py index 54440f26..d208ad1f 100644 --- a/tests/v2/test_properties.py +++ b/tests/v2/test_properties.py @@ -187,10 +187,17 @@ def test_particle_fork_is_never_double_reported(text: str) -> None: # multi-word phrase: every field below except given_name_titles is # matched one word at a time, so a multi-word draw in a per-word field # would be a dead entry that trips Lexicon's multi-word warning. +# The CJK tail (#271) is what lets a drawn `surnames` set activate +# script_segment at all: hangul is the script segmented by default, so +# "김"/"남궁" are what make the stage fire (see _names_using, which +# supplies the unspaced token to fire it ON), and the Han rows ride +# along for script_orders -- Han segmentation is opt-in via +# locales.ZH, which these policies do not draw. _VOCAB = st.sampled_from([ "van", "de", "la", "bin", "abdul", "abu", "dr", "sir", "prof", "md", "jr", "iii", "esq", "ma", "do", "and", "y", "née", "geb", "a", "b", "ph.d", "عبد", "фон", "μεγα", + "김", "남궁", "毛", "欧阳", ]) # given_name_titles is the one field matched as a space-joined run, so @@ -271,10 +278,24 @@ def _names_using(draw: st.DrawFn, lexicon: Lexicon) -> str: """ vocab = sorted({w for name in _SET_FIELDS for w in getattr(lexicon, name)}) + # script_segment (#271) is the one stage a space-joined name can + # never reach: it splits an UNSPACED token whose PREFIX is a drawn + # surname, so every drawn surname is also offered concatenated with + # a given name. Waiting instead for a drawn surname and a matching + # literal to coincide left the stage unexercised on all 250 + # examples (measured); deriving the token from the draw itself + # reaches it on a handful. A Latin surname makes a mixed-script + # token the stage correctly declines -- useful input in its own + # right. + # sorted for the same reason `vocab` above is: frozenset iteration + # order is not stable across runs, and an unsorted pool shifts + # every index sampled_from draws -- which would defeat + # derandomize=True on the whole strategy, not just this slice. + unspaced = sorted(w + "민준" for w in lexicon.surnames) # plain names and structure characters are always available, so the # pool is never empty even for an empty lexicon pieces = st.sampled_from( - vocab + ["John", "Smith", "Q.", ",", "(", "'"]) + vocab + unspaced + ["John", "Smith", "Q.", ",", "(", "'"]) return " ".join(draw(st.lists(pieces, min_size=1, max_size=8))) From bce274905ca10d602620e6c0360c6296461f60e1 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Tue, 28 Jul 2026 11:45:18 -0700 Subject: [PATCH 08/20] Pin the #271 default CJK behavior and zh segmentation in the case table Co-Authored-By: Claude Fable 5 --- tests/v2/cases.py | 93 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 8756dead..fb6e50ba 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -469,4 +469,97 @@ def __post_init__(self) -> None: {"title": "Dr.", "family": "Smith"}, classification="fix(comma-family)", notes="pre-comma is definitionally family; v1 put it in first"), + + # -- #271: script-scoped order + segmentation (amendment 2026-07-27) + Case("ko_unspaced_default", "김민준", + {"family": "김", "given": "민준"}, + classification="fix(#271)", + notes="hangul is unambiguously Korean: census surnames ship " + "as default vocabulary and HANGUL segmentation is " + "default-on"), + Case("ko_two_syllable_surname_default", "남궁민수", + {"family": "남궁", "given": "민수"}, + classification="fix(#271)", + ambiguities=("segmentation",), + notes="남 is itself a shipped surname; longest-first takes " + "남궁 and records the decided fork"), + Case("ko_bare_two_syllable_surname", "남궁", + {"family": "남궁"}, + classification="fix(#271)", + notes="a token that IS a surname never splits by its " + "shorter prefix (남+궁); whole token takes the script " + "order's first role; nothing was split, so no fork is " + "recorded"), + Case("ko_family_comma_stays_whole", "남궁민수, 지훈", + {"family": "남궁민수", "given": "지훈"}, + classification="fix(#271)", + notes="the comma already decided the family: segmentation " + "is inert under FAMILY_COMMA (comma doctrine -- see " + "the script_segment stage docstring, which uses this " + "exact example)"), + Case("ko_suffix_comma_name_part_splits", "Dr 김민준, Jr.", + {"title": "Dr", "family": "김", "given": "민준", "suffix": "Jr."}, + classification="fix(#271)", + notes="the one comma structure where segmentation still " + "fires: a second word before the comma makes it " + "SUFFIX_COMMA, and the name part is a full positional " + "name"), + Case("ko_spaced_family_first_default", "김 민준", + {"family": "김", "given": "민준"}, + classification="fix(#271)", + notes="script_orders, no segmentation involved"), + Case("han_spaced_family_first_default", "毛 泽东", + {"family": "毛", "given": "泽东"}, + classification="fix(#271)", + notes="Han ORDER is default-safe without knowing zh from ja " + "(both write family-first natively); only " + "SEGMENTATION needs the zh opt-in"), + Case("han_unspaced_unsegmented_default", "毛泽东", + {"family": "毛泽东"}, + classification="fix(#271)", + notes="no default Han segmentation: one token, and a lone " + "wholly-Han token takes the script order's first " + "role = family"), + Case("mixed_script_untouched_by_script_orders", "John 王", + {"given": "John", "family": "王"}, + notes="single_script is None for a mixed name: script_orders " + "declines and the positional default governs"), + Case("two_han_scripts_untouched_by_script_orders", "毛 김", + {"given": "毛", "family": "김"}, + notes="two scripts also decline -- ONE script for the whole " + "name"), + + Case("zh_unspaced", "毛泽东", + {"family": "毛", "given": "泽东"}, + locale="zh", classification="fix(#271)"), + Case("zh_unspaced_two_char", "张伟", + {"family": "张", "given": "伟"}, + locale="zh", classification="fix(#271)"), + Case("zh_compound_tie_break", "夏侯惇", + {"family": "夏侯", "given": "惇"}, + locale="zh", classification="fix(#271)", + ambiguities=("segmentation",), + notes="夏 (rank 65) is itself listed, so longest-first " + "decides a real fork here and records it; most " + "compounds' first chars are NOT listed"), + Case("zh_compound_two_char_given", "司马相如", + {"family": "司马", "given": "相如"}, + locale="zh", classification="fix(#271)"), + Case("zh_traditional_compound", "諸葛亮", + {"family": "諸葛", "given": "亮"}, + locale="zh", classification="fix(#271)"), + Case("zh_no_surname_match", "阿明", + {"family": "阿明"}, + locale="zh", classification="fix(#271)", + notes="no surname prefix in the vocabulary: the token stays " + "whole and takes the script order's first role"), + Case("zh_japanese_kanji_tradeoff", "高橋一郎", + {"family": "高", "given": "橋一郎"}, + locale="zh", classification="fix(#271)", + notes="the RECORDED tradeoff, not a bug: applying the zh " + "pack declares the data Chinese, so Japanese kanji " + "names mis-split (高橋 is the real surname). This is " + "why Han segmentation is opt-in and why the gate " + "cannot guard it (DEVIATES declares all Han); " + "Japanese is #272's pluggable segmenter"), ) From 7da35477d105a65ba3dcd329fe2f917e9546765b Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Tue, 28 Jul 2026 12:00:28 -0700 Subject: [PATCH 09/20] Extend the property/sync guard rails and AGENTS.md to the #271 fields The _policies strategy hand-lists Policy's fields, so segment_scripts and script_orders -- 2.1's whole new configuration surface -- were the only ones nothing fuzzed. Draw both: script_orders from four sampled tables (the values are as restricted as name_order's), segment_scripts over the full Script enum, which is what lets a drawn hangul or Han surname actually reach the script_segment stage. AGENTS.md gains the scoped exception to the language-agnostic rule the amendment asks the implementing commit to record -- script-conditional behavior exactly where the script itself, not statistics about it, determines the convention -- plus surnames.py and maiden_markers.py in the config-module list, noted as the two data modules Constants has no attribute for. The cross-cutting test-module list was stale by six files. Co-Authored-By: Claude Fable 5 --- AGENTS.md | 6 ++++-- tests/v2/test_properties.py | 27 +++++++++++++++++++++++++-- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 331c82fe..7c3d6ddc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -72,7 +72,7 @@ logging.getLogger('HumanName').setLevel(logging.DEBUG) The library has two layers: `nameparser/config/` (data) and `nameparser/parser.py` (logic). -**Design philosophy — positional and language-agnostic.** The parser assigns parts by *position* plus small sets of words that join to neighbors; it never detects language. A name's language can't be reliably inferred from Latin-script transliteration ("Ali" is Arabic or Italian; "Van"/"Della"/"Bin" are first names in some cultures, particles in others), so language-specific rules belong in opt-in `Constants` config, never global defaults. Many "wrong for language X" reports (#133, #150, #130, #85, #103, #146, #83) are irreducible ambiguities — e.g. `de Mesnil` (want last name) vs `Van Johnson` (want first name) are the same `[prefix][word]` shape. Before adding a rule, confirm it doesn't break the opposite case (run the full suite — Portuguese and "Van Johnson" tests are the usual canaries). +**Design philosophy — positional and language-agnostic.** The parser assigns parts by *position* plus small sets of words that join to neighbors; it never detects language. A name's language can't be reliably inferred from Latin-script transliteration ("Ali" is Arabic or Italian; "Van"/"Della"/"Bin" are first names in some cultures, particles in others), so language-specific rules belong in opt-in `Constants` config, never global defaults. Many "wrong for language X" reports (#133, #150, #130, #85, #103, #146, #83) are irreducible ambiguities — e.g. `de Mesnil` (want last name) vs `Van Johnson` (want first name) are the same `[prefix][word]` shape. Before adding a rule, confirm it doesn't break the opposite case (run the full suite — Portuguese and "Van Johnson" tests are the usual canaries). **The one scoped exception (2.1, #271): script-conditional behavior is permitted exactly where the SCRIPT ITSELF — not statistics about it — determines the convention.** The never-detect-language rule above is about Latin *transliteration*, where the signal genuinely is destroyed; native script is a different question, and it is answered per behavior rather than per script. Three defaults fall out of it. Wholly-Han and wholly-Hangul names read family-first (`Policy.script_orders`) — no language detection needed, because zh and ja both write family-first in native script, so order cannot be misread even though the language is unknowable. Unspaced hangul splits into surname + given name (`config/surnames.py` ships the Korean census list as DEFAULT vocabulary) — nothing but Korean is written in hangul and the surnames are a closed census set, and the vocabulary is self-selecting besides: a hangul entry can only ever match hangul text. Han segmentation stays OPT-IN (`locales.ZH`) — a zh surname list corrupts Japanese kanji names, since 高 is a common Chinese surname and 高橋一郎 would split 高+橋一郎 where the correct reading is 高橋+一郎 (Japanese needs #272's pluggable segmenter). Latin-script input is never touched by any of this: "Kim Min-jun" is genuinely order-ambiguous and stays governed by `name_order` and opt-in packs. Before adding a script-conditional rule, work out which of the three it is — certain, certain for this one behavior only, or a statistical guess wearing a script's clothes. ### Configuration layer (`nameparser/config/`) @@ -83,6 +83,8 @@ Most modules define a plain Python set of known name pieces; `capitalization.py` - `prefixes.py` — `PREFIXES` (lastname particles, e.g. "de", "van") - `bound_first_names.py` — `BOUND_FIRST_NAMES` (bound given-name prefixes, e.g. "abdul", "abu"); `_join_bound_first_name` joins the first non-title piece to its following piece before the main assignment loop - `conjunctions.py` — `CONJUNCTIONS` (e.g. "and", "of") used to chain multi-word titles +- `maiden_markers.py` — `MAIDEN_MARKERS` (e.g. "née", "geb.") routing the following name to `maiden` +- `surnames.py` — `KOREAN_SURNAMES`, the census list the 2.0 API splits unspaced hangul on (#271). With `maiden_markers.py` it is one of the two data modules `Constants` has **no** attribute for: both reach the parse only through `Constants._snapshot()` → `Lexicon`, so the v1 surface stays frozen and there is no v1 knob to turn either off (the opt-out is the 2.0 `Policy`) - `capitalization.py` — `CAPITALIZATION_EXCEPTIONS` mapping (e.g. `{'phd': 'Ph.D.'}`) - `regexes.py` — dict of compiled regular expressions (wrapped in `RegexTupleManager` by `Constants`) @@ -131,7 +133,7 @@ The 2.0 rewrite lands as underscore-private modules alongside the v1 code. These - **Typing/docs**: `from __future__ import annotations`; `frozen=True, slots=True` on every public dataclass; strict-profile mypy flags via per-module overrides in pyproject (`strict = true` itself is not valid per-module). Docstrings state contracts in prose with **no doctest blocks** — `--doctest-modules` makes every example a test; behavior examples go to unit tests per the lean-docs rule. **Document the positive direction of a partial property**: "a non-empty `ambiguities` is a signal to act on" is checkable, while "an empty one means no fork occurred" is a universal negative needing exhaustive verification -- that claim was written twice and falsified twice, at sites the author had not audited. - **Pickling**: v2 types must round-trip (`Parser` will be picklable by construction, and it holds a `Lexicon`). Every frozen type assigns `_guarded_getstate`/`_guarded_setstate` (`_types.py`) in its class body (`@dataclass(slots=True)` would override inherited pickle methods) — unpickling fails at the LOAD site on field-layout skew, and values are deliberately NOT re-validated (pickle is not a security boundary; canonical state comes from a validated instance). `Lexicon` keeps its own copy of the guard (layering) plus the `mappingproxy` slot rebuild; a new unpicklable slot type needs the same treatment plus a round-trip test. - **One sanctioned global**: the (future) cached default `Parser`. Lazily cached FROZEN singletons (`Lexicon.default()`'s `functools.cache`, the future default parser) are constants, not state; any second piece of module-level MUTABLE state requires amending the conventions doc, on purpose, in review. Sanctioned exceptions, facade layer only: `_config_shim.CONSTANTS` (the v1 shared singleton, mutable by design) and `_facade._WARNED_SUBCLASSES` (the once-per-subclass hook-warning dedup set) — both deleted with the layer in 3.0. -- **Tests**: all v2 tests live in the `tests/v2/` package (its `conftest.py` overrides the v1 dual-run fixture — v2 code never reads shared `CONSTANTS`), one test module per source module plus cross-cutting `test_reprs.py` and `test_layering.py`, names stating behavior. Never assert `Lexicon.default()` contents; the narrow sourcing spot-checks in `test_default_sources_v1_vocabulary` that pin the v1→v2 migration contract (e.g. the flipped `particles_ambiguous` model) are the sanctioned exception. +- **Tests**: all v2 tests live in the `tests/v2/` package (its `conftest.py` overrides the v1 dual-run fixture — v2 code never reads shared `CONSTANTS`), one test module per source module plus the cross-cutting ones (`test_reprs.py`, `test_layering.py`, `test_contracts.py`, `test_properties.py`, `test_benchmark.py`, the `cases.py`/`test_cases.py` table, and `test_regex_sync.py`, which pins every hand-copied pattern or codepoint table against its source wherever the copy lives — including copies outside the package), names stating behavior. Never assert `Lexicon.default()` contents; the narrow sourcing spot-checks in `test_default_sources_v1_vocabulary` that pin the v1→v2 migration contract (e.g. the flipped `particles_ambiguous` model) are the sanctioned exception. ## Extension Patterns diff --git a/tests/v2/test_properties.py b/tests/v2/test_properties.py index d208ad1f..c17c6942 100644 --- a/tests/v2/test_properties.py +++ b/tests/v2/test_properties.py @@ -13,8 +13,8 @@ from hypothesis import strategies as st from nameparser import ( - FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST, GIVEN_FIRST, Lexicon, Parser, - PatronymicRule, Policy, parse, + DEFAULT_SCRIPT_ORDERS, FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST, + GIVEN_FIRST, Lexicon, Parser, PatronymicRule, Policy, Script, parse, ) from nameparser._lexicon import _VOCAB_FIELDS from nameparser._pipeline import run @@ -243,6 +243,22 @@ def _lexicons(draw: st.DrawFn) -> Lexicon: **_fix_invariants(**fields)) +# script_orders' legal values are as restricted as name_order's (only +# the three exported orders, keyed by Script), so they are sampled +# rather than generated. The four cover the axes that matter: the +# shipped default, the full opt-out, one script alone, and an order +# that disagrees with the default -- FAMILY_FIRST_GIVEN_LAST on hangul +# reads "김민준 수" differently from every other entry here, which is +# what makes a script table that is merely PRESENT distinguishable +# from one that is actually consulted. +_SCRIPT_ORDER_TABLES = [ + DEFAULT_SCRIPT_ORDERS, + (), + ((Script.HAN, FAMILY_FIRST),), + ((Script.HANGUL, FAMILY_FIRST_GIVEN_LAST),), +] + + @st.composite def _policies(draw: st.DrawFn) -> Policy: pairs = st.sampled_from([("(", ")"), ('"', '"'), ("'", "'"), @@ -254,6 +270,13 @@ def _policies(draw: st.DrawFn) -> Policy: return Policy( name_order=draw(st.sampled_from( [GIVEN_FIRST, FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST])), + script_orders=draw(st.sampled_from(_SCRIPT_ORDER_TABLES)), + # no max_size: Script has two members, so the unbounded draw + # already reaches every subset -- including the empty one, + # which is the documented segmentation opt-out, and the full + # one, which turns on the Han segmentation that only + # locales.ZH turns on in shipped configuration + segment_scripts=draw(st.frozensets(st.sampled_from(list(Script)))), patronymic_rules=draw(st.frozensets( st.sampled_from(list(PatronymicRule)), max_size=2)), middle_as_family=draw(st.booleans()), From d62c745cc76c96d60408109387c6d6918c87ab76 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Tue, 28 Jul 2026 12:08:25 -0700 Subject: [PATCH 10/20] Document script-scoped CJK defaults and the ZH pack (#271) Adds the East Asian sections the 2.1 behavior needs (usage.rst for what it does, customize.rst for turning each half off, locales.rst for why Korean needs no pack and Chinese does), and corrects the claims #271 made false: locales.rst said both shipped packs were policy-only and told contributors to keep them that way, concepts.rst said name_order and a comma were the only things deciding first-vs-last, and the RFC listed the Chinese/Korean work as still staged. The opt-out story is stated carefully because the obvious reading of it is wrong: script_orders={} restores the positional read but leaves the token split, so on a Korean name it moves the surname into "given" rather than undoing anything. Clearing both fields is what restores 2.0. Also fixes AGENTS.md's doctest note, which claimed Sphinx isolates local variables between blocks. It does not -- an example added here bound "name" and broke four passing examples further down usage.rst. Co-Authored-By: Claude Fable 5 --- AGENTS.md | 2 +- docs/concepts.rst | 6 ++- docs/customize.rst | 49 ++++++++++++++++++++ docs/design/nameparser-2.0-rfc.md | 21 ++++++--- docs/locales.rst | 74 ++++++++++++++++++++++++------- docs/migrate.rst | 21 +++++++++ docs/modules.rst | 18 ++++++++ docs/release_log.rst | 14 ++++++ docs/usage.rst | 62 +++++++++++++++++++++++++- 9 files changed, 239 insertions(+), 28 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 7c3d6ddc..3b1f88e2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -198,7 +198,7 @@ Add a dedicated `copy.deepcopy()` round-trip test for it too (see `test_regexes_ **Doctests** — docstring examples in `nameparser/*.py` run under `uv run pytest` (`--doctest-modules`; `testpaths` is `tests` + `nameparser` only). The `.rst` doctests in `docs/` (`usage.rst`, `customize.rst`) and `README.rst` are also run in CI: the `python-package.yml` workflow runs `sphinx-build -b doctest docs dist/doctest` (covers every `.rst` under `docs/`) and `python -m doctest README.rst` as separate steps after the existing `-b html` build. Both should pass with zero failures — expect a clean run, not baked-in noise. -Examples that mutate the shared `CONSTANTS` singleton (e.g. `capitalize_name`, `titles.clear()`) must reset it at the end of the same doctest block, since Sphinx doctest groups only isolate local variables, not global state — an unreset mutation leaks into every later example in the same build, in document order (this caused several real failures before it was fixed). If an example isn't actually demonstrating shared-singleton behavior, prefer a local `Constants()` instance instead of `CONSTANTS` — no reset needed, and no ordering dependency on other examples. `SetManager.__repr__` sorts its elements for this reason too: plain `set()` iteration order depends on per-process string hash randomization, so an unsorted repr would make ellipsis-style doc examples (`SetManager({'10th', ..., 'zoologist'})`) unreproducible across runs. +Every `.. doctest::` block in one `.rst` file shares a single namespace — the per-document default group — so **both** local variables and global state carry forward, in document order. Two consequences, and the first is the one that bites. A variable bound in one block clobbers that name in every later block, so inserting a section that binds something common breaks examples further down the file that were passing and that you never touched: writing the #271 usage.rst section with `name = parse("김민준")` in it made `name.initials()` six blocks later return `'민. 김.'` instead of `'J. Q. X. V.'`. Pick a distinctive variable name (`minjun`), or reuse whatever the surrounding blocks already bind. Second, examples that mutate the shared `CONSTANTS` singleton (e.g. `capitalize_name`, `titles.clear()`) must reset it at the end of the same block, because nothing else will — an unreset mutation leaks into every later example in the same build (this caused several real failures before it was fixed). If an example isn't actually demonstrating shared-singleton behavior, prefer a local `Constants()` instance instead of `CONSTANTS` — no reset needed, and no ordering dependency on other examples. `SetManager.__repr__` sorts its elements for this reason too: plain `set()` iteration order depends on per-process string hash randomization, so an unsorted repr would make ellipsis-style doc examples (`SetManager({'10th', ..., 'zoologist'})`) unreproducible across runs. Don't use the bare `python3 -m doctest .rst` CLI (no `optionflags`) to check files under `docs/` — it ignores each block's `:options:` directive (e.g. `+NORMALIZE_WHITESPACE`, `+ELLIPSIS`) and reports false-positive whitespace failures that look like real regressions. `uv run sphinx-build -b doctest docs ` is the faithful check for those. `README.rst` has no `:options:` directives (plain `::` blocks, not `.. doctest::`), so the bare `python -m doctest README.rst` CLI is fine there — that's what CI actually runs. diff --git a/docs/concepts.rst b/docs/concepts.rst index fd575468..dda5d825 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -56,8 +56,10 @@ title; particles join forward, so ``de la`` attaches to ``Vega``. Whatever the vocabulary layer has not claimed is left to a positional layer, which assigns purely by where a word sits: the first unclaimed word is the given name, the last is the family name, and anything -between them is the middle name. ``name_order`` and an explicit comma -change what "first" and "last" mean here; nothing else does. +between them is the middle name. ``name_order``, an explicit comma, +and — for a name written wholly in one East Asian script — +``script_orders`` change what "first" and "last" mean here; nothing +else does. This is the whole parser in two sentences, and it explains its character. A word nameparser has never seen still gets a sensible role, diff --git a/docs/customize.rst b/docs/customize.rst index 940acb7a..fa60c142 100644 --- a/docs/customize.rst +++ b/docs/customize.rst @@ -303,6 +303,55 @@ family-then-given regardless of the configured order: >>> family_first.parse("Thomas, John").family 'Thomas' +East Asian defaults, and turning them off +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Two defaults key on the *script* a name is written in rather than on +anything you set: a name written wholly in Han or Hangul reads +family-first (``script_orders``), and an unspaced hangul name is split +into surname and given name against the shipped Korean census list +(``segment_scripts``). :ref:`east-asian-names` shows what they do — +this is how to switch them off, which you can do separately: + +.. doctest:: + + >>> parse("김민준").family # both defaults on + '김' + >>> positional = Parser(policy=Policy(script_orders={})) + >>> positional.parse("김민준").family # still split + '민준' + >>> unsplit = Parser(policy=Policy(segment_scripts=())) + >>> unsplit.parse("김민준").family # one token, not split + '김민준' + +The middle line is the one to read twice: emptying ``script_orders`` +restores the purely positional reading but leaves the token split, so +the surname lands in ``given`` instead. Clear both fields to get +nameparser 2.0's behavior back exactly. + +To teach the splitter a surname it doesn't ship with, add it to the +``surnames`` vocabulary like any other word: + +.. doctest:: + + >>> lex = Lexicon.default().add(surnames={"김민"}) + >>> Parser(lexicon=lex).parse("김민준").family + '김민' + +Chinese surnames are deliberately absent from that default set, +because splitting Han text requires knowing Chinese from Japanese; +:doc:`locales` covers the opt-in ``zh`` pack that supplies them. + +.. note:: + + Both fields are annotated with their canonical *storage* type + rather than with everything the constructor accepts — the same as + ``capitalization_exceptions``. Under mypy the readable spellings + above (``script_orders={...}``, ``segment_scripts=(...)``) need a + ``# type: ignore[arg-type]``; ``script_orders=()`` and + ``segment_scripts=frozenset(...)`` check clean and mean the same + thing. + Nicknames, maiden names, and brackets ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/design/nameparser-2.0-rfc.md b/docs/design/nameparser-2.0-rfc.md index 81b4149c..74e51819 100644 --- a/docs/design/nameparser-2.0-rfc.md +++ b/docs/design/nameparser-2.0-rfc.md @@ -201,12 +201,21 @@ parser_for(locales.RU).parse("Ivanova Anna Sergeevna").given # "Anna" A locale pack is pure data: a vocabulary fragment plus a partial policy patch, folded in at parser construction. 2.0.0 ships `RU` and `TR_AZ` -(formalizing the patronymic support added in 1.3.0); Chinese/Korean -(bundled surname lists + segmentation of unspaced input), Vietnamese, -and Japanese (via a `nameparser[ja]` extra with a pluggable segmenter) -are designed and staged for 2.x minors. Vocabulary that cannot misfire -on other languages' names (e.g. Cyrillic honorifics) goes in the -default lexicon instead of packs. +(formalizing the patronymic support added in 1.3.0); Vietnamese and +Japanese (via a `nameparser[ja]` extra with a pluggable segmenter) are +designed and staged for later 2.x minors. Vocabulary that cannot +misfire on other languages' names (e.g. Cyrillic honorifics) goes in +the default lexicon instead of packs. + +*Amended for 2.1 (#271):* the Chinese/Korean work landed, and split +along that last line rather than shipping as one pack. Korean surnames +and family-first order for native-script CJK cannot misfire on other +languages' names, so both became defaults; only the Chinese surname +list needed a pack (`ZH`), because it would corrupt the Japanese names +written in the same characters. The never-auto-detect rule holds as +written for a name's *language*; its **script** is a different signal, +and where the script alone settles a convention, behavior may key on +it. ## Migration: what actually happens to existing code diff --git a/docs/locales.rst b/docs/locales.rst index 354121f7..5e1b08d5 100644 --- a/docs/locales.rst +++ b/docs/locales.rst @@ -55,6 +55,18 @@ precedes the *given* name — as Arabic ones do — so the word after it isn't read as a family name. Listing it in ``given_name_titles`` alone raises ``ValueError`` rather than quietly doing nothing. +Two East Asian behaviors are on by default for the same reason, except +that what selects them is the *script* rather than the word: a name +written wholly in Han or Hangul reads family-first, and an unspaced +Korean name is split into surname and given name. Chinese and Japanese +both write family-first natively, so reading the order takes no guess +about which language it is; hangul is written by nothing but Korean, +whose surnames are a closed census set, so splitting is safe too. See +:ref:`east-asian-names` in :doc:`usage` for what that looks like and +how to turn either half off. Splitting an unspaced *Han* name is the +one piece that does need to know the language — a Chinese surname list +mangles Japanese kanji names — so that half waits for the ``zh`` pack. + A pack is for something different: a *structural* rule, like reordering a patronymic, that vocabulary alone can't express. @@ -114,10 +126,19 @@ parsing, equivalent to ``parser_for(locales.get("ru"))``. (``oglu``, ``qizi``, ``uulu``, and their Latin- and Cyrillic-script variants) and reads the name around it as given/middle/family. - -Both shipped packs are policy-only — they carry no vocabulary of their -own; see :doc:`concepts` for why that split (language vocabulary vs. -behavior) is drawn where it is. + * - ``zh`` + - Chinese surname segmentation — splits an unspaced Han name into + surname and given name (``毛泽东`` → family ``毛``, given + ``泽东``) against the surname list the pack ships. It sets no + name order: native-script Han already reads family-first + without a pack. + +``ru`` and ``tr_az`` are policy-only — they carry no vocabulary of +their own. ``zh`` is both halves at once: a surname list, plus the one +policy field that turns segmentation on for the script it covers. See +:doc:`concepts` for how that split (language vocabulary vs. behavior) +is drawn, and `Contributing a pack to nameparser`_ for which half a +new naming rule belongs in. .. warning:: @@ -158,7 +179,9 @@ right and is validated on its own, before it is unioned onto the base — so a fragment that marks a word must also carry the word it marks. To make an existing base title precede the given name, restate the title in the fragment rather than listing it in ``given_name_titles`` -alone. Both shipped packs are policy-only, so neither hits this. +alone. ``zh`` is the shipped worked example: its +``Lexicon(surnames=...)`` has to satisfy every ``Lexicon`` rule +standing alone, before anything unions it onto the base. .. doctest:: @@ -202,24 +225,41 @@ by ``tests/v2/test_locales.py``: string, return whether *this pack alone* might parse it differently from the default parser. Over-declaring is safe; under-declaring is not — when in doubt, ``DEVIATES`` should say yes. -#. Add a rotator list to ``tests/v2/test_locales.py`` with at least one - name exercising every alternation branch of every marker regex the - pack defines — ``test_rotators_cover_every_marker_branch`` fails - until each branch is hit. +#. Add a rotator list to ``tests/v2/test_locales.py``. Every pack needs + one, but what it has to contain follows from how the pack declares + its scope. A pack declaring by *marker regex* (``ru``, ``tr_az``) + needs at least one name exercising every alternation branch of every + regex it defines — ``test_rotators_cover_every_marker_branch`` fails + until each branch is hit. A pack declaring by *codepoint range* + (``zh``) has no branches to sweep and drops out of that test, so its + rotators have to carry the same weight by hand: the unspaced names + the pack must split, one per shape of the vocabulary it ships — + single surname, compound surname, and any spelling variant it means + to cover. #. Keep the non-interference gate green over the shared corpus plus your rotators: every name the packed parser parses differently from the default must be one your ``DEVIATES`` predicate flags — no silent, undeclared side effects on names outside the pack's stated scope. -#. Keep the pack policy-only in 2.0 — ``ru`` and ``tr_az`` both ship - an empty :class:`~nameparser.Lexicon`; a pack that wants to carry - its own vocabulary is a later conversation. +#. Sort the vocabulary before shipping it, if the pack carries any. + Vocabulary that is *self-selecting* — able to match only text of + the tradition it came from, the way a hangul surname can only ever + match hangul — is default-safe, and belongs in the default lexicon + (``nameparser/config/``) rather than in a pack: a pack nobody knows + to ask for is vocabulary nobody gets. Vocabulary that *declares a + language its script does not* — a Chinese surname list, which + silently mangles the Japanese names written in the same characters + — belongs in the pack, where asking for it is the declaration. + ``ru`` and ``tr_az`` need no vocabulary at all and ship an empty + :class:`~nameparser.Lexicon`; ``nameparser/locales/zh.py`` is the + template for one that does. #. Curate vocabulary conservatively, the same rule as :doc:`customize`: when you're unsure whether a word or a marker belongs, leave it out. -``nameparser/locales/ru.py`` is the reference implementation to copy -from. Staged packs in progress are tracked in issues `#271 -`_, `#272 -`_, and `#146 -`_. +``nameparser/locales/ru.py`` is the reference implementation for a +policy-only pack, ``nameparser/locales/zh.py`` for one that carries +vocabulary. Packs still in progress are tracked in issues `#272 +`_ (Japanese) +and `#146 `_ +(Vietnamese). diff --git a/docs/migrate.rst b/docs/migrate.rst index cf1566c2..b3a76ac7 100644 --- a/docs/migrate.rst +++ b/docs/migrate.rst @@ -339,3 +339,24 @@ into ``middle``/``last`` (``"Jane Smith née Jones"``), and with a custom suffix delimiter configured, a no-space delimiter group renders whole (``"RN/CRNA"``) where 1.x split it (``"RN, CRNA"``) — the role assignment is identical, only the rendered string differs. + +2.1 adds two more, and unlike most of the 2.0 API these do reach +``HumanName``: a name written wholly in Han or Hangul is read +family-first, and an unspaced Korean name is split into surname and +given name. + +.. doctest:: + + >>> HumanName("毛 泽东").last + '毛' + >>> HumanName("김민준").last, HumanName("김민준").first + ('김', '민준') + +1.4 read the first as ``first="毛"``/``last="泽东"`` and left the +second whole in ``last``. ``Constants`` has no switch for either — the +v1 configuration surface is frozen for 2.x — so the way out is the 2.0 +API: ``Parser(policy=Policy(script_orders={}, segment_scripts=()))`` +restores the old reading of both. Unspaced *Chinese* is unaffected +either way, because splitting Han text is opt-in through a locale pack +and ``HumanName`` cannot apply one: ``HumanName("毛泽东")`` still +returns the whole string as ``last``. diff --git a/docs/modules.rst b/docs/modules.rst index 025acadc..19f3b07a 100644 --- a/docs/modules.rst +++ b/docs/modules.rst @@ -73,6 +73,9 @@ Configuration .. autoclass:: nameparser.PatronymicRule :members: +.. autoclass:: nameparser.Script + :members: + .. _name-order-constants: Name-order constants @@ -102,6 +105,19 @@ because only these three orders have defined assignment semantics. Family name first, given name *last*, the words between middle — e.g. Vietnamese full-name order. +.. py:data:: nameparser.DEFAULT_SCRIPT_ORDERS + :value: ((Script.HAN, FAMILY_FIRST), (Script.HANGUL, FAMILY_FIRST)) + + The default :attr:`~nameparser.Policy.script_orders` table: a name + written wholly in Han or Hangul reads family-first, whatever + ``name_order`` says. The values are drawn from the same three + constants above, and the same restriction applies. Build on it for + additive customization — + ``script_orders=dict(DEFAULT_SCRIPT_ORDERS) | {Script.HAN: + GIVEN_FIRST}`` — and pass ``script_orders={}`` to opt out entirely + and get the purely positional read back. Latin-script and + mixed-script names are never affected either way. + Delimiter defaults ^^^^^^^^^^^^^^^^^^ @@ -171,6 +187,8 @@ HumanName.config Defaults :members: .. automodule:: nameparser.config.maiden_markers :members: +.. automodule:: nameparser.config.surnames + :members: .. automodule:: nameparser.config.capitalization :members: .. automodule:: nameparser.config.regexes diff --git a/docs/release_log.rst b/docs/release_log.rst index e3392043..7bce4e99 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -1,5 +1,19 @@ Release Log =========== +* 2.1.0 - Unreleased + + **East Asian names** + + - Fix names written wholly in Han or Hangul parsing given-first: native-script CJK now reads family-first by default, through the new ``Policy.script_orders`` table, so ``"毛 泽东"`` gives family ``毛`` where 2.0 gave family ``泽东``. No language detection is involved — Chinese and Japanese both write the family name first in native script, so the script settles the order without anyone having to know which language it is. Latin-script and mixed-script names are never affected, and an explicit comma still wins. **Default-on: changes parse output for wholly-CJK names**, through ``HumanName`` as well as the 2.0 API (closes #271) + - Fix unspaced Korean names not splitting: the census surname list now ships as default vocabulary (``Lexicon.surnames``) with hangul segmentation on by default (``Policy.segment_scripts``), so ``"김민준"`` parses family ``김``, given ``민준`` instead of landing whole in the family name. Nothing but Korean is written in hangul and its surnames are a closed census set, which is what makes the split safe as a default rather than a pack. **Default-on**, and it reaches ``HumanName`` too. ``Policy(segment_scripts=())`` turns the split off; ``Policy(script_orders={})`` separately restores the positional reading; clearing both restores 2.0 behavior exactly (#271) + - Add the Chinese locale pack ``locales.ZH`` — opt-in Han segmentation for unspaced names like ``毛泽东``, with the surname vocabulary it needs. A pack rather than a default because a Chinese surname list corrupts the Japanese names written in the same characters (``高橋一郎`` would split ``高`` + ``橋一郎``); Japanese needs the pluggable segmenter staged in #272. It sets no name order, since native-script Han already reads family-first without it. ``locales.available()`` is now ``('ru', 'tr_az', 'zh')`` (#271) + - Add ``Lexicon.surnames``, ``Policy.script_orders``, ``Policy.segment_scripts``, the ``Script`` enum and the ``DEFAULT_SCRIPT_ORDERS`` constant to the public API. This is the first behavior nameparser keys on the script a name is written in; it is allowed only where the script itself settles a convention, never as a proxy for guessing the language (#271) + - Add ``AmbiguityKind.SEGMENTATION``, reported when a surname split had a vocabulary-supported alternative: ``"남궁민수"`` is 남궁 + 민수 by the compound surname but 남 + 궁민수 by the single-syllable one, and longest-match had to pick. A name with only one possible split decided nothing and reports nothing (#271) + + **Breaking Changes** + + - Change the pickle compatibility of the 2.0 API's ``Policy`` and ``Lexicon``: the new fields change the field layout their guarded ``__setstate__`` checks, so a pickle written by 2.0.0 raises ``ValueError`` naming the missing fields instead of loading. Re-pickle after upgrading. ``HumanName`` pickles are unaffected — the facade pickles v1-shaped component state, not these objects + * 2.0.0 - July 27, 2026 Two release candidates preceded this release (rc1 on 2026-07-23, diff --git a/docs/usage.rst b/docs/usage.rst index ef06085a..a188735a 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -52,7 +52,9 @@ given-then-family: 'Doe' For family-first input *without* a comma — common outside Europe — set -``name_order``; see :doc:`customize`. +``name_order``; see :doc:`customize`. Names written in Han or Hangul +are the exception that needs no setting at all: see `East Asian +names`_ below. Words that attach to their neighbors -------------------------------------- @@ -128,7 +130,63 @@ one that can double as a given name: 'de Mesnil' :doc:`customize` covers how to change which words are in each of these -sets, including which particles may double as given names. +sets, including which particles may double as given names. One shipped +vocabulary works the other way round and so is not in the table above: +:mod:`surnames ` *splits* a word instead of +merging two, and is covered next. + +.. _east-asian-names: + +East Asian names +----------------- + +A name written wholly in Han (Chinese and Japanese characters) or +Hangul needs no configuration to come out right: the script settles the +convention by itself, so those names read family-first by default. +Chinese and Japanese differ in most things but not this one — both put +the family name first in native script — so nothing has to guess which +language it is looking at. + +.. doctest:: + + >>> parse("毛 泽东").family + '毛' + >>> parse("山田 太郎").family + '山田' + +Korean names go a step further, because they are usually written with +no space in them at all. The census surname list ships as default +vocabulary, so an unspaced hangul name is split into its two parts: + +.. doctest:: + + >>> minjun = parse("김민준") + >>> minjun.family, minjun.given + ('김', '민준') + +That works out of the box because nothing but Korean is written in +hangul. The same trick on Han characters would have to know Chinese +from Japanese first — a Chinese surname list splits ``高橋一郎`` after +``高``, wrecking an ordinary Japanese name — so unspaced *Chinese* is +opt-in through the ``zh`` locale pack, and Japanese waits on a +segmenter of its own (`#272 +`_): + +.. doctest:: + + >>> from nameparser import locales, parser_for + >>> parser_for(locales.ZH).parse("毛泽东").family + '毛' + +A comma switches both behaviors off, on the reasoning ``name_order`` +already follows: someone who wrote one has said where the family name +ends. When the vocabulary supported more than one split — ``남궁민수`` +is 남궁 + 민수 by the compound surname but 남 + 궁민수 by the +single-syllable one — the longest match wins and the parse reports the +choice as an ``AmbiguityKind.SEGMENTATION``, described under `When the +parser had to guess`_; a name with only one possible split decided +nothing and reports nothing. :doc:`customize` covers turning either +behavior off. Aggregate views ---------------- From f5859cae9bb9af6d5ceb103f564a14f754499a50 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Tue, 28 Jul 2026 12:13:11 -0700 Subject: [PATCH 11/20] Classify the #271 CJK fixes in the differential harness Adds the expected_changes.toml rule for the two default changes, scoped to a CJK character class so it can only claim names the behavior can actually reach, and regenerates corpus_issues.jsonl (198 -> 200 names; the generator only ever adds). The rule goes FIRST, which turned out to be load-bearing. classify() takes the first matching rule and fix(suffix-routing) declares no name_regex at all, so with the new rule appended at the end -- where it started -- every CJK diff was claimed by suffix-routing under the wrong label and the new rule was dead on arrival. Caught by running the harness against a synthetic CJK corpus, since neither checked-in corpus contains a single CJK character: build_issues_corpus.py requires an internal space and unspaced names are the shape this classifies. The ordering rule is now written down in both the file header and the README. The character class is a hand copy of _vocab._SCRIPT_RANGES that no import can keep honest, so test_regex_sync.py pins it -- the module's first pin on a file outside the package, and its quietest copy, since the harness runs by hand rather than in CI. Han's astral block is excluded on both sides, deliberately: no corpus name reaches it. Co-Authored-By: Claude Fable 5 --- AGENTS.md | 2 +- tests/v2/test_regex_sync.py | 47 ++++++++++++++++++++++++ tools/differential/README.md | 9 +++++ tools/differential/corpus_issues.jsonl | 2 + tools/differential/expected_changes.toml | 34 +++++++++++++++++ 5 files changed, 93 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 3b1f88e2..6cbb4b9a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -117,7 +117,7 @@ Each named attribute (`title`, `first`, etc.) is a `@property` that joins its co The 2.0 rewrite lands as underscore-private modules alongside the v1 code. These conventions apply to all new-API code and are stricter than the v1 sections above. The full design record (rationale, settled-decision logs, dated amendments) lives in untracked `docs/superpowers/specs/`; this section is the enforceable subset. **A commit that establishes or amends one of these conventions must update this section in the same commit** — grep-driven staleness sweeps miss paraphrased prose (see Workflow above), so write-time maintenance is the mechanism, audits are the backstop. - **Module layout**: every new module is underscore-private (`_types.py`, `_lexicon.py`, `_policy.py`, `_locale.py`, `_render.py`, `_pipeline/`, `_parser.py`, plus the facade layer: `_facade.py`, `_config_shim.py`). The public import surface is exactly `nameparser` and `nameparser.locales`; `nameparser/__init__.py` holds re-exports and `__all__` only — no logic. Since the M11 swap, the old paths are import-path-preserving re-exports: `nameparser.parser` re-exports the `_facade` `HumanName`, `nameparser.config` re-exports the `_config_shim` names (`Constants`, `CONSTANTS`, `SetManager`, `TupleManager`, `RegexTupleManager`); the `config/` DATA modules stay the vocabulary source through 2.x. The whole facade layer is deleted in 3.0. -- **Facade layer** (`_facade.py`, `_config_shim.py`): the v1-compat `HumanName`/`Constants` over the core. Key mechanisms: `Constants._generation` dirty-tracking (every mutation bumps; facades resolve their `Parser` lazily via `_cached_parser(lexicon, policy)`); `Constants._snapshot()` mirrors `_lexicon._default_lexicon()` (equality-pinned); the facade pickles v1-SHAPED state (component lists, one `__setstate__` path for 1.4 and 2.x blobs; components rebuild via `replace()`, never a re-parse); `_V1_HOOKS` overrides warn once per subclass (#280). The compat contract is the migration spec's promise: warning-free 1.4 code behaves identically except release-log-classified fixes — `tools/differential/` (dev-only, not shipped) verifies this against 1.4-on-PyPI over a 652-name corpus (two files: `corpus.jsonl` from the v1 test banks at a pinned ref, `corpus_issues.jsonl` harvested from the issue tracker; `compare.py` globs `corpus*.jsonl` and fails loudly if none match). `parser.py:NNNN` citations throughout the 2.0 code refer to the PRE-swap v1 file, deleted at the M11 swap; resolve them with `git show 2d5d8c2:nameparser/parser.py`. +- **Facade layer** (`_facade.py`, `_config_shim.py`): the v1-compat `HumanName`/`Constants` over the core. Key mechanisms: `Constants._generation` dirty-tracking (every mutation bumps; facades resolve their `Parser` lazily via `_cached_parser(lexicon, policy)`); `Constants._snapshot()` mirrors `_lexicon._default_lexicon()` (equality-pinned); the facade pickles v1-SHAPED state (component lists, one `__setstate__` path for 1.4 and 2.x blobs; components rebuild via `replace()`, never a re-parse); `_V1_HOOKS` overrides warn once per subclass (#280). The compat contract is the migration spec's promise: warning-free 1.4 code behaves identically except release-log-classified fixes — `tools/differential/` (dev-only, not shipped) verifies this against 1.4-on-PyPI over a checked-in corpus of ~650 names (no exact count here: `corpus_issues.jsonl` grows whenever it is regenerated, and the run prints its own per-file totals) (two files: `corpus.jsonl` from the v1 test banks at a pinned ref, `corpus_issues.jsonl` harvested from the issue tracker; `compare.py` globs `corpus*.jsonl` and fails loudly if none match). `parser.py:NNNN` citations throughout the 2.0 code refer to the PRE-swap v1 file, deleted at the M11 swap; resolve them with `git show 2d5d8c2:nameparser/parser.py`. - **Layering is enforced by `tests/v2/test_layering.py`** (exact-module matching; `if TYPE_CHECKING:` imports don't count): `_types` imports nothing internal at module level — its rendering delegates import `_render` at call time; `_lexicon` and `_policy` sit above `_types` independently (`_lexicon` may import `nameparser.config.*` DATA modules only — vocabulary is single-sourced from the v1 data modules through 2.x, e.g. `config/maiden_markers.py`); `_locale` sits on `_lexicon`+`_policy` (plus `_types` for the shared pickle mixin); `_render` imports `_types` and `_lexicon` (for `Lexicon.default()` and `_normalize`); `_pipeline/*` imports `_types`+`_lexicon`+`_policy` plus in-package `_pipeline` helpers; `_parser` sits on everything except `_render`; the facade layer (`_facade`, `_config_shim`, `parser`, `config/__init__`, `__main__`) may import anything public plus `_render`; locale pack modules (`locales/*.py`) import `_locale`/`_lexicon`/`_policy` only, and the `locales/__init__` additionally lazy-imports its packs (PEP 562). Extend the test's `ALLOWED` table when adding a module. - **Canonical field order** — the seven roles in `Role` enum declaration order, defined once and derived everywhere (properties, `as_dict`, reprs, `comparison_key`). Never restate the order literally. `Role` is a `StrEnum`: members compare as their field-name strings, and `tokens_for()` coerces strings via `_coerce_enum`. - **Method organization**, fixed section order in every class: fields + `__post_init__` validation → alternative constructors → dunders (construction/equality → protocol → operators) → properties → public methods by concern (access → editing → comparison → rendering delegates) → private helpers last, except a helper serving exactly one section may sit at that section's head. Sanctioned deviation, facade layer only: `HumanName` and the shim `Constants` organize by v1 concern groups (`# -- render defaults --`, `# -- config / parsing --`, `# -- fields --`, ..., dunders and pickle last) — the classes mirror v1's own surface and die in 3.0; the canonical order still binds every core type. diff --git a/tests/v2/test_regex_sync.py b/tests/v2/test_regex_sync.py index 55e20927..68b7f6f6 100644 --- a/tests/v2/test_regex_sync.py +++ b/tests/v2/test_regex_sync.py @@ -8,8 +8,16 @@ config/regexes.py changed, the copies would silently diverge with no CI signal. Tests may legally import both sides (test_layering.py's own convention), so this module is where the promise gets checked. + +Layering is the usual reason for a copy but not the only one, so this +module's scope is the PROMISE rather than that one pair of packages: +the comma-set pin below reads _pipeline._state instead of config, and +the last test reaches outside the package altogether, to a TOML file +that could not import a Python constant if it wanted to. """ import re +import tomllib +from pathlib import Path import pytest @@ -149,3 +157,42 @@ def test_comma_char_matches_the_pipeline_comma_set() -> None: from nameparser._pipeline._state import COMMA_CHARS assert set(_render._COMMA_CHAR.pattern.strip("[]")) == set(COMMA_CHARS) + + +def test_differential_cjk_rule_matches_the_script_ranges() -> None: + """The #271 rule in tools/differential/expected_changes.toml hand- + copies the CJK spans from _vocab._SCRIPT_RANGES into a character + class. A TOML file cannot import the constant, so this is the one + copy with no possible alternative -- and the one whose divergence + is quietest, because the harness is run by hand rather than in CI. + + Both failure directions matter, which is why this compares sets + rather than checking coverage. A span MISSING from the class turns + an intended #271 change into an UNEXPLAINED diff (a release + blocker for the wrong reason); a span that should not be there + silently classifies a real regression as intended, which is the + failure the whole harness exists to prevent. + + Han's astral block is out of scope on both sides. The rule omits + it deliberately -- no corpus name reaches it, see the comment + there -- so the comparison runs over the BMP spans only, and a new + BMP script added to _SCRIPT_RANGES still fails here until the rule + covers it. + """ + toml_path = (Path(__file__).parents[2] / "tools" / "differential" + / "expected_changes.toml") + rules = tomllib.loads(toml_path.read_text())["change"] + matched = [r for r in rules if "#271" in r["issue"]] + assert len(matched) == 1, ( + f"expected exactly one #271 rule in {toml_path.name}, " + f"found {len(matched)}") + declared = { + (int(lo, 16), int(hi, 16)) + for lo, hi in re.findall(r"\\u([0-9A-Fa-f]{4})-\\u([0-9A-Fa-f]{4})", + matched[0]["name_regex"])} + expected = {span + for spans in _vocab._SCRIPT_RANGES.values() + for span in spans if span[1] <= 0xFFFF} + assert declared == expected, ( + f"{toml_path.name}'s #271 name_regex declares {sorted(declared)}; " + f"_SCRIPT_RANGES' BMP spans are {sorted(expected)}") diff --git a/tools/differential/README.md b/tools/differential/README.md index 8f4e0e1d..b9b2cd6b 100644 --- a/tools/differential/README.md +++ b/tools/differential/README.md @@ -116,6 +116,15 @@ matches only if the observed diff fields are a subset of this list). Keep both as tight as the actual diff allows -- a loose rule can mask a real regression. +Rules are tried in file order and the first match wins, so a rule +narrowed by `name_regex` has to sit **above** any rule narrowed only +by `fields`. The `fields`-only rules are broad by construction: a +`fields = ["first", "last", "suffix"]` rule claims every diff confined +to those three columns, whatever the name looked like. Adding the +`fix(#271)` CJK rule at the end of the file was enough to make it dead +on arrival -- `fix(suffix-routing)` claimed all of its diffs first, and +labelled them as something else. + Some entries in the seed list are for behavior families that this particular corpus (pre-M12 v1 test strings) happens not to contain any example of (e.g. custom suffix-delimiter rendering, which only fires diff --git a/tools/differential/corpus_issues.jsonl b/tools/differential/corpus_issues.jsonl index 65884271..1f40c2f4 100644 --- a/tools/differential/corpus_issues.jsonl +++ b/tools/differential/corpus_issues.jsonl @@ -102,6 +102,7 @@ "John Smith MA" "John Smith Ma" "John Smith, Dr." +"John Smith, LEED AP" "John Smith, Ph. D." "John Smith, Ph.D." "John V" @@ -166,6 +167,7 @@ "Smith, John" "Smith, John ," "Smith, John E, III, Jr" +"Smith, LEED AP" "St. ___" "Steven Hardman" "Steven Hardman, MD - DO - DDS" diff --git a/tools/differential/expected_changes.toml b/tools/differential/expected_changes.toml index e6168acb..ebdbbc83 100644 --- a/tools/differential/expected_changes.toml +++ b/tools/differential/expected_changes.toml @@ -3,6 +3,40 @@ # S5). Rules are seeded from docs/superpowers/plans/notes-m12-diffs.md # and the `classification="fix(...)"` rows in tests/v2/cases.py; keep # each entry's `name_regex`/`fields` as tight as the diff allows. +# ORDER MATTERS: classify() takes the FIRST matching rule, so a rule +# narrowed by `name_regex` must sit ahead of any rule narrowed only by +# `fields`, or the broad one silently claims its diffs under the wrong +# label. See the #271 entry immediately below for a case that did. + +[[change]] +issue = "fix(#271) native-script CJK: family-first order + hangul segmentation" +# '毛 泽东', '김민준': script_orders flips first/last for a name written +# wholly in Han or Hangul, and the Korean surnames that now ship as +# default vocabulary additionally split an unspaced hangul token into +# last + first. Seeded from the fix(#271) rows in tests/v2/cases.py. +# +# Scoped by script, which is exactly the scope of the behavior -- +# neither change can touch a name with no CJK character in it. Han's +# astral block (U+20000-U+323AF) is deliberately left out of the class: +# no name in either corpus reaches it, and a rule should be no wider +# than the diffs it has to explain. Extend it -- and the sync pin in +# tests/v2/test_regex_sync.py, which fails if the two disagree -- if +# one ever does. +# +# FIRST on purpose, and this position is load-bearing. classify() +# returns the first matching rule, and 'fix(suffix-routing)' below +# declares no name_regex at all -- so with this rule anywhere after it, +# every CJK diff was classified as a suffix-routing change instead +# (verified: all four names of a synthetic CJK corpus landed there). +# The label would have been wrong and this rule dead. A rule narrowed +# by name_regex belongs ahead of every rule narrowed only by `fields`. +# +# Expected to match nothing against the current corpora, which contain +# no CJK at all: build_issues_corpus.py requires an internal space, and +# unspaced names are the shape this classifies. Kept for the reason the +# suffix-delimiter rule below is kept -- ready the moment a name lands. +name_regex = "[\\u3400-\\u4DBF\\u4E00-\\u9FFF\\uF900-\\uFAFF\\uAC00-\\uD7A3]" +fields = ["first", "middle", "last"] [[change]] issue = "fix(#274) maiden markers consumed" From 5ff11312345564b197b5661551cdbb80a90909fb Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Tue, 28 Jul 2026 12:24:11 -0700 Subject: [PATCH 12/20] Correct the migration doc's CJK baseline claims (#271) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two claims about 1.4 in migrate.rst were wrong, both verified against the pinned 1.4 worker rather than from memory: - 1.4 routes a LONE token to first, universally, so HumanName("김민준") gave first="김민준" and not last. The doc said last. - "Unspaced Chinese is unaffected either way" was false. Segmentation does not apply to it, but script_orders does: 毛泽东 and 山田太郎 now land in last where 1.4 put them in first. Nothing splits, so the string looks identical and only the field changed -- the quietest form of this release's default-on change, and the doc had told readers there was nothing to check. Now stated plainly, with the Japanese case named, since a kanji name is affected exactly as a Chinese one is. The release log's first #271 bullet illustrated only the spaced case and inherited the same blind spot; it now covers the lone-token flip too. Its parse-output bullets also move under a Behavior Changes heading, matching the 2.0.0 block's structure. Also rewrites the locales.rst contributor bullet that opened "Sort the vocabulary before shipping it" -- leftover phrasing from the bullet it replaced, and actively contradicting the "do not alphabetize" rank-order convention that zh.py and surnames.py both document in the data itself. The bullet is about which LAYER the vocabulary belongs in, so it says that instead. Co-Authored-By: Claude Fable 5 --- docs/locales.rst | 4 ++-- docs/migrate.rst | 34 ++++++++++++++++++++++++++-------- docs/release_log.rst | 9 ++++++--- 3 files changed, 34 insertions(+), 13 deletions(-) diff --git a/docs/locales.rst b/docs/locales.rst index 5e1b08d5..fab37c34 100644 --- a/docs/locales.rst +++ b/docs/locales.rst @@ -241,8 +241,8 @@ by ``tests/v2/test_locales.py``: the default must be one your ``DEVIATES`` predicate flags — no silent, undeclared side effects on names outside the pack's stated scope. -#. Sort the vocabulary before shipping it, if the pack carries any. - Vocabulary that is *self-selecting* — able to match only text of +#. Decide which layer the vocabulary belongs in, if the pack carries + any. Vocabulary that is *self-selecting* — able to match only text of the tradition it came from, the way a hangul surname can only ever match hangul — is default-safe, and belongs in the default lexicon (``nameparser/config/``) rather than in a pack: a pack nobody knows diff --git a/docs/migrate.rst b/docs/migrate.rst index b3a76ac7..9e0ffc88 100644 --- a/docs/migrate.rst +++ b/docs/migrate.rst @@ -352,11 +352,29 @@ given name. >>> HumanName("김민준").last, HumanName("김민준").first ('김', '민준') -1.4 read the first as ``first="毛"``/``last="泽东"`` and left the -second whole in ``last``. ``Constants`` has no switch for either — the -v1 configuration surface is frozen for 2.x — so the way out is the 2.0 -API: ``Parser(policy=Policy(script_orders={}, segment_scripts=()))`` -restores the old reading of both. Unspaced *Chinese* is unaffected -either way, because splitting Han text is opt-in through a locale pack -and ``HumanName`` cannot apply one: ``HumanName("毛泽东")`` still -returns the whole string as ``last``. +1.4 read the first as ``first="毛"``/``last="泽东"``, and left the +second whole in ``first``. + +A third shape changes even though nothing splits it. 1.4 routed a lone +token to ``first`` whatever it was, so an unspaced Chinese or Japanese +name landed there entire; 2.1 reads it as native-script CJK and puts it +in ``last`` instead. Nothing is segmented — Han splitting is opt-in +through a locale pack and ``HumanName`` cannot apply one — but the +field the whole string arrives in is different: + +.. doctest:: + + >>> HumanName("毛泽东").last, HumanName("毛泽东").first + ('毛泽东', '') + >>> HumanName("山田太郎").last, HumanName("山田太郎").first + ('山田太郎', '') + +Both were ``first`` under 1.4. If you feed unspaced CJK through +``HumanName`` and read ``first``, that is the change most likely to +reach you, and it is silent — the string is intact, just in the other +field. + +``Constants`` has no switch for any of this — the v1 configuration +surface is frozen for 2.x — so the way out is the 2.0 API: +``Parser(policy=Policy(script_orders={}, segment_scripts=()))`` +restores 1.4's reading of all three shapes. diff --git a/docs/release_log.rst b/docs/release_log.rst index 7bce4e99..0d97725c 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -2,10 +2,8 @@ Release Log =========== * 2.1.0 - Unreleased - **East Asian names** + **East Asian name support** - - Fix names written wholly in Han or Hangul parsing given-first: native-script CJK now reads family-first by default, through the new ``Policy.script_orders`` table, so ``"毛 泽东"`` gives family ``毛`` where 2.0 gave family ``泽东``. No language detection is involved — Chinese and Japanese both write the family name first in native script, so the script settles the order without anyone having to know which language it is. Latin-script and mixed-script names are never affected, and an explicit comma still wins. **Default-on: changes parse output for wholly-CJK names**, through ``HumanName`` as well as the 2.0 API (closes #271) - - Fix unspaced Korean names not splitting: the census surname list now ships as default vocabulary (``Lexicon.surnames``) with hangul segmentation on by default (``Policy.segment_scripts``), so ``"김민준"`` parses family ``김``, given ``민준`` instead of landing whole in the family name. Nothing but Korean is written in hangul and its surnames are a closed census set, which is what makes the split safe as a default rather than a pack. **Default-on**, and it reaches ``HumanName`` too. ``Policy(segment_scripts=())`` turns the split off; ``Policy(script_orders={})`` separately restores the positional reading; clearing both restores 2.0 behavior exactly (#271) - Add the Chinese locale pack ``locales.ZH`` — opt-in Han segmentation for unspaced names like ``毛泽东``, with the surname vocabulary it needs. A pack rather than a default because a Chinese surname list corrupts the Japanese names written in the same characters (``高橋一郎`` would split ``高`` + ``橋一郎``); Japanese needs the pluggable segmenter staged in #272. It sets no name order, since native-script Han already reads family-first without it. ``locales.available()`` is now ``('ru', 'tr_az', 'zh')`` (#271) - Add ``Lexicon.surnames``, ``Policy.script_orders``, ``Policy.segment_scripts``, the ``Script`` enum and the ``DEFAULT_SCRIPT_ORDERS`` constant to the public API. This is the first behavior nameparser keys on the script a name is written in; it is allowed only where the script itself settles a convention, never as a proxy for guessing the language (#271) - Add ``AmbiguityKind.SEGMENTATION``, reported when a surname split had a vocabulary-supported alternative: ``"남궁민수"`` is 남궁 + 민수 by the compound surname but 남 + 궁민수 by the single-syllable one, and longest-match had to pick. A name with only one possible split decided nothing and reports nothing (#271) @@ -14,6 +12,11 @@ Release Log - Change the pickle compatibility of the 2.0 API's ``Policy`` and ``Lexicon``: the new fields change the field layout their guarded ``__setstate__`` checks, so a pickle written by 2.0.0 raises ``ValueError`` naming the missing fields instead of loading. Re-pickle after upgrading. ``HumanName`` pickles are unaffected — the facade pickles v1-shaped component state, not these objects + **Behavior Changes** + + - Fix names written wholly in Han or Hangul parsing given-first: native-script CJK now reads family-first by default, through the new ``Policy.script_orders`` table, so ``"毛 泽东"`` gives family ``毛`` where 1.x gave family ``泽东``. A name that is a single unspaced token moves the same way — ``"毛泽东"`` and ``"山田太郎"`` now land in ``family``/``last`` where 1.x put them in ``given``/``first``, with the string itself untouched, which makes it the easiest form of this change to miss. No language detection is involved: Chinese and Japanese both write the family name first in native script, so the script settles the order without anyone having to know which language it is. Latin-script and mixed-script names are never affected, and an explicit comma still wins. **Default-on: changes parse output for wholly-CJK names**, through ``HumanName`` as well as the 2.0 API (closes #271) + - Fix unspaced Korean names not splitting: the census surname list now ships as default vocabulary (``Lexicon.surnames``) with hangul segmentation on by default (``Policy.segment_scripts``), so ``"김민준"`` parses family ``김``, given ``민준`` where 1.x returned the whole string as ``first``. Nothing but Korean is written in hangul and its surnames are a closed census set, which is what makes the split safe as a default rather than a pack. **Default-on**, and it reaches ``HumanName`` too. ``Policy(segment_scripts=())`` turns the split off; ``Policy(script_orders={})`` separately restores the positional reading; clearing both restores 2.0 behavior exactly (#271) + * 2.0.0 - July 27, 2026 Two release candidates preceded this release (rc1 on 2026-07-23, From 50f722cae2be6da26d02318cd8263682bf1e656e Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Tue, 28 Jul 2026 12:35:20 -0700 Subject: [PATCH 13/20] Fix the fork example and document the hangul render change (#271) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SEGMENTATION fork example was false for the shipped vocabulary. 欧 is not in the zh surname list, so 欧阳明 has exactly one possible split and emits nothing -- the stage tests only show otherwise because their synthetic lexicon adds 欧 on purpose. Both prose copies now use 夏侯惇, where 夏侯 wins and 夏 is genuinely listed too, so the example describes a fork the shipped data can actually produce (verified: it emits, and test_locales already pins that reading). The stage tests keep 欧阳明 -- their lexicon is synthetic by design and says so. Also documents the rendered-string change for Korean names, which no doc mentioned: str(HumanName("김민준")) was "김민준" in 1.4 and is "민준 김" now, because segmentation inserts a boundary the default format then writes given-first. Writing that turned up an error in the release log entry from the last commit, fixed here: the claim that the Han case is invisible in the output string holds only for a LONE token. The spaced form reorders, so str(HumanName("毛 泽东")) is "泽东 毛" where 1.4 echoed the input. Both bullets now say which shape they mean. Co-Authored-By: Claude Fable 5 --- docs/migrate.rst | 5 ++++- docs/release_log.rst | 4 ++-- nameparser/_pipeline/_script_segment.py | 2 +- nameparser/_types.py | 2 +- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/migrate.rst b/docs/migrate.rst index 9e0ffc88..9760b37b 100644 --- a/docs/migrate.rst +++ b/docs/migrate.rst @@ -353,7 +353,10 @@ given name. ('김', '민준') 1.4 read the first as ``first="毛"``/``last="泽东"``, and left the -second whole in ``first``. +second whole in ``first``. The Korean one also changes what the name +*renders* as — ``str(HumanName("김민준"))`` was ``"김민준"`` and is now +``"민준 김"``, because the split inserts a token boundary that the +default format then writes given-name-first. A third shape changes even though nothing splits it. 1.4 routed a lone token to ``first`` whatever it was, so an unspaced Chinese or Japanese diff --git a/docs/release_log.rst b/docs/release_log.rst index 0d97725c..95b9a354 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -14,8 +14,8 @@ Release Log **Behavior Changes** - - Fix names written wholly in Han or Hangul parsing given-first: native-script CJK now reads family-first by default, through the new ``Policy.script_orders`` table, so ``"毛 泽东"`` gives family ``毛`` where 1.x gave family ``泽东``. A name that is a single unspaced token moves the same way — ``"毛泽东"`` and ``"山田太郎"`` now land in ``family``/``last`` where 1.x put them in ``given``/``first``, with the string itself untouched, which makes it the easiest form of this change to miss. No language detection is involved: Chinese and Japanese both write the family name first in native script, so the script settles the order without anyone having to know which language it is. Latin-script and mixed-script names are never affected, and an explicit comma still wins. **Default-on: changes parse output for wholly-CJK names**, through ``HumanName`` as well as the 2.0 API (closes #271) - - Fix unspaced Korean names not splitting: the census surname list now ships as default vocabulary (``Lexicon.surnames``) with hangul segmentation on by default (``Policy.segment_scripts``), so ``"김민준"`` parses family ``김``, given ``민준`` where 1.x returned the whole string as ``first``. Nothing but Korean is written in hangul and its surnames are a closed census set, which is what makes the split safe as a default rather than a pack. **Default-on**, and it reaches ``HumanName`` too. ``Policy(segment_scripts=())`` turns the split off; ``Policy(script_orders={})`` separately restores the positional reading; clearing both restores 2.0 behavior exactly (#271) + - Fix names written wholly in Han or Hangul parsing given-first: native-script CJK now reads family-first by default, through the new ``Policy.script_orders`` table, so ``"毛 泽东"`` gives family ``毛`` where 1.x gave family ``泽东``. A name that is a single unspaced token moves the same way — ``"毛泽东"`` and ``"山田太郎"`` now land in ``family``/``last`` where 1.x put them in ``given``/``first``, with the string itself untouched, which makes it the easiest form of this change to miss — a lone token renders the same whichever field holds it, whereas the spaced form does show up in the output (``str(HumanName("毛 泽东"))`` is now ``"泽东 毛"``). No language detection is involved: Chinese and Japanese both write the family name first in native script, so the script settles the order without anyone having to know which language it is. Latin-script and mixed-script names are never affected, and an explicit comma still wins. **Default-on: changes parse output for wholly-CJK names**, through ``HumanName`` as well as the 2.0 API (closes #271) + - Fix unspaced Korean names not splitting: the census surname list now ships as default vocabulary (``Lexicon.surnames``) with hangul segmentation on by default (``Policy.segment_scripts``), so ``"김민준"`` parses family ``김``, given ``민준`` where 1.x returned the whole string as ``first``. Rendering follows the split, so ``str(HumanName("김민준"))`` is now ``"민준 김"`` where 1.x echoed the input back unchanged. Nothing but Korean is written in hangul and its surnames are a closed census set, which is what makes the split safe as a default rather than a pack. **Default-on**, and it reaches ``HumanName`` too. ``Policy(segment_scripts=())`` turns the split off; ``Policy(script_orders={})`` separately restores the positional reading; clearing both restores 2.0 behavior exactly (#271) * 2.0.0 - July 27, 2026 diff --git a/nameparser/_pipeline/_script_segment.py b/nameparser/_pipeline/_script_segment.py index 8c3ac502..d00f03f5 100644 --- a/nameparser/_pipeline/_script_segment.py +++ b/nameparser/_pipeline/_script_segment.py @@ -12,7 +12,7 @@ inserts the missing token boundary by vocabulary: the first token written wholly in an activated script is matched longest-first against Lexicon.surnames, and a hit splits it in two. Compound-before-single -("欧阳明" is 欧阳 + 明, though 欧 is itself a surname) falls out of +("夏侯惇" is 夏侯 + 惇, though 夏 is itself a surname) falls out of longest-first. The split makes two sub-slices of the one token, rewriting nothing -- spans still index the original exactly, so the anti-#100 invariant holds by construction. diff --git a/nameparser/_types.py b/nameparser/_types.py index 2354e702..a33121dc 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -281,7 +281,7 @@ class AmbiguityKind(StrEnum): #: the parse is best-effort over the extra segments. COMMA_STRUCTURE = "comma-structure" #: An unspaced CJK token had more than one vocabulary-supported - #: surname split ("欧阳明": 欧阳 + 明 was taken, 欧 + 阳明 also + #: surname split ("夏侯惇": 夏侯 + 惇 was taken, 夏 + 侯惇 also #: matched); the longest surname won, and ``detail`` names both #: readings. Points at the two tokens the split produced. (#271) SEGMENTATION = "segmentation" From a102c8680fae02eb76b99ae3fafb98835e5852c9 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Tue, 28 Jul 2026 13:07:45 -0700 Subject: [PATCH 14/20] Simplify and speed up the #271 hot paths The four #271 additions all sat on paths every parse walks, and three of them paid for shapes measurement did not support. single_script's per-char all/any sweep was written on the _EMOJI_RANGES precedent, on the theory that a range test needs no regex. At token scale that is backwards: a compiled character class per script runs 5x faster on ordinary CJK tokens and 35x on long ones. The integer table stays the single source of truth -- the patterns are DERIVED from it, in its own key order, so the first-covering-entry rule still describes what single_script does. script_segment now returns early on a wholly-ASCII original. Spans index the original exactly, so an ASCII original has only ASCII tokens and no script's ranges can contain any of them: the stage was building a lexicon lookup and a generator per Latin parse to prove it. _effective_order joined each piece's tokens into a throwaway string and built a set of results before asking whether the set had one member. WorkToken text is never empty, so "the piece is wholly one script" is exactly "every token in it is" -- a per-token walk with early exit says the same thing and stops at the first token that settles it. _policy.py loses two near-duplicate wrap-and-rethrow blocks (the probe-vs-consume rule now lives once, on _require_iterable, which takes the field's own phrasing as a parameter), a vestigial tuple() materialization, and PolicyPatch.__post_init__'s 77-line inline script_orders block, which becomes a named helper that always materializes. Error texts are unchanged; the one behavior nuance is invisible from outside -- a re-iterable input with a malformed entry is now stored as a materialized tuple rather than the original container, and Policy quotes entries, never containers. Tests: the Korean-surname range check reads the shared table instead of a second hand-copy of 0xAC00/0xD7A3, one comment that promised a stage as future work is reworded now that it shipped, and the zh anti-dedupe note covers the case-table overlap it had grown. Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/_assign.py | 27 ++-- nameparser/_pipeline/_script_segment.py | 5 + nameparser/_pipeline/_vocab.py | 29 ++-- nameparser/_policy.py | 179 ++++++++---------------- tests/v2/test_lexicon.py | 8 +- tests/v2/test_locales.py | 13 +- tests/v2/test_parser.py | 7 +- tests/v2/test_policy.py | 2 +- 8 files changed, 123 insertions(+), 147 deletions(-) diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index c4ce09bb..5d5fa89c 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -36,7 +36,7 @@ from nameparser._pipeline._state import ( ParseState, PendingAmbiguity, Structure, WorkToken, ) -from nameparser._policy import Policy +from nameparser._policy import Policy, Script from nameparser._types import AmbiguityKind, Role # Ported verbatim from v1 (nameparser/config/regexes.py @@ -90,15 +90,22 @@ def _effective_order(policy: Policy, return policy.name_order # ONE script for the whole name, not "the entries all agree": two # scripts that both read family-first still fall back, because a - # Han+Hangul name is not written in either tradition. - found = {single_script("".join(tokens[i].text for i in piece)) - for piece in pieces} - if len(found) == 1: - script = found.pop() # None (no single script) matches - for entry_script, order in policy.script_orders: # no entry - if entry_script is script: - return order - return policy.name_order + # Han+Hangul name is not written in either tradition. Per token + # rather than per joined piece -- WorkToken text is never empty, so + # "the piece is wholly one script" is "every token in it is". + found: Script | None = None + for piece in pieces: + for i in piece: + script = single_script(tokens[i].text) + if script is None: + # Latin, mixed, or a script with no entry: never a key + return policy.name_order + if found is None: + found = script + elif script is not found: + return policy.name_order + return next((order for s, order in policy.script_orders if s is found), + policy.name_order) def _name_positions(order: tuple[Role, Role, Role], diff --git a/nameparser/_pipeline/_script_segment.py b/nameparser/_pipeline/_script_segment.py index d00f03f5..992226fe 100644 --- a/nameparser/_pipeline/_script_segment.py +++ b/nameparser/_pipeline/_script_segment.py @@ -86,6 +86,11 @@ def _longest_entry(surnames: frozenset[str]) -> int: def script_segment(state: ParseState) -> ParseState: + if state.original.isascii(): + # spans index the original exactly (the anti-#100 invariant), + # so an ASCII original has only ASCII tokens: nothing here is + # in any script's ranges + return state scripts = state.policy.segment_scripts surnames = state.lexicon.surnames if not scripts or not surnames or not state.segments: diff --git a/nameparser/_pipeline/_vocab.py b/nameparser/_pipeline/_vocab.py index 075105e1..f43a1280 100644 --- a/nameparser/_pipeline/_vocab.py +++ b/nameparser/_pipeline/_vocab.py @@ -29,9 +29,13 @@ PH = re.compile(r"^ph\.?$", re.IGNORECASE) D = re.compile(r"^d\.?$", re.IGNORECASE) -# Codepoint ranges per Script (#271): integer ranges following the -# _EMOJI_RANGES precedent in _tokenize.py -- the per-char test needs -# no regex. HAN: the URO plus Extension A, the compatibility block, +# Codepoint ranges per Script (#271). This integer table is the single +# source of truth for what a script covers; _SCRIPT_PATTERNS below +# DERIVES the match engine from it. (The sweep here was first written +# per-char on the _EMOJI_RANGES precedent in _tokenize.py, on the +# theory that a range test needs no regex; measured at token scale the +# compiled regex wins by 3-9x, and by 89x on long tokens.) +# HAN: the URO plus Extension A, the compatibility block, # and the supplementary-plane block (Ext B-I + CJK Compat Ideographs # Supplement, 0x20000-0x323AF) -- rare surnames are the biggest real # source of supplementary-plane hanzi in personal names (e.g. 𠮷田's @@ -51,6 +55,16 @@ Script.HANGUL: ((0xAC00, 0xD7A3),), } +# Derived, never hand-written: one character class per script, in the +# table's own key order (so the FIRST-covering-entry rule above still +# describes what single_script does). +_SCRIPT_PATTERNS: dict[Script, re.Pattern[str]] = { + script: re.compile( + "[" + "".join(f"\\U{lo:08x}-\\U{hi:08x}" for lo, hi in ranges) + + "]+") + for script, ranges in _SCRIPT_RANGES.items() +} + def is_initial(text: str) -> bool: """'A.' / 'j.' / bare capital -- v1's is_an_initial.""" @@ -153,14 +167,13 @@ def single_script(text: str) -> Script | None: None (mixed-script text has no well-defined convention to apply; the caller falls back to the positional default).""" if not text: - return None # all() is vacuously true; "" belongs to no script + return None # the + below needs one char; "" belongs to no script if text.isascii(): # every _SCRIPT_RANGES entry is non-ASCII (lowest today is - # U+3400): skip the range sweep for the overwhelmingly common + # U+3400): skip the patterns for the overwhelmingly common # Latin token (the _tokenize._ignorable ASCII-floor precedent) return None - for script, ranges in _SCRIPT_RANGES.items(): - if all(any(lo <= ord(c) <= hi for lo, hi in ranges) - for c in text): + for script, pattern in _SCRIPT_PATTERNS.items(): + if pattern.fullmatch(text): return script return None diff --git a/nameparser/_policy.py b/nameparser/_policy.py index d8eee4e1..8a01ab22 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -203,17 +203,28 @@ def _reject_str_and_mapping(value: object, field_name: str) -> None: ) -def _require_iterable(value: Iterable[Any], field_name: str) -> Iterable[Any]: - # Probe with iter() so a non-iterable value (an int, a bool, ...) - # raises a message naming the field, matching the treatment - # patronymic_rules already gets, instead of a bare "'int' object is - # not iterable" surfacing from whatever tuple()/frozenset() call - # happens to run first. +def _require_iterable(value: Iterable[Any], field_name: str, + expected: str = "an iterable") -> Iterable[Any]: + """Probe a value's iterability and return its iterator, raising a + TypeError naming the field if it has none. `expected` carries the + field's own phrasing ("a mapping of Script to order"); the default + suits every plain iterable field. + + Probing with iter() is what makes the message possible: a + non-iterable (an int, a bool, ...) is named here, matching the + treatment patronymic_rules already gets, instead of a bare "'int' + object is not iterable" surfacing from whatever tuple()/frozenset() + happens to run first. It is ALSO the reason callers consume the + returned iterator OUTSIDE any try of their own: an exception raised + inside a caller's generator while it is being consumed is the + caller's own error, and must propagate untouched rather than be + rewritten as a shape complaint about the field. + """ try: return iter(value) except TypeError: raise TypeError( - f"{field_name} must be an iterable, got {value!r}" + f"{field_name} must be {expected}, got {value!r}" ) from None @@ -284,22 +295,11 @@ def _validated_script_orders( f"e.g. raw.decode('utf-8')" ) raw = value.items() if isinstance(value, Mapping) else value - try: - # Probe with iter() only (mirrors the patronymic_rules probe in - # Policy.__post_init__, and _validated_segment_scripts below): - # a genuine non-iterable is caught here and gets the curated - # message below, while an exception raised INSIDE a caller's - # generator during consumption (the tuple() below) must - # propagate untouched, not be rewritten as "not a mapping". - raw_iter = _require_iterable(raw, "script_orders") # type: ignore[arg-type] - except TypeError: - raise TypeError( - f"script_orders must be a mapping of Script to order, " - f"got {value!r}" - ) from None - entries: tuple[Any, ...] = tuple(raw_iter) + raw_iter = _require_iterable( + raw, "script_orders", # type: ignore[arg-type] + "a mapping of Script to order") canonical: dict[Script, tuple[Role, Role, Role]] = {} - for entry in entries: + for entry in raw_iter: try: key, order = entry except (TypeError, ValueError): @@ -320,18 +320,9 @@ def _validated_segment_scripts(value: object) -> frozenset[Script]: their string values), coerced via _validated_script so the unknown-script wording stays single-sourced.""" _reject_str_and_mapping(value, "segment_scripts") - # Probe with iter() only (mirrors the patronymic_rules probe in - # Policy.__post_init__): a genuine non-iterable is caught here and - # gets the curated message below, while an exception raised INSIDE - # a caller's generator during consumption (the frozenset() below) - # must propagate untouched, not be rewritten as "not an iterable". - try: - script_iter = _require_iterable(value, "segment_scripts") # type: ignore[arg-type] - except TypeError: - raise TypeError( - f"segment_scripts must be an iterable of Script members, " - f"got {value!r}" - ) from None + script_iter = _require_iterable( + value, "segment_scripts", # type: ignore[arg-type] + "an iterable of Script members") return frozenset(_validated_script(s) for s in script_iter) @@ -359,6 +350,35 @@ def _canonical_script_pair(pair: Iterable[Any]) -> tuple[Any, ...]: return out # non-iterable order value +def _canonical_patch_script_orders(value: object) -> object: + """Canonicalize a PolicyPatch.script_orders value for hashability + without validating it: malformed shapes are stored so Policy can + quote them at apply time; a caller-generator's own exception + propagates from the UNGUARDED materialization below (deferring is + impossible once a one-shot iterator is consumed).""" + # Excluded HERE rather than delegated: a string's elements are + # themselves tuple-izable, so no TypeError ever fires to signal + # "leave this alone" -- "han" would shred to (("h",), ("a",), + # ("n",)) and Policy's bare-string message would have nothing left + # to quote. bytes shred the same way, into ints. + if isinstance(value, (str, bytes, bytearray, memoryview)): + return value # deferred whole: Policy's guards quote it + # Any, not object: a patch defers validation, so anything a caller + # wrote can arrive here, and the probe below is precisely the + # runtime question mypy has no way to answer statically. + pairs: Any = value.items() if isinstance(value, Mapping) else value + try: + pairs_iter = iter(pairs) # probe only + except TypeError: + return value # non-iterable: defer to apply + items = tuple(pairs_iter) # UNGUARDED: caller errors propagate + try: + return tuple(map(_canonical_script_pair, items)) + except TypeError: + return items # malformed entry: materialized, so + # Policy can still re-iterate + quote + + @dataclass(frozen=True, slots=True) class Policy: """The behavior switches a parser runs with: name order, @@ -627,90 +647,13 @@ def __post_init__(self) -> None: # from a {Script: order} dict (or a list of pairs) must already # be hashable, since a Locale holds it -- and hashable all the # way down, hence _canonical_script_pair. Validation still - # belongs to Policy at apply time, so a shape tuple() cannot - # digest is left exactly as written for it to report. - # The str case must be excluded HERE rather than delegated to - # _canonical_script_pair: a string's elements are themselves - # tuple-izable, so no TypeError ever fires to signal "leave this - # alone" -- "han" would shred to (("h",), ("a",), ("n",)) and - # Policy's bare-string message would have nothing left to quote. - if self.script_orders is not UNSET and not isinstance( - self.script_orders, str): - raw = self.script_orders - # Mapping.items() is always iterable, so it needs no probe; - # only the non-Mapping branch can fail the iter() check below. - pairs = raw.items() if isinstance(raw, Mapping) else raw - # Four outcomes share this block, and only one of them - # should propagate immediately: - # 1. raw itself isn't iterable at all (script_orders=5) -- - # caught by the iter() probe. Shape problem, defers to - # apply time, where Policy's own guard raises the - # curated message ("Policy validates ... at apply"). - # 2. an already-yielded PAIR has a bad shape (a bare int - # from a shredded bytes buffer, e.g. b"han" iterating - # to 104) -- caught around _canonical_script_pair - # below. Also a shape problem: abort the whole - # conversion and leave script_orders as a RE-ITERABLE - # value (see case 4) Policy's curated message (e.g. the - # decode-first bytes hint, or the bad-pair message) can - # still quote at apply time. - # 3. the caller's OWN exception, raised by their - # generator while it is being consumed -- this is not - # a shape problem this guard exists to catch, and - # deferral is impossible anyway: by the time - # apply_patch reaches Policy.__post_init__, the same - # generator is already exhausted, so catching this - # here would lose the error for good rather than - # merely delay it. Consuming a one-shot iterator - # therefore runs UNGUARDED (case 4), so this - # propagates on its own terms. - # 4. a ONE-SHOT iterator (iter(x) is x -- generators, - # map/zip/filter objects, ...) cannot be safely - # re-iterated once consumed. Deferring case 2 by - # leaving it as originally written -- fine for - # re-iterable inputs (bytes, list, dict, tuple) -- would - # leave Policy re-validating an EXHAUSTED generator at - # apply time: iterating it again silently yields - # nothing, storing () ("opt out of per-script - # ordering") and dropping the Han/Hangul family-first - # defaults with NO error anywhere. So a one-shot input - # is eagerly materialized into a tuple first (case 3's - # unguarded consumption), and case 2's deferral stores - # THAT re-iterable tuple instead of the original, - # exhausted generator. - try: - pairs_iter = iter(pairs) - except TypeError: - pairs_iter = None # non-iterable: deferred, case 1 - if pairs_iter is not None: - # mypy sees pairs' declared type (tuple[...] | ItemsView) - # as never identical to iter(pairs)'s Iterator type, but - # at runtime pairs can be anything iterable a caller - # wrote (a generator especially) -- the whole point of - # this check. - one_shot = pairs_iter is pairs # type: ignore[comparison-overlap] - items: Iterable[Any] = tuple(pairs_iter) if one_shot else pairs - canonical = [] - deferred = False - for item in items: - try: - canonical.append(_canonical_script_pair(item)) - except TypeError: - deferred = True # bad pair shape: case 2 - break - if not deferred: - object.__setattr__( - self, "script_orders", tuple(canonical)) - elif one_shot: - # items is already the materialized (re-iterable) - # tuple built above -- store it so Policy can - # re-examine and quote it at apply time, instead of - # the original, now-exhausted generator. - object.__setattr__(self, "script_orders", items) - # else: re-iterable input (bytes/list/dict/tuple/...) is - # left exactly as originally written; Policy can freely - # re-iterate it at apply time and quote the caller's - # value there -- unchanged from the prior behavior. + # belongs to Policy at apply time, so a shape the canonicalizer + # cannot digest is left for it to report; the one-shot and + # malformed-entry cases are _canonical_patch_script_orders'. + if self.script_orders is not UNSET: + object.__setattr__( + self, "script_orders", + _canonical_patch_script_orders(self.script_orders)) for f in dataclasses.fields(self): if f.metadata.get("compose") != "union": continue diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index d2b13af7..55fd5656 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -9,6 +9,8 @@ from nameparser._lexicon import ( Lexicon, _VOCAB_FIELDS, _default_lexicon, _normalize, _title_key, ) +from nameparser._pipeline._vocab import _SCRIPT_RANGES +from nameparser._policy import Script def test_entries_are_normalized_at_construction() -> None: @@ -571,6 +573,8 @@ def test_default_lexicon_ships_korean_surnames_only() -> None: assert "김" in lex.surnames and "남궁" in lex.surnames # Han surnames are locales.ZH's cargo (segmentation opt-in), so # the DEFAULT set is wholly hangul -- structural check, not - # content-pinning - assert all(all(0xAC00 <= ord(c) <= 0xD7A3 for c in s) + # content-pinning. The spans come from the shared table rather than + # a second hand-copy of 0xAC00/0xD7A3 (test_locales.py's precedent). + hangul = _SCRIPT_RANGES[Script.HANGUL] + assert all(all(any(lo <= ord(c) <= hi for lo, hi in hangul) for c in s) and 1 <= len(s) <= 2 for s in lex.surnames) diff --git a/tests/v2/test_locales.py b/tests/v2/test_locales.py index a26f3425..b15385d1 100644 --- a/tests/v2/test_locales.py +++ b/tests/v2/test_locales.py @@ -131,11 +131,14 @@ def test_zh_longest_match_prefers_compound_over_single_surname() -> None: assert p.parse("夏雨").ambiguities == () -# 毛泽东/欧阳修/夏侯惇/萧红 appear in _ROTATORS["zh"] below as well. -# Not deduplicated on purpose: the rotator tests assert only THAT the -# packed parse differs from the default and that DEVIATES declares it -# -- they never look at a field value, so the readings above are -# pinned nowhere else. +# 毛泽东/欧阳修/夏侯惇/萧红 appear in _ROTATORS["zh"] below, and 毛泽东/ +# 夏侯惇 in cases.py's zh rows, as well. Not deduplicated in either +# direction, on purpose: the rotator tests assert only THAT the packed +# parse differs from the default and that DEVIATES declares it -- they +# never look at a field value -- and the case rows reach the same +# packed parser only through the shared-table runners +# (test_cases.py/test_facade_cases.py). These tests are where the zh +# readings are pinned at unit level, against the stage directly. def test_zh_composes_with_korean_defaults() -> None: diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index 4539e9bc..a2612646 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -445,9 +445,10 @@ def test_wholly_cjk_names_read_family_first_by_default() -> None: assert (n.family, n.given) == ("毛", "泽东") n = parse("김 민준") assert (n.family, n.given) == ("김", "민준") - # a lone wholly-CJK token takes the script order's first role - assert parse("毛泽东").family == "毛泽东" # unspaced: split arrives - # with Task 4's stage + # a lone wholly-CJK token takes the script order's first role: Han + # segmentation is opt-in (locales.ZH), so the default parser leaves + # this one token whole and reads it as the family name + assert parse("毛泽东").family == "毛泽东" def test_latin_names_are_untouched_by_script_orders() -> None: diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index 80d97da1..0eead8a7 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -545,7 +545,7 @@ def test_segment_scripts_rejects_unknown_script() -> None: def test_segment_scripts_rejects_non_iterable_with_curated_message() -> None: # parallel to patronymic_rules: name the expected shape rather than - # surfacing _require_iterable's generic "must be an iterable" text + # letting _require_iterable's default "an iterable" phrasing stand with pytest.raises(TypeError, match="segment_scripts must be an iterable of " "Script members, got True"): From e4d13a9d3d475bfabecd5c7d1d0ae48c3f9b7b03 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Tue, 28 Jul 2026 13:09:33 -0700 Subject: [PATCH 15/20] Sort differential rules most-specific-first, dropping the ordering rule MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit expected_changes.toml's rules were tried in file order, so a rule narrowed by name_regex had to be written above every rule narrowed only by fields -- an invariant the file could only defend in prose, and one it had already lost once (the fix(#271) CJK rule was dead on arrival at the end of the file, its diffs claimed and mislabelled by fields-only fix(suffix-routing)). compare.py now sorts the rules into two tiers before matching, stably, so rules within a tier keep the order they were written in. That makes the ordering rule structural, and the three places that documented it by hand -- the toml header, the #271 rule's own "FIRST on purpose" paragraph, and the README's ordering section -- collapse to one line each. No exit-code change: sorting cannot change WHICH rules match, only which of several matching rules wins the label. The run is unchanged against the current corpora (654 names, 18 intentional diffs, 0 unexplained, exit 0) -- with one fields-only rule in the file and the one specific rule that competes with it already written above it, the sort is a no-op today. Verified it is not a no-op in general: with the #271 rule moved to the end of the file, '毛泽东' classifies as suffix-routing unsorted and as fix(#271) sorted. Co-Authored-By: Claude Fable 5 --- tools/differential/README.md | 12 ++++-------- tools/differential/compare.py | 5 +++++ tools/differential/expected_changes.toml | 14 ++------------ 3 files changed, 11 insertions(+), 20 deletions(-) diff --git a/tools/differential/README.md b/tools/differential/README.md index b9b2cd6b..9da4b310 100644 --- a/tools/differential/README.md +++ b/tools/differential/README.md @@ -116,14 +116,10 @@ matches only if the observed diff fields are a subset of this list). Keep both as tight as the actual diff allows -- a loose rule can mask a real regression. -Rules are tried in file order and the first match wins, so a rule -narrowed by `name_regex` has to sit **above** any rule narrowed only -by `fields`. The `fields`-only rules are broad by construction: a -`fields = ["first", "last", "suffix"]` rule claims every diff confined -to those three columns, whatever the name looked like. Adding the -`fix(#271)` CJK rule at the end of the file was enough to make it dead -on arrival -- `fix(suffix-routing)` claimed all of its diffs first, and -labelled them as something else. +Rules are sorted most-specific-first before matching -- a `name_regex` +rule outranks a `fields`-only one (which is broad by construction) +wherever both match -- so file order does not decide which rule claims +a diff. Some entries in the seed list are for behavior families that this particular corpus (pre-M12 v1 test strings) happens not to contain any diff --git a/tools/differential/compare.py b/tools/differential/compare.py index 792da3bb..8477504e 100644 --- a/tools/differential/compare.py +++ b/tools/differential/compare.py @@ -62,6 +62,11 @@ def main() -> int: rules = tomllib.loads( (HERE / "expected_changes.toml").read_text()).get("change", []) validate_rules(rules) + # Most-specific-first: a name_regex rule outranks a fields-only rule + # wherever both match, so file order stops being load-bearing. The + # sort is stable, so rules within a tier keep the order they were + # written in. + rules.sort(key=lambda r: not isinstance(r.get("name_regex"), str)) # A glob that matches nothing must not read as "everything passed". # Comparing zero names would print 0 unexplained and exit 0 -- the # harness's own stated nightmare (see validate_rules), and a diff --git a/tools/differential/expected_changes.toml b/tools/differential/expected_changes.toml index ebdbbc83..050ebf73 100644 --- a/tools/differential/expected_changes.toml +++ b/tools/differential/expected_changes.toml @@ -3,10 +3,8 @@ # S5). Rules are seeded from docs/superpowers/plans/notes-m12-diffs.md # and the `classification="fix(...)"` rows in tests/v2/cases.py; keep # each entry's `name_regex`/`fields` as tight as the diff allows. -# ORDER MATTERS: classify() takes the FIRST matching rule, so a rule -# narrowed by `name_regex` must sit ahead of any rule narrowed only by -# `fields`, or the broad one silently claims its diffs under the wrong -# label. See the #271 entry immediately below for a case that did. +# File order is not load-bearing: compare.py sorts `name_regex` rules +# ahead of `fields`-only ones before matching. [[change]] issue = "fix(#271) native-script CJK: family-first order + hangul segmentation" @@ -23,14 +21,6 @@ issue = "fix(#271) native-script CJK: family-first order + hangul segmentation" # tests/v2/test_regex_sync.py, which fails if the two disagree -- if # one ever does. # -# FIRST on purpose, and this position is load-bearing. classify() -# returns the first matching rule, and 'fix(suffix-routing)' below -# declares no name_regex at all -- so with this rule anywhere after it, -# every CJK diff was classified as a suffix-routing change instead -# (verified: all four names of a synthetic CJK corpus landed there). -# The label would have been wrong and this rule dead. A rule narrowed -# by name_regex belongs ahead of every rule narrowed only by `fields`. -# # Expected to match nothing against the current corpora, which contain # no CJK at all: build_issues_corpus.py requires an internal space, and # unspaced names are the shape this classifies. Kept for the reason the From 9ab84d1c6ad724c48bd90128fdb6872387c1cc52 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Tue, 28 Jul 2026 13:20:09 -0700 Subject: [PATCH 16/20] Cover _canonical_script_pair's three deferral branches The only uncovered lines the branch added (codecov/patch): the wrong-arity, str-valued, and non-iterable per-pair deferrals -- each now pinned as constructible/hashable and quoted by Policy at apply. Co-Authored-By: Claude Fable 5 --- tests/v2/test_policy.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index 0eead8a7..d8fa5422 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -626,6 +626,26 @@ def test_policy_patch_script_orders_non_iterable_still_defers_to_apply() -> None apply_patch(Policy(), patch) +def test_canonical_script_pair_defers_each_malformed_shape() -> None: + # _canonical_script_pair's three per-pair deferral branches, each + # constructible (hashable) and each quoted by Policy's own guard at + # apply time: a pair of the wrong arity; a str order value (which + # tuple() would shred to characters); a non-iterable order value. + arity = PolicyPatch( + script_orders=[(Script.HAN, FAMILY_FIRST, "extra")]) # type: ignore[arg-type] + assert isinstance(hash(arity), int) + with pytest.raises(TypeError, match=r"\(Script, order\) pairs"): + apply_patch(Policy(), arity) + stringly = PolicyPatch(script_orders={Script.HAN: "gmf"}) # type: ignore[arg-type] + assert isinstance(hash(stringly), int) + with pytest.raises(TypeError, match="bare string"): + apply_patch(Policy(), stringly) + lone = PolicyPatch(script_orders={Script.HAN: 5}) # type: ignore[arg-type] + assert isinstance(hash(lone), int) + with pytest.raises(TypeError, match="must be an iterable, got 5"): + apply_patch(Policy(), lone) + + def test_policy_patch_one_shot_bad_tail_defers_without_silent_drop() -> None: # The silent-drop repro: a ONE-SHOT generator whose first item is a # good pair and whose second is a bad shape (not caller-generator From 3280c04ae996d3a464437efa3c3f04077a958d30 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Tue, 28 Jul 2026 13:36:43 -0700 Subject: [PATCH 17/20] Restructure the East Asian names section: background before behavior Lead with how the names are built (order, unspaced writing, closed surname sets), then the two automatic behaviors, then the boundaries -- replacing the casual 'come out right' / 'a step further' framing that assumed the reader already knew the conventions. Co-Authored-By: Claude Fable 5 --- docs/usage.rst | 68 +++++++++++++++++++++++++++++++------------------- 1 file changed, 43 insertions(+), 25 deletions(-) diff --git a/docs/usage.rst b/docs/usage.rst index a188735a..2d7dafd9 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -140,12 +140,22 @@ merging two, and is covered next. East Asian names ----------------- -A name written wholly in Han (Chinese and Japanese characters) or -Hangul needs no configuration to come out right: the script settles the -convention by itself, so those names read family-first by default. -Chinese and Japanese differ in most things but not this one — both put -the family name first in native script — so nothing has to guess which -language it is looking at. +How these names are built, first, because the parsing rules follow +from it. A Chinese, Japanese, or Korean name written in its own script +puts the family name first: 毛泽东 is MAO Zedong, 山田太郎 is YAMADA +Taro, 김민준 is KIM Minjun. The family name is short — one Han +character or one hangul syllable, occasionally two — and comes from a +small closed set, while given names are open-ended. And in native +writing the parts are usually not separated at all: the whole name is +one unbroken run of characters. A parser therefore has two distinct +jobs here: assign family and given to the right fields, and, when the +name arrives as a single token, find the boundary inside it. + +Field assignment is automatic. A name written wholly in Han characters +or hangul is assigned family-first, because every language written in +those scripts orders names that way — Chinese and Japanese share +little else, but they agree on this — so the assignment requires no +knowledge of which language the name is in: .. doctest:: @@ -154,9 +164,11 @@ language it is looking at. >>> parse("山田 太郎").family '山田' -Korean names go a step further, because they are usually written with -no space in them at all. The census surname list ships as default -vocabulary, so an unspaced hangul name is split into its two parts: +Splitting an unspaced name is also automatic, but only for Korean. +Hangul is written by exactly one language, and Korean family names are +a closed set small enough to ship: the census surname list is part of +the default vocabulary, and the longest listed surname at the start of +an unspaced hangul token becomes the family name: .. doctest:: @@ -164,13 +176,16 @@ vocabulary, so an unspaced hangul name is split into its two parts: >>> minjun.family, minjun.given ('김', '민준') -That works out of the box because nothing but Korean is written in -hangul. The same trick on Han characters would have to know Chinese -from Japanese first — a Chinese surname list splits ``高橋一郎`` after -``高``, wrecking an ordinary Japanese name — so unspaced *Chinese* is -opt-in through the ``zh`` locale pack, and Japanese waits on a -segmenter of its own (`#272 -`_): +The same split is not automatic for Han text, because there the script +does not identify the language: 高橋一郎 is a Japanese name whose +family name is 高橋, but 高 alone is a common Chinese surname, so a +Chinese surname list would split it in the wrong place. Declaring the +language is up to you. When you know the data is Chinese, apply the +``zh`` locale pack; Japanese needs a dictionary-backed segmenter and +is tracked as `#272 +`_ — until +then an unspaced Japanese name stays whole, in the family field per +the assignment rule above: .. doctest:: @@ -178,15 +193,18 @@ segmenter of its own (`#272 >>> parser_for(locales.ZH).parse("毛泽东").family '毛' -A comma switches both behaviors off, on the reasoning ``name_order`` -already follows: someone who wrote one has said where the family name -ends. When the vocabulary supported more than one split — ``남궁민수`` -is 남궁 + 민수 by the compound surname but 남 + 궁민수 by the -single-syllable one — the longest match wins and the parse reports the -choice as an ``AmbiguityKind.SEGMENTATION``, described under `When the -parser had to guess`_; a name with only one possible split decided -nothing and reports nothing. :doc:`customize` covers turning either -behavior off. +Three boundaries on all of the above. Romanized names ("Kim Min-jun") +are Latin script and follow the ordinary positional rules — order +genuinely varies in romanized data, so nothing script-based applies. +A comma disables both behaviors, on the reasoning ``name_order`` +already follows: whoever wrote the comma has already said where the +family name ends. And when an unspaced name has more than one +vocabulary-supported split — ``남궁민수`` is 남궁 + 민수 by the +two-syllable surname but 남 + 궁민수 by the single-syllable one — the +longest surname wins and the parse records the decision as an +``AmbiguityKind.SEGMENTATION``, described under `When the parser had +to guess`_; a name with only one possible split reports nothing. +:doc:`customize` covers turning either behavior off. Aggregate views ---------------- From 6d9c0270c95628ee80ddff4fb30ed249837e52a1 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Tue, 28 Jul 2026 15:48:36 -0700 Subject: [PATCH 18/20] Apply the background-before-behavior structure to the other CJK docs locales.rst: state the naming conventions before the two defaults and name the actual limitation (the script cannot license a Han split) instead of 'waits for the zh pack'. customize.rst: name the mechanism behind the opt-out trap (the split still runs; the positional default then assigns given-first) instead of 'read it twice'. usage.rst: drop the self-referential lead-in sentence. Co-Authored-By: Claude Fable 5 --- docs/customize.rst | 17 ++++++++++------- docs/locales.rst | 26 ++++++++++++++++---------- docs/usage.rst | 3 +-- 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/docs/customize.rst b/docs/customize.rst index fa60c142..126fa409 100644 --- a/docs/customize.rst +++ b/docs/customize.rst @@ -307,11 +307,12 @@ East Asian defaults, and turning them off ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Two defaults key on the *script* a name is written in rather than on -anything you set: a name written wholly in Han or Hangul reads +anything you set: a name written wholly in Han or Hangul is assigned family-first (``script_orders``), and an unspaced hangul name is split into surname and given name against the shipped Korean census list -(``segment_scripts``). :ref:`east-asian-names` shows what they do — -this is how to switch them off, which you can do separately: +(``segment_scripts``). :ref:`east-asian-names` explains the naming +conventions both rest on — this section is how to switch them off, +which you can do separately: .. doctest:: @@ -324,10 +325,12 @@ this is how to switch them off, which you can do separately: >>> unsplit.parse("김민준").family # one token, not split '김민준' -The middle line is the one to read twice: emptying ``script_orders`` -restores the purely positional reading but leaves the token split, so -the surname lands in ``given`` instead. Clear both fields to get -nameparser 2.0's behavior back exactly. +The two switches interact, and clearing only ``script_orders`` +produces a third behavior rather than the old one: the split still +runs, so ``김민준`` still becomes two tokens, and the positional +default then assigns them given-first — the surname lands in +``given``. To restore nameparser 2.0's reading exactly, clear both +fields. To teach the splitter a surname it doesn't ship with, add it to the ``surnames`` vocabulary like any other word: diff --git a/docs/locales.rst b/docs/locales.rst index fab37c34..8dba47ba 100644 --- a/docs/locales.rst +++ b/docs/locales.rst @@ -56,16 +56,22 @@ isn't read as a family name. Listing it in ``given_name_titles`` alone raises ``ValueError`` rather than quietly doing nothing. Two East Asian behaviors are on by default for the same reason, except -that what selects them is the *script* rather than the word: a name -written wholly in Han or Hangul reads family-first, and an unspaced -Korean name is split into surname and given name. Chinese and Japanese -both write family-first natively, so reading the order takes no guess -about which language it is; hangul is written by nothing but Korean, -whose surnames are a closed census set, so splitting is safe too. See -:ref:`east-asian-names` in :doc:`usage` for what that looks like and -how to turn either half off. Splitting an unspaced *Han* name is the -one piece that does need to know the language — a Chinese surname list -mangles Japanese kanji names — so that half waits for the ``zh`` pack. +that what selects them is the *script* rather than the word. The +background (covered fully under :ref:`east-asian-names` in +:doc:`usage`): Chinese, Japanese, and Korean names put the family name +first in native script and are usually written with no space between +the parts. Both defaults follow from facts the script alone +establishes. A name written wholly in Han or Hangul is assigned +family-first, because every language written in those scripts orders +names that way — no language guess is involved. An unspaced hangul +name is additionally split into surname and given name, because hangul +is written by nothing but Korean and Korean surnames are a closed +census set that ships as default vocabulary. Splitting an unspaced +*Han* name is the one behavior the script cannot license — the same +characters could be a Chinese or a Japanese name, and a Chinese +surname list splits Japanese names in the wrong place — so it is not a +default: the ``zh`` pack below turns it on when you can declare the +data Chinese. A pack is for something different: a *structural* rule, like reordering a patronymic, that vocabulary alone can't express. diff --git a/docs/usage.rst b/docs/usage.rst index 2d7dafd9..845a0235 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -140,8 +140,7 @@ merging two, and is covered next. East Asian names ----------------- -How these names are built, first, because the parsing rules follow -from it. A Chinese, Japanese, or Korean name written in its own script +A Chinese, Japanese, or Korean name written in its own script puts the family name first: 毛泽东 is MAO Zedong, 山田太郎 is YAMADA Taro, 김민준 is KIM Minjun. The family name is short — one Han character or one hangul syllable, occasionally two — and comes from a From 0724e8d665fc15df8f4f8c8c6416c336477c9fe7 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Tue, 28 Jul 2026 15:49:32 -0700 Subject: [PATCH 19/20] Reword the Korean surname-set clause per review Co-Authored-By: Claude Fable 5 --- docs/usage.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/usage.rst b/docs/usage.rst index 845a0235..7eed0661 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -165,9 +165,9 @@ knowledge of which language the name is in: Splitting an unspaced name is also automatic, but only for Korean. Hangul is written by exactly one language, and Korean family names are -a closed set small enough to ship: the census surname list is part of -the default vocabulary, and the longest listed surname at the start of -an unspaced hangul token becomes the family name: +limited to a small closed set: the census surname list is part of the +default vocabulary, and the longest listed surname at the start of an +unspaced hangul token becomes the family name: .. doctest:: From 5a52aab3e46ac868ccc4628bd1373a5d44fde5a9 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Wed, 29 Jul 2026 00:26:26 -0700 Subject: [PATCH 20/20] State the script_orders precedence rule instead of gesturing at it Co-Authored-By: Claude Fable 5 --- docs/modules.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/modules.rst b/docs/modules.rst index 19f3b07a..8c8e7f60 100644 --- a/docs/modules.rst +++ b/docs/modules.rst @@ -109,9 +109,11 @@ because only these three orders have defined assignment semantics. :value: ((Script.HAN, FAMILY_FIRST), (Script.HANGUL, FAMILY_FIRST)) The default :attr:`~nameparser.Policy.script_orders` table: a name - written wholly in Han or Hangul reads family-first, whatever - ``name_order`` says. The values are drawn from the same three - constants above, and the same restriction applies. Build on it for + written wholly in Han or Hangul reads family-first. A matching + entry in this table takes precedence over ``name_order``, including + a ``name_order`` you set explicitly — ``name_order`` governs only + the names no entry matches. The values are drawn from the same + three constants above, and the same restriction applies. Build on it for additive customization — ``script_orders=dict(DEFAULT_SCRIPT_ORDERS) | {Script.HAN: GIVEN_FIRST}`` — and pass ``script_orders={}`` to opt out entirely