From 75fc1abcf9d20c293c20b815fe1e6d0b36a66bec Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Wed, 29 Jul 2026 20:06:06 -0700 Subject: [PATCH 01/11] Move _SCRIPT_RANGES to _policy, beside Script The codepoint table is data about Script -- which spans each enum member covers -- so it belongs beside the enum in the config layer, not in the pipeline. With one copy readable from below the layering boundary (packs may import _policy, never the pipeline), the hand-copied spans in the zh/ja locale packs can die in the following commits instead of being kept in sync by hand. This deliberately reverses the #271 placement decision that put the table in _pipeline/_vocab.py: that placement is what forced each pack to hand-copy its spans in the first place. _vocab now imports the table and still derives its per-script patterns from it; behavior is unchanged, and only test imports retarget. Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/_vocab.py | 72 ++++---------------------------- nameparser/_policy.py | 73 ++++++++++++++++++++++++++++++++- nameparser/locales/ja.py | 9 ++-- nameparser/locales/zh.py | 8 ++-- tests/v2/pipeline/test_vocab.py | 4 +- tests/v2/test_lexicon.py | 3 +- tests/v2/test_locales.py | 5 ++- tests/v2/test_regex_sync.py | 9 ++-- 8 files changed, 98 insertions(+), 85 deletions(-) diff --git a/nameparser/_pipeline/_vocab.py b/nameparser/_pipeline/_vocab.py index 6638d8d..fad1941 100644 --- a/nameparser/_pipeline/_vocab.py +++ b/nameparser/_pipeline/_vocab.py @@ -13,7 +13,7 @@ from collections.abc import Iterable from nameparser._lexicon import Lexicon, _normalize -from nameparser._policy import Script +from nameparser._policy import Script, _SCRIPT_RANGES # Ported verbatim from v1 (nameparser/config/regexes.py "initial") minus # its empty-string alternative -- WorkToken text is never empty. Kept in @@ -31,74 +31,16 @@ PH = re.compile(r"^ph\.?$", re.IGNORECASE) D = re.compile(r"^d\.?$", re.IGNORECASE) -# Codepoint ranges per Script (#271). This integer table is the single -# source of truth for what a script covers; _SCRIPT_PATTERNS below -# DERIVES the match engine from it. (The sweep here was first written +# The codepoint table lives in _policy beside Script -- one copy +# importable from the pipeline and the locale packs alike; everything +# here DERIVES from it. (single_script's sweep was first written # per-char on the _EMOJI_RANGES precedent in _tokenize.py, on the # theory that a range test needs no regex; measured at token scale the # compiled regex wins by 3-9x, and by 89x on long tokens.) -# HAN: the ideographic iteration mark U+3005, the URO plus Extension -# A, the compatibility block, and the supplementary-plane block -# (Ext B-I + CJK Compat Ideographs Supplement, 0x20000-0x323AF) -- -# rare surnames are the biggest real source of supplementary-plane -# hanzi in personal names (e.g. 𠮷田's 𠮷, U+20BB7), so leaving them -# out silently mis-orders those names; unassigned gaps inside the span -# are harmless, since no real name contains an unassigned codepoint. -# U+3005 々 is the block-vs-Script case, running the OPPOSITE way to -# U+30FB below: 々 already IS Script=Han under UAX #24 (Scripts.txt -# reads `3005 ; Han`), but it sits in CJK Symbols and Punctuation, -# outside every CJK ideograph block this table spans -- so the -# singleton entry is what a BLOCK table needs to reach a character the -# Script property would have classified correctly for free. It earns -# the reach: 々 repeats the preceding kanji and appears only inside -# Han-written names -- 佐々木 (Sasaki, a top-20 Japanese surname), -# 野々村, 奈々. Omitting it made 佐々木 a mixed-script token: the name -# reversed and never gated into segmentation. -# HANGUL: precomposed syllables only -- modern Korean -# text never writes names as bare jamo. -# HIRAGANA/KATAKANA (#272): the two kana blocks, each in full. There -# IS a supplementary-plane kana repertoire (Kana Supplement, Kana -# Extended-A/B, Small Kana Extension, U+1AFF0-U+1B16F, a few hundred -# assigned codepoints -- no exact count here, it moves with the -# Unicode version) but none of it is WORTH chasing the way Han's astral -# block is: those codepoints are hentaigana and other archaic/ -# phonetic-extension forms no modern Japanese name uses, unlike -# supplementary Han, which real surnames genuinely need. The Katakana -# Phonetic Extensions block (U+31F0-U+31FF, 16 small katakana for Ainu -# transcription) is excluded for the same reason -- no modern Japanese -# personal name uses them. Halfwidth kana (U+FF65-U+FF9F, including -# the voiced/semi-voiced sound marks U+FF9E/U+FF9F) is likewise -# deliberately excluded -- legacy bank/CSV data uses it, but it is a -# separate normalization problem; Task 2b's separator handling only -# touches the halfwidth DOT (U+FF65), not the rest of that block. -# This table classifies by Unicode BLOCK, not the UAX #24 Script -# property: U+30A0, U+30FB (the middle dot), and U+30FC (the -# prolonged sound mark) all carry Script=Common under UAX #24, and the -# four kana voicing marks U+3099-U+309C split two and two -- U+3099 -# and U+309A are the COMBINING forms (Script=Inherited), U+309B and -# U+309C the spacing ones (Script=Common) -- yet every one of them is -# needed here, and block membership, not the Script property, is what -# puts them in range. The katakana block's upper end (U+30FF) takes in -# the middle dot U+30FB, kept rather than carved out for a smaller -# reason than it looks: tokenize (#272 Task 2b) turns U+30FB into a -# token separator, so no real parse shows this classifier a string -# containing one. It is kept so that a DIRECT whole-string call -- -# effective_script("マイケル・ジャクソン"), which the unit tests make -- -# still classifies instead of returning None. The ranges below must stay -# mutually disjoint: single_script returns the FIRST covering entry -# (dict iteration order), so an overlapping future script would make -# the result order-dependent instead of well-defined. -_SCRIPT_RANGES: dict[Script, tuple[tuple[int, int], ...]] = { - Script.HAN: ((0x3005, 0x3005), (0x3400, 0x4DBF), (0x4E00, 0x9FFF), - (0xF900, 0xFAFF), (0x20000, 0x323AF)), - Script.HANGUL: ((0xAC00, 0xD7A3),), - Script.HIRAGANA: ((0x3040, 0x309F),), - Script.KATAKANA: ((0x30A0, 0x30FF),), -} - # Derived, never hand-written: one character class per script, in the -# table's own key order (so the FIRST-covering-entry rule above still -# describes what single_script does). +# table's own key order (so single_script's FIRST-covering-entry rule +# still describes what it does; the disjointness requirement that +# makes that well-defined is stated with the table in _policy). _SCRIPT_PATTERNS: dict[Script, re.Pattern[str]] = { script: re.compile( "[" + "".join(f"\\U{lo:08x}-\\U{hi:08x}" for lo, hi in ranges) diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 78a1fb5..c235afd 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -35,8 +35,8 @@ class Script(StrEnum): unspaced-name segmentation (``Policy.segment_scripts``). The rule that admits these (amendment 2026-07-27): script-conditional behavior only where the script itself determines the convention -- - Latin-script input is never affected. Character tables live in - nameparser/_pipeline/_vocab.py, not here.""" + Latin-script input is never affected. The codepoint table backing + these members is internal.""" #: Chinese Hanzi -- and Japanese Kanji: a pure-Han string cannot #: say which language it is, which is fine for ORDER (both write @@ -58,6 +58,75 @@ class Script(StrEnum): KATAKANA = "katakana" +# Codepoint ranges per Script (#271). This integer table is the single +# source of truth for what a script covers; every matcher DERIVES from +# it -- _pipeline/_vocab.py compiles its per-script patterns from it, +# and it is importable from the pipeline and the locale packs alike, +# so the packs' predicates can build on it too (their hand-copies +# date from when this table lived in the pipeline, which packs must +# not import). +# HAN: the ideographic iteration mark U+3005, the URO plus Extension +# A, the compatibility block, and the supplementary-plane block +# (Ext B-I + CJK Compat Ideographs Supplement, 0x20000-0x323AF) -- +# rare surnames are the biggest real source of supplementary-plane +# hanzi in personal names (e.g. 𠮷田's 𠮷, U+20BB7), so leaving them +# out silently mis-orders those names; unassigned gaps inside the span +# are harmless, since no real name contains an unassigned codepoint. +# U+3005 々 is the block-vs-Script case, running the OPPOSITE way to +# U+30FB below: 々 already IS Script=Han under UAX #24 (Scripts.txt +# reads `3005 ; Han`), but it sits in CJK Symbols and Punctuation, +# outside every CJK ideograph block this table spans -- so the +# singleton entry is what a BLOCK table needs to reach a character the +# Script property would have classified correctly for free. It earns +# the reach: 々 repeats the preceding kanji and appears only inside +# Han-written names -- 佐々木 (Sasaki, a top-20 Japanese surname), +# 野々村, 奈々. Omitting it made 佐々木 a mixed-script token: the name +# reversed and never gated into segmentation. +# HANGUL: precomposed syllables only -- modern Korean +# text never writes names as bare jamo. +# HIRAGANA/KATAKANA (#272): the two kana blocks, each in full. There +# IS a supplementary-plane kana repertoire (Kana Supplement, Kana +# Extended-A/B, Small Kana Extension, U+1AFF0-U+1B16F, a few hundred +# assigned codepoints -- no exact count here, it moves with the +# Unicode version) but none of it is WORTH chasing the way Han's astral +# block is: those codepoints are hentaigana and other archaic/ +# phonetic-extension forms no modern Japanese name uses, unlike +# supplementary Han, which real surnames genuinely need. The Katakana +# Phonetic Extensions block (U+31F0-U+31FF, 16 small katakana for Ainu +# transcription) is excluded for the same reason -- no modern Japanese +# personal name uses them. Halfwidth kana (U+FF65-U+FF9F, including +# the voiced/semi-voiced sound marks U+FF9E/U+FF9F) is likewise +# deliberately excluded -- legacy bank/CSV data uses it, but it is a +# separate normalization problem; #272 Task 2b's separator handling +# only touches the halfwidth DOT (U+FF65), not the rest of that block. +# This table classifies by Unicode BLOCK, not the UAX #24 Script +# property: U+30A0, U+30FB (the middle dot), and U+30FC (the +# prolonged sound mark) all carry Script=Common under UAX #24, and the +# four kana voicing marks U+3099-U+309C split two and two -- U+3099 +# and U+309A are the COMBINING forms (Script=Inherited), U+309B and +# U+309C the spacing ones (Script=Common) -- yet every one of them is +# needed here, and block membership, not the Script property, is what +# puts them in range. The katakana block's upper end (U+30FF) takes in +# the middle dot U+30FB, kept rather than carved out for a smaller +# reason than it looks: tokenize (#272 Task 2b) turns U+30FB into a +# token separator, so no real parse shows the classifier a string +# containing one. It is kept so that a DIRECT whole-string call -- +# effective_script (_pipeline/_vocab.py) on "マイケル・ジャクソン", which +# the unit tests (tests/v2/pipeline/test_vocab.py) make -- still +# classifies instead of returning None. The ranges below must +# stay mutually disjoint: single_script (_pipeline/_vocab.py) returns +# the FIRST covering entry (dict iteration order), so an overlapping +# future script would make the result order-dependent instead of +# well-defined. +_SCRIPT_RANGES: dict[Script, tuple[tuple[int, int], ...]] = { + Script.HAN: ((0x3005, 0x3005), (0x3400, 0x4DBF), (0x4E00, 0x9FFF), + (0xF900, 0xFAFF), (0x20000, 0x323AF)), + Script.HANGUL: ((0xAC00, 0xD7A3),), + Script.HIRAGANA: ((0x3040, 0x309F),), + Script.KATAKANA: ((0x30A0, 0x30FF),), +} + + # Order-spec constants (#270). Each reads as its contents because roles # are named given/family, not first/last. diff --git a/nameparser/locales/ja.py b/nameparser/locales/ja.py index a0f9c26..4e645be 100644 --- a/nameparser/locales/ja.py +++ b/nameparser/locales/ja.py @@ -72,10 +72,11 @@ ) # The Japanese repertoire's codepoint spans -- hiragana, katakana and -# Han -- kept in sync BY HAND with _pipeline/_vocab.py's _SCRIPT_RANGES -# entries for those three scripts (layering forbids a pack importing -# the pipeline; the sync test in tests/v2/test_locales.py pins the -# equality). Katakana is in the table even though the pack never +# Han -- kept in sync BY HAND with _policy.py's _SCRIPT_RANGES entries +# for those three scripts (a hold-over from when that table lived in +# the pipeline, which packs must not import; the sync test in +# tests/v2/test_locales.py pins the equality until the copy goes). +# Katakana is in the table even though the pack never # ACTIVATES it: both consumers below quantify over the whole # repertoire, because a katakana-bearing token can still be a # kana-licensed composite the pack acts on (山田エミ) -- DEVIATES must diff --git a/nameparser/locales/zh.py b/nameparser/locales/zh.py index 1ac8cbe..67d0cdf 100644 --- a/nameparser/locales/zh.py +++ b/nameparser/locales/zh.py @@ -97,10 +97,10 @@ policy=PolicyPatch(segment_scripts=frozenset({Script.HAN})), ) -# Han codepoint spans, kept in sync BY HAND with -# _pipeline/_vocab.py's _SCRIPT_RANGES[Script.HAN] (layering forbids -# a pack importing the pipeline; the sync test in -# tests/v2/test_locales.py pins the equality). +# Han codepoint spans, kept in sync BY HAND with _policy.py's +# _SCRIPT_RANGES[Script.HAN] (a hold-over from when that table lived +# in the pipeline, which packs must not import; the sync test in +# tests/v2/test_locales.py pins the equality until the copy goes). _HAN_RANGES = ((0x3005, 0x3005), (0x3400, 0x4DBF), (0x4E00, 0x9FFF), (0xF900, 0xFAFF), (0x20000, 0x323AF)) diff --git a/tests/v2/pipeline/test_vocab.py b/tests/v2/pipeline/test_vocab.py index 1ca664e..52dc96e 100644 --- a/tests/v2/pipeline/test_vocab.py +++ b/tests/v2/pipeline/test_vocab.py @@ -2,10 +2,10 @@ from nameparser._lexicon import Lexicon from nameparser._pipeline._vocab import ( - _SCRIPT_RANGES, effective_script, is_initial, is_suffix_lenient, + effective_script, is_initial, is_suffix_lenient, is_suffix_strict, resolve_script_set, single_script, ) -from nameparser._policy import Script +from nameparser._policy import Script, _SCRIPT_RANGES _LEX = Lexicon( suffix_acronyms=frozenset({"phd", "ma"}), diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index 55fd565..6988cdd 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -9,8 +9,7 @@ from nameparser._lexicon import ( Lexicon, _VOCAB_FIELDS, _default_lexicon, _normalize, _title_key, ) -from nameparser._pipeline._vocab import _SCRIPT_RANGES -from nameparser._policy import Script +from nameparser._policy import Script, _SCRIPT_RANGES def test_entries_are_normalized_at_construction() -> None: diff --git a/tests/v2/test_locales.py b/tests/v2/test_locales.py index 9796eae..4f8e2a9 100644 --- a/tests/v2/test_locales.py +++ b/tests/v2/test_locales.py @@ -12,8 +12,9 @@ from nameparser import Locale, Parser, locales, parse, parser_for from nameparser._lexicon import _VOCAB_FIELDS, Lexicon -from nameparser._pipeline._vocab import _SCRIPT_RANGES -from nameparser._policy import UNSET, PatronymicRule, Policy, Script +from nameparser._policy import ( + UNSET, PatronymicRule, Policy, Script, _SCRIPT_RANGES, +) from nameparser._types import AmbiguityKind from nameparser.locales import ja as _ja from nameparser.locales import ru as _ru diff --git a/tests/v2/test_regex_sync.py b/tests/v2/test_regex_sync.py index 6a9c72f..8f6933d 100644 --- a/tests/v2/test_regex_sync.py +++ b/tests/v2/test_regex_sync.py @@ -23,6 +23,7 @@ from nameparser.config import regexes as _config from nameparser._pipeline import _assign, _post_rules, _tokenize, _vocab +from nameparser import _policy from nameparser._policy import Script from nameparser import _render @@ -165,7 +166,7 @@ def test_comma_char_matches_the_pipeline_comma_set() -> None: def test_differential_cjk_rule_matches_the_script_ranges() -> None: """The CJK rule in tools/differential/expected_changes.toml hand- - copies the script spans from _vocab._SCRIPT_RANGES into a character + copies the script spans from _policy._SCRIPT_RANGES into a character class. A TOML file cannot import the constant, so this is the one copy with no possible alternative -- and the one whose divergence is quietest, because the harness is run by hand rather than in CI. @@ -212,7 +213,7 @@ def test_differential_cjk_rule_matches_the_script_ranges() -> None: f"found {len(matched)}") # A new table entry must force an explicit decision rather than # quietly widening (or failing to widen) the rule above. - assert set(_vocab._SCRIPT_RANGES) == { + assert set(_policy._SCRIPT_RANGES) == { Script.HAN, Script.HANGUL, Script.HIRAGANA, Script.KATAKANA}, ( "a Script joined _SCRIPT_RANGES: decide whether the " f"differential rule in {toml_path.name} should cover it, then " @@ -224,11 +225,11 @@ def test_differential_cjk_rule_matches_the_script_ranges() -> None: # the one span the rule carries that no Script claims (see above) halfwidth_dot = (0xFF65, 0xFF65) assert not any(lo <= halfwidth_dot[0] <= hi - for spans in _vocab._SCRIPT_RANGES.values() + for spans in _policy._SCRIPT_RANGES.values() for lo, hi in spans), ( "U+FF65 is classified now; drop it from the sanctioned extras") expected = {span - for spans in _vocab._SCRIPT_RANGES.values() + for spans in _policy._SCRIPT_RANGES.values() for span in spans if span[1] <= 0xFFFF} | {halfwidth_dot} assert declared == expected, ( From 701466f2c6313f374ce0695045ad7b2634e78bda Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Wed, 29 Jul 2026 20:21:28 -0700 Subject: [PATCH 02/11] Add _script_matcher: closure-compiled script predicates A factory beside _SCRIPT_RANGES that compiles one character class from the union of the named scripts' spans and returns a named closure predicate (an empty union raises ValueError rather than a cryptic pattern error). Two modes: whole=False answers CONTAINS-any -- DEVIATES' contract, where over-declaring is the gate's safe direction; whole=True answers non-empty-and-WHOLLY -- the ja adapter's repertoire guard. The compiled re.Pattern lives inside the closure, and compilation lives in _policy rather than in a pack-local closure, deliberately: tests/v2/test_locales.py classifies any pack module holding a module-level re.Pattern as a marker pack requiring rotator branch coverage, and its registry gate fails a pack that merely IMPORTS re without exposing one -- so packs must not import re at all. Predicates built here keep the zh/ja packs invisible to that sweep by construction. Co-Authored-By: Claude Fable 5 --- nameparser/_policy.py | 39 ++++++++++++++++++++++++++++++++++++++- tests/v2/test_policy.py | 36 +++++++++++++++++++++++++++++++++++- 2 files changed, 73 insertions(+), 2 deletions(-) diff --git a/nameparser/_policy.py b/nameparser/_policy.py index c235afd..eab659c 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -6,7 +6,8 @@ from __future__ import annotations import dataclasses -from collections.abc import Iterable, Mapping +import re +from collections.abc import Callable, Iterable, Mapping from dataclasses import dataclass, field from enum import Enum, StrEnum, auto from typing import Any @@ -127,6 +128,42 @@ class Script(StrEnum): } +def _script_matcher(*scripts: Script, + whole: bool = False) -> Callable[[str], bool]: + """A predicate over strings, compiled once from the union of the + named scripts' spans in _SCRIPT_RANGES. whole=False: True when the + string CONTAINS any such character -- DEVIATES' contract, where + over-declaring is the gate's safe direction. whole=True: True when + the string is non-empty and consists WHOLLY of such characters -- + the ja adapter's repertoire guard. Meant to be called at MODULE + scope: "compiled once" is per matcher, and each call compiles a + fresh pattern. The compiled pattern lives in the closure ON + PURPOSE, and the compilation lives HERE rather than in a pack- + local closure: tests/v2/test_locales.py classifies any pack module + holding a module-level re.Pattern as a marker pack needing rotator + branch coverage, and its registry gate goes further -- a pack that + so much as IMPORTS re without exposing such a pattern fails + "imports re but exposes no module-level pattern" -- so a pack must + not import re at all; predicates built here keep the packs + invisible to that sweep by construction.""" + if not scripts: + raise ValueError("_script_matcher needs at least one Script") + cls = "".join(f"\\U{lo:08x}-\\U{hi:08x}" + for script in scripts + for lo, hi in _SCRIPT_RANGES[script]) + if whole: + pattern = re.compile(f"[{cls}]+") + + def wholly(text: str) -> bool: + return pattern.fullmatch(text) is not None + return wholly + single = re.compile(f"[{cls}]") + + def contains(text: str) -> bool: + return single.search(text) is not None + return contains + + # Order-spec constants (#270). Each reads as its contents because roles # are named given/family, not first/last. diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index 3fc6910..023b3cd 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -6,7 +6,7 @@ from nameparser._policy import ( DEFAULT_SCRIPT_ORDERS, FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST, GIVEN_FIRST, PatronymicRule, Policy, PolicyPatch, Script, UNSET, - apply_patch, + _script_matcher, apply_patch, ) from nameparser._types import Role @@ -66,6 +66,40 @@ def test_script_values_are_the_public_names() -> None: assert Script.HIRAGANA == "hiragana" and Script.KATAKANA == "katakana" +def test_script_matcher_contains_any() -> None: + has_han = _script_matcher(Script.HAN) + assert has_han("毛泽东") + assert has_han("Bruce 李") # mixed input: CONTAINS is enough + assert not has_han("Bruce Lee") + assert not has_han("") + + +def test_script_matcher_whole() -> None: + wholly_ja = _script_matcher( + Script.HAN, Script.HIRAGANA, Script.KATAKANA, whole=True) + assert wholly_ja("高橋みなみ") + assert wholly_ja("マイケル") # katakana is in this union + assert not wholly_ja("高橋 みなみ") # the space is outside it + assert not wholly_ja("") # empty is not WHOLLY anything + # the same union in contains mode accepts what wholly rejects + assert _script_matcher( + Script.HAN, Script.HIRAGANA, Script.KATAKANA)("高橋 みなみ") + + +def test_script_matcher_rejects_zero_scripts() -> None: + # an empty union would compile "[]" and raise a cryptic pattern + # error far from the mistake + with pytest.raises(ValueError, match="at least one Script"): + _script_matcher() + + +def test_script_matcher_reaches_astral_han() -> None: + # the supplementary-plane span must survive the class compilation + # (\U escapes, not \u): 𠮷 is U+20BB7 + assert _script_matcher(Script.HAN)("𠮷田") + assert _script_matcher(Script.HAN, whole=True)("𠮷田") + + def test_patronymic_rules_coerce_and_reject() -> None: p = Policy(patronymic_rules=frozenset({"east-slavic"})) # type: ignore[arg-type] assert p.patronymic_rules == frozenset({PatronymicRule.EAST_SLAVIC}) From d5831fcd26356abd64a186d89d3ac5220bd2758a Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Wed, 29 Jul 2026 20:35:18 -0700 Subject: [PATCH 03/11] zh: derive DEVIATES from the shared table via _script_matcher The hand-copied Han spans and their sync test die because the table is now importable from _policy. DEVIATES keeps its documented def-with-docstring shape (the pack-authoring contract in docs/locales.rst) and delegates to the factory closure; the per-character loop is gone (the compiled character class was measured 16-30x faster at DEVIATES' scan scale). Co-Authored-By: Claude Fable 5 --- nameparser/locales/zh.py | 26 +++++++++++++------------- tests/v2/test_locales.py | 22 +++++++++------------- 2 files changed, 22 insertions(+), 26 deletions(-) diff --git a/nameparser/locales/zh.py b/nameparser/locales/zh.py index 67d0cdf..0bc0b53 100644 --- a/nameparser/locales/zh.py +++ b/nameparser/locales/zh.py @@ -25,15 +25,17 @@ Declared deviations (spec §2 authoring requirement 3): the pack adds vocabulary and one union policy field, both self-selecting by script, so it can only change names containing Han characters -- DEVIATES -below declares exactly that (over-declaring within the script: a Han -name needs an unspaced surname match to actually change, but -per-character scanning is the safe direction). +below declares exactly that (over-declaring within the script: only +segmentation can actually change a parse -- an unspaced surname +match, or a ``Parser(segmenter=...)`` answer where the vocabulary +declines -- but declaring every Han-bearing name is the safe +direction). """ from __future__ import annotations from nameparser._lexicon import Lexicon from nameparser._locale import Locale -from nameparser._policy import PolicyPatch, Script +from nameparser._policy import PolicyPatch, Script, _script_matcher # 2020 census top-100 plus the annotated additions below, simplified # forms, census rank order, 10 per row -- append new entries at the @@ -97,17 +99,15 @@ policy=PolicyPatch(segment_scripts=frozenset({Script.HAN})), ) -# Han codepoint spans, kept in sync BY HAND with _policy.py's -# _SCRIPT_RANGES[Script.HAN] (a hold-over from when that table lived -# in the pipeline, which packs must not import; the sync test in -# tests/v2/test_locales.py pins the equality until the copy goes). -_HAN_RANGES = ((0x3005, 0x3005), (0x3400, 0x4DBF), (0x4E00, 0x9FFF), - (0xF900, 0xFAFF), (0x20000, 0x323AF)) +# Compiled by _policy's factory from the shared codepoint table, so +# the pack carries no Han spans of its own to drift out of sync. +_has_han = _script_matcher(Script.HAN) def DEVIATES(name: str) -> bool: """True when this pack may parse `name` differently from the default parser (the declared-deviation predicate the - non-interference gate consumes).""" - return any(any(lo <= ord(c) <= hi for lo, hi in _HAN_RANGES) - for c in name) + non-interference gate consumes): any Han-bearing name -- see the + module docstring's declared-deviations note for why over-declaring + is safe.""" + return _has_han(name) diff --git a/tests/v2/test_locales.py b/tests/v2/test_locales.py index 4f8e2a9..380ec7c 100644 --- a/tests/v2/test_locales.py +++ b/tests/v2/test_locales.py @@ -124,6 +124,9 @@ def test_zh_pack_contents() -> None: assert "王" in locales.ZH.lexicon.surnames assert "欧阳" in locales.ZH.lexicon.surnames # compound assert "歐陽" in locales.ZH.lexicon.surnames # traditional + assert _zh.DEVIATES("毛泽东") + assert not _zh.DEVIATES("John Smith") + assert not _zh.DEVIATES("김민준") # HAN alone, not a wider union def test_zh_surname_entries_are_wellformed() -> None: @@ -191,14 +194,6 @@ def test_zh_composes_with_korean_defaults() -> None: assert (n.family, n.given) == ("김", "민준") -def test_zh_han_ranges_stay_in_sync_with_vocab() -> None: - # zh.py hand-copies the Han spans from _pipeline/_vocab.py for its - # DEVIATES predicate (layering forbids a pack importing the - # pipeline -- see the module docstring); pin the equality here, the - # codepoint-range twin of the marker-regex sync test above. - assert _zh._HAN_RANGES == _SCRIPT_RANGES[Script.HAN] - - def test_ja_pack_contents() -> None: assert locales.JA.code == "ja" # segmentation activation ONLY: no vocabulary (no list settles a @@ -325,9 +320,9 @@ def test_ja_adapter_gbdt_flag_selects_the_gbdt_divider( def test_ja_ranges_stay_in_sync_with_vocab() -> None: - # the twin of the zh pin above: ja.py hand-copies the three - # Japanese-repertoire scripts' spans for DEVIATES and for the - # adapter's own repertoire check + # ja.py hand-copies the three Japanese-repertoire scripts' spans + # for DEVIATES and for the adapter's own repertoire check; pinned + # here until the pack derives from the shared table assert set(_ja._JA_RANGES) == { span for script in (Script.HAN, Script.HIRAGANA, Script.KATAKANA) for span in _SCRIPT_RANGES[script]} @@ -755,8 +750,9 @@ def _marker_regexes(module: ModuleType) -> list[re.Pattern]: # Packs whose DEVIATES surface is defined by marker REGEXES -- derived, # never listed, same rule as the registry itself: a pack qualifies by # having any module-level re.Pattern, so zh (which declares by Han -# codepoint range, pinned by its own sync test) drops out on its own -# and a future marker-regex pack cannot silently miss branch coverage. +# codepoint range, derived from the shared table via _script_matcher, +# so it holds no pattern of its own) drops out on its own and a future +# marker-regex pack cannot silently miss branch coverage. _MARKER_REGEX_PACKS = tuple(code for code in sorted(_PACKS) if _marker_regexes(_PACKS[code])) From 251b09f41f15a36a964031c8b0b4d255cd44d45d Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Wed, 29 Jul 2026 20:47:04 -0700 Subject: [PATCH 04/11] ja: derive DEVIATES and the repertoire guard from the shared table The three-script hand-copy (_JA_RANGES) and both per-char loops die: the pack now names its scripts once (_JA_SCRIPTS) and lets _policy's _script_matcher compile the two predicates from the shared codepoint table, so there are no spans here to drift out of sync. DEVIATES keeps the documented def shape, delegating to the contains-any matcher; the adapter's guard becomes one whole-token predicate call with identical semantics (empty text declines one check earlier, same None outcome). The sync pin in tests/v2/test_locales.py is replaced by floor/ceiling declaration assertions in test_ja_pack_contents. Co-Authored-By: Claude Fable 5 --- nameparser/_policy.py | 6 ++--- nameparser/locales/ja.py | 55 ++++++++++++++++++++-------------------- tests/v2/test_locales.py | 17 +++++-------- 3 files changed, 37 insertions(+), 41 deletions(-) diff --git a/nameparser/_policy.py b/nameparser/_policy.py index eab659c..6c9a747 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -63,9 +63,9 @@ class Script(StrEnum): # source of truth for what a script covers; every matcher DERIVES from # it -- _pipeline/_vocab.py compiles its per-script patterns from it, # and it is importable from the pipeline and the locale packs alike, -# so the packs' predicates can build on it too (their hand-copies -# date from when this table lived in the pipeline, which packs must -# not import). +# so the packs' predicates build on it too, through _script_matcher +# below (the hand-copies this replaced dated from when the table +# lived in the pipeline, which packs must not import). # HAN: the ideographic iteration mark U+3005, the URO plus Extension # A, the compatibility block, and the supplementary-plane block # (Ext B-I + CJK Compat Ideographs Supplement, 0x20000-0x323AF) -- diff --git a/nameparser/locales/ja.py b/nameparser/locales/ja.py index 4e645be..6169fb0 100644 --- a/nameparser/locales/ja.py +++ b/nameparser/locales/ja.py @@ -50,7 +50,8 @@ names containing characters of the Japanese repertoire -- DEVIATES below declares exactly that, over-declaring within the repertoire (a name also needs an unspaced token and a segmenter to actually change, -but per-character scanning is the safe direction, zh's precedent). +but declaring every repertoire-bearing name is the safe direction, +zh's precedent). """ from __future__ import annotations @@ -58,7 +59,7 @@ from nameparser._lexicon import Lexicon from nameparser._locale import Locale -from nameparser._policy import PolicyPatch, Script +from nameparser._policy import PolicyPatch, Script, _script_matcher from nameparser._types import Segmentation if TYPE_CHECKING: @@ -71,24 +72,24 @@ segment_scripts=frozenset({Script.HAN, Script.HIRAGANA})), ) -# The Japanese repertoire's codepoint spans -- hiragana, katakana and -# Han -- kept in sync BY HAND with _policy.py's _SCRIPT_RANGES entries -# for those three scripts (a hold-over from when that table lived in -# the pipeline, which packs must not import; the sync test in -# tests/v2/test_locales.py pins the equality until the copy goes). -# Katakana is in the table even though the pack never -# ACTIVATES it: both consumers below quantify over the whole -# repertoire, because a katakana-bearing token can still be a -# kana-licensed composite the pack acts on (山田エミ) -- DEVIATES must -# declare it, and the adapter must not decline it. A pure-katakana -# token never reaches THE ADAPTER: KATAKANA is in no activation set, so -# the stage's own gate stops it before the segmenter is consulted. -# DEVIATES still declares one, and must: it is a predicate over any -# name at all, called before any gate runs, and over-declaring is its -# safe direction by design. -_JA_RANGES = ((0x3005, 0x3005), (0x3040, 0x309F), (0x30A0, 0x30FF), - (0x3400, 0x4DBF), (0x4E00, 0x9FFF), (0xF900, 0xFAFF), - (0x20000, 0x323AF)) +#: The scripts DEVIATES and the adapter quantify over. KATAKANA is +#: here even though the pack never ACTIVATES it: a katakana-bearing +#: token can still be a kana-licensed composite the pack acts on +#: (山田エミ) -- DEVIATES must declare it, and the adapter must not +#: decline it. A +#: pure-katakana token never reaches THE ADAPTER: KATAKANA is in no +#: activation set, so the stage's own gate stops it before the +#: segmenter is consulted. DEVIATES still declares one, and must: it +#: is a predicate over any name at all, called before any gate runs, +#: and over-declaring is its safe direction by design. +_JA_SCRIPTS = (Script.HAN, Script.HIRAGANA, Script.KATAKANA) + +# Both compiled by _policy's factory from the shared codepoint table, +# so the pack carries no spans of its own to drift out of sync: the +# contains-any form backs DEVIATES, the whole-token form is the +# adapter's repertoire guard. +_has_japanese = _script_matcher(*_JA_SCRIPTS) +_wholly_japanese = _script_matcher(*_JA_SCRIPTS, whole=True) #: How far outside [0, 1] a namedivider score may stray and still count @@ -97,15 +98,13 @@ _SCORE_EPSILON = 1e-6 -def _in_repertoire(char: str) -> bool: - return any(lo <= ord(char) <= hi for lo, hi in _JA_RANGES) - - def DEVIATES(name: str) -> bool: """True when this pack may parse `name` differently from the default parser (the declared-deviation predicate the - non-interference gate consumes).""" - return any(_in_repertoire(c) for c in name) + non-interference gate consumes): any name bearing a repertoire + character -- see the repertoire note above for why katakana is + declared and why over-declaring is safe.""" + return _has_japanese(name) def ja_segmenter(*, gbdt: bool = False) -> Segmenter: @@ -166,8 +165,8 @@ def segment(text: str) -> Segmentation | None: # receive tokens of any other activated script (hangul, under # the default policy). Decline anything outside the Japanese # repertoire rather than hand it to namedivider, which would - # answer for it regardless -- the same table DEVIATES scans. - if not all(map(_in_repertoire, text)): + # answer for it regardless -- the same union DEVIATES declares. + if not _wholly_japanese(text): return None # namedivider RAISES below two characters ("Name length needs # at least 2 chars"), and a segmenter's exceptions propagate by diff --git a/tests/v2/test_locales.py b/tests/v2/test_locales.py index 380ec7c..149aa4b 100644 --- a/tests/v2/test_locales.py +++ b/tests/v2/test_locales.py @@ -204,6 +204,12 @@ def test_ja_pack_contents() -> None: assert locales.JA.policy.name_order is UNSET assert locales.JA.policy.script_orders is UNSET assert locales.JA.lexicon == Lexicon.empty() + assert _ja.DEVIATES("山田太郎") + # katakana declared (see the pack's repertoire note) + assert _ja.DEVIATES("マイケル") + assert _ja.DEVIATES("みなみ") # hiragana declared + assert not _ja.DEVIATES("John Smith") + assert not _ja.DEVIATES("김민준") # the three JA scripts, not a wider union def test_ja_segmenter_without_extra_raises_helpfully() -> None: @@ -319,15 +325,6 @@ def test_ja_adapter_gbdt_flag_selects_the_gbdt_divider( assert divide == ["gbdt"] -def test_ja_ranges_stay_in_sync_with_vocab() -> None: - # ja.py hand-copies the three Japanese-repertoire scripts' spans - # for DEVIATES and for the adapter's own repertoire check; pinned - # here until the pack derives from the shared table - assert set(_ja._JA_RANGES) == { - span for script in (Script.HAN, Script.HIRAGANA, Script.KATAKANA) - for span in _SCRIPT_RANGES[script]} - - def test_ja_pack_alone_is_inert() -> None: # The pack's own claim, and the reason it is safe to register # without the extra: activating segmentation while shipping nothing @@ -459,7 +456,7 @@ def test_ja_divides_an_astral_kanji_name() -> None: # the supplementary plane -- the reason _SCRIPT_RANGES carries the # astral Han span at all. It has to survive the whole chain as one # character and not two: the script gate, the adapter's repertoire - # scan, namedivider itself, and the offset arithmetic that turns + # guard, namedivider itself, and the offset arithmetic that turns # the answer into spans (Python indexes by codepoint, so an offset # of 2 here spans four UTF-16 units -- a surrogate-pair bug would # land the split in the middle of the character). From 08026d359fdcbbbe42688f1d3787923a7a587474 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Wed, 29 Jul 2026 21:00:41 -0700 Subject: [PATCH 05/11] vocab: derive the script matchers through _script_matcher The two hand-written class-construction expressions were semantically identical to factory calls: _SCRIPT_PATTERNS was per-script wholly-of in the table's key order, _JA_PATTERN wholly-of over the three-script union. Folding them into _script_matcher(whole=True) deletes the last derivation duplicates of the factory's character-class expression and the _JA_PATTERN roster row in test_regex_sync (neither replacement is a module-level re.Pattern, so the completeness sweep no longer sees them). This is a review-driven addition beyond the original spec's "vocab matchers stay unchanged" non-goal, taken because the shared factory made that non-goal's rationale moot. Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/_vocab.py | 45 ++++++++++++++++------------------ nameparser/_policy.py | 7 ++++-- tests/v2/test_regex_sync.py | 3 --- 3 files changed, 26 insertions(+), 29 deletions(-) diff --git a/nameparser/_pipeline/_vocab.py b/nameparser/_pipeline/_vocab.py index fad1941..88b911a 100644 --- a/nameparser/_pipeline/_vocab.py +++ b/nameparser/_pipeline/_vocab.py @@ -10,10 +10,10 @@ import re import unicodedata -from collections.abc import Iterable +from collections.abc import Callable, Iterable from nameparser._lexicon import Lexicon, _normalize -from nameparser._policy import Script, _SCRIPT_RANGES +from nameparser._policy import Script, _SCRIPT_RANGES, _script_matcher # Ported verbatim from v1 (nameparser/config/regexes.py "initial") minus # its empty-string alternative -- WorkToken text is never empty. Kept in @@ -36,16 +36,16 @@ # here DERIVES from it. (single_script's sweep was first written # per-char on the _EMOJI_RANGES precedent in _tokenize.py, on the # theory that a range test needs no regex; measured at token scale the -# compiled regex wins by 3-9x, and by 89x on long tokens.) -# Derived, never hand-written: one character class per script, in the -# table's own key order (so single_script's FIRST-covering-entry rule -# still describes what it does; the disjointness requirement that -# makes that well-defined is stated with the table in _policy). -_SCRIPT_PATTERNS: dict[Script, re.Pattern[str]] = { - script: re.compile( - "[" + "".join(f"\\U{lo:08x}-\\U{hi:08x}" for lo, hi in ranges) - + "]+") - for script, ranges in _SCRIPT_RANGES.items() +# compiled regex wins by 3-9x, and by roughly 70x on long tokens.) +# Derived, never hand-written -- even the class construction goes +# through the shared factory: one wholly-of predicate per script, in +# the table's own key order (a dict comprehension over _SCRIPT_RANGES +# preserves it, so single_script's FIRST-covering-entry rule still +# describes what it does; the disjointness requirement that makes +# that well-defined is stated with the table in _policy). +_SCRIPT_MATCHERS: dict[Script, Callable[[str], bool]] = { + script: _script_matcher(script, whole=True) + for script in _SCRIPT_RANGES } #: The Japanese repertoire: the union effective_script's kana license @@ -53,16 +53,13 @@ #: A frozenset, not the tuple this started as: resolve_script_set #: below is the "later task that needs membership" the tuple's #: original comment anticipated. Membership doesn't care about order, -#: and the pattern built below doesn't either (a regex character +#: and the matcher built below doesn't either (a regex character #: class matches the same set regardless of the order its ranges are #: written in). _JA_SCRIPTS = frozenset({Script.HAN, Script.HIRAGANA, Script.KATAKANA}) -_JA_PATTERN = re.compile( - "[" - + "".join(f"\\U{lo:08x}-\\U{hi:08x}" - for s in _JA_SCRIPTS - for lo, hi in _SCRIPT_RANGES[s]) - + "]+") +# Not _wholly_japanese: that name belongs to the ja pack's own +# predicate over the same union (nameparser/locales/ja.py). +_wholly_ja = _script_matcher(*_JA_SCRIPTS, whole=True) def is_initial(text: str) -> bool: @@ -192,11 +189,11 @@ def _normalized_for_script(text: str) -> str | None: def _classify(normalized: str) -> Script | None: - """The FIRST _SCRIPT_PATTERNS entry covering all of `normalized` + """The FIRST _SCRIPT_MATCHERS entry covering all of `normalized` (already NFC, via _normalized_for_script), else None. Shared by both public classifiers so each of them normalizes exactly once.""" - for script, pattern in _SCRIPT_PATTERNS.items(): - if pattern.fullmatch(normalized): + for script, matcher in _SCRIPT_MATCHERS.items(): + if matcher(normalized): return script return None @@ -224,7 +221,7 @@ def effective_script(text: str) -> Script | None: HIRAGANA carrier entry. Pure-katakana stays KATAKANA (single_script's answer): a lone katakana token is predominantly a transcribed foreign name, so nothing defaults on it.""" - # None for both shapes _JA_PATTERN could never match anyway (empty + # None for both shapes _wholly_ja could never match anyway (empty # text, or all-ASCII text): real work, not a leftover "if text" # guard, since the ASCII case is one a bare emptiness check would # let through. The single normalized copy then serves both the @@ -235,7 +232,7 @@ def effective_script(text: str) -> Script | None: script = _classify(normalized) if script is not None: return script - if _JA_PATTERN.fullmatch(normalized): + if _wholly_ja(normalized): return Script.HIRAGANA return None diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 6c9a747..b0af1ac 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -135,7 +135,8 @@ def _script_matcher(*scripts: Script, string CONTAINS any such character -- DEVIATES' contract, where over-declaring is the gate's safe direction. whole=True: True when the string is non-empty and consists WHOLLY of such characters -- - the ja adapter's repertoire guard. Meant to be called at MODULE + the ja adapter's repertoire guard and _vocab's script + classifiers. Meant to be called at MODULE scope: "compiled once" is per matcher, and each call compiles a fresh pattern. The compiled pattern lives in the closure ON PURPOSE, and the compilation lives HERE rather than in a pack- @@ -145,7 +146,9 @@ def _script_matcher(*scripts: Script, so much as IMPORTS re without exposing such a pattern fails "imports re but exposes no module-level pattern" -- so a pack must not import re at all; predicates built here keep the packs - invisible to that sweep by construction.""" + invisible to that sweep by construction (and keep derived + matchers out of tests/v2/test_regex_sync.py's hand-copy + completeness sweep, which scans for module-level patterns).""" if not scripts: raise ValueError("_script_matcher needs at least one Script") cls = "".join(f"\\U{lo:08x}-\\U{hi:08x}" diff --git a/tests/v2/test_regex_sync.py b/tests/v2/test_regex_sync.py index 8f6933d..a8d6839 100644 --- a/tests/v2/test_regex_sync.py +++ b/tests/v2/test_regex_sync.py @@ -116,9 +116,6 @@ def test_initial_copies_agree_with_each_other_and_config() -> None: ("_tokenize", "_BIDI"): None, # re_bidi, not a REGEXES key # Mirrors _pipeline._state.COMMA_CHARS, not nameparser.config ("_render", "_COMMA_CHAR"): None, - # #272: derived from _SCRIPT_RANGES itself (like _SCRIPT_PATTERNS), - # not hand-copied from anywhere -- no config counterpart to pin. - ("_vocab", "_JA_PATTERN"): None, } _MODULES = {"_assign": _assign, "_post_rules": _post_rules, From b2650585fd351c77f53781aec47b28f7201edae3 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Wed, 29 Jul 2026 21:09:58 -0700 Subject: [PATCH 06/11] Pin zh/ja out of the marker-pack classification _MARKER_REGEX_PACKS classifies any pack holding a module-level re.Pattern as a marker pack whose regex branches the rotator sweep must cover. Since zh and ja now derive their DEVIATES predicates through _script_matcher, that boundary is load-bearing: the factory's closure is what hides the compiled class and keeps range-declaring packs out of a branch-coverage sweep a codepoint-range predicate cannot satisfy. Nothing pinned it -- a pack author who "simplifies" a closure back to a module-level compiled pattern would silently flip zh or ja into the sweep. The new test converts that failure from silent to loud, and the comment above the derivation now names both packs (it predated ja's switch and named only zh). Co-Authored-By: Claude Fable 5 --- tests/v2/test_locales.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/tests/v2/test_locales.py b/tests/v2/test_locales.py index 149aa4b..c57e28f 100644 --- a/tests/v2/test_locales.py +++ b/tests/v2/test_locales.py @@ -746,14 +746,25 @@ def _marker_regexes(module: ModuleType) -> list[re.Pattern]: # Packs whose DEVIATES surface is defined by marker REGEXES -- derived, # never listed, same rule as the registry itself: a pack qualifies by -# having any module-level re.Pattern, so zh (which declares by Han +# having any module-level re.Pattern, so zh and ja (which declare by # codepoint range, derived from the shared table via _script_matcher, -# so it holds no pattern of its own) drops out on its own and a future -# marker-regex pack cannot silently miss branch coverage. +# so they hold no pattern of their own) drop out on their own and a +# future marker-regex pack cannot silently miss branch coverage. _MARKER_REGEX_PACKS = tuple(code for code in sorted(_PACKS) if _marker_regexes(_PACKS[code])) +def test_range_declaring_packs_stay_out_of_marker_classification() -> None: + # Load-bearing since the packs derive from the shared table: zh/ja + # hold no module-level re.Pattern because _script_matcher's closure + # hides the compiled class. A pack author who "simplifies" one back + # to a module-level pattern flips its classification into the + # rotator branch-coverage sweep, which a codepoint-range predicate + # cannot satisfy. + assert "zh" not in _MARKER_REGEX_PACKS + assert "ja" not in _MARKER_REGEX_PACKS + + def test_registry_is_the_pack_contract() -> None: # spec §2 authoring requirement 3, enforced structurally (design # note 2026-07-18, option C): every registered pack module must From c78addeb1e84f07cadfaeca3b02573aecf68adc6 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Wed, 29 Jul 2026 21:16:59 -0700 Subject: [PATCH 07/11] =?UTF-8?q?Classify=20=E3=80=86=20(U+3006)=20as=20Ha?= =?UTF-8?q?n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The one behavior change of this branch: the HAN entry's opening span widens from the 々 singleton (0x3005, 0x3005) to (0x3005, 0x3006), taking in 〆, the shime mark. Unlike 々 -- Script=Han under UAX #24, merely outside every CJK ideograph block -- 〆 carries Script=Common, so this is the table deliberately reaching PAST the Script property, not around a block boundary. It earns the reach: 〆 functions solely as a component of Japanese surnames (〆木 Shimeki, 〆谷 Shimetani, 〆野) and appears in no other script's names. Before the change 〆木 was a mixed-script token: unlicensed, positional, given-first -- the name reversed and never gated into segmentation. The widening also makes 〆木太郎 wholly Japanese to the ja pack's adapter, which would have handed it to namedivider -- whose 0.4.x rule path cuts every attested 〆 surname at offset 1 (〆 | 木太郎, score 1.0, algorithm='rule'; measured): a wrong family arriving with NO SEGMENTATION report, since 1.0 clears the floor. The adapter now declines 〆-bearing tokens before the divider is consulted (pinned stub-based in tests/v2/test_locales.py, no extra needed): the family-first ORDER fix stands regardless, and division can return when a divider knows the mark. The differential toml's character class is a hand copy of the table, and its sync guard tripped in between exactly as designed: test_differential_cjk_rule_matches_the_script_ranges failed with "expected_changes.toml's CJK name_regex declares [(12293, 12293), ...]; _SCRIPT_RANGES' BMP spans are [(12293, 12294), ...]" until the CJK rule's 々-々 was widened to 々-〆 to match. Differential harness re-run by hand: 654 corpus names, 18 intentional diffs, 0 unexplained, exit 0 -- neither corpus contains a 〆 name, so totals are unchanged. Co-Authored-By: Claude Fable 5 --- docs/release_log.rst | 1 + nameparser/_policy.py | 27 ++++++++++++++++-------- nameparser/locales/ja.py | 15 +++++++++++-- tests/v2/cases.py | 17 ++++++++++++++- tests/v2/pipeline/test_vocab.py | 2 +- tests/v2/test_locales.py | 20 ++++++++++++++++++ tools/differential/expected_changes.toml | 7 +++--- 7 files changed, 73 insertions(+), 16 deletions(-) diff --git a/docs/release_log.rst b/docs/release_log.rst index 0cae824..31446fa 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -22,6 +22,7 @@ Release Log - Fix names written wholly in Han or Hangul parsing given-first: native-script CJK now reads family-first by default, through the new ``Policy.script_orders`` table, so ``"毛 泽东"`` gives family ``毛`` where 1.x gave family ``泽东``. A name that is a single unspaced token moves the same way — ``"毛泽东"`` and ``"山田太郎"`` now land in ``family``/``last`` where 1.x put them in ``given``/``first``, with the string itself untouched, which makes it the easiest form of this change to miss — a lone token renders the same whichever field holds it, whereas the spaced form does show up in the output (``str(HumanName("毛 泽东"))`` is now ``"泽东 毛"``). No language detection is involved: Chinese and Japanese both write the family name first in native script, so the script settles the order without anyone having to know which language it is. Latin-script and mixed-script names are never affected, and an explicit comma still wins. **Default-on: changes parse output for wholly-CJK names**, through ``HumanName`` as well as the 2.0 API (closes #271) - Fix unspaced Korean names not splitting: the census surname list now ships as default vocabulary (``Lexicon.surnames``) with hangul segmentation on by default (``Policy.segment_scripts``), so ``"김민준"`` parses family ``김``, given ``민준`` where 1.x returned the whole string as ``first``. Rendering follows the split, so ``str(HumanName("김민준"))`` is now ``"민준 김"`` where 1.x echoed the input back unchanged. Nothing but Korean is written in hangul and its surnames are a closed census set, which is what makes the split safe as a default rather than a pack. **Default-on**, and it reaches ``HumanName`` too. ``Policy(segment_scripts=())`` turns the split off; ``Policy(script_orders={})`` separately restores the positional reading; clearing both restores 2.0 behavior exactly (#271) - Fix Japanese names carrying kana parsing given-first: the family-first rule above extends to any name whose characters stay within kanji and kana while carrying at least one kana character, so ``"高橋 みなみ"`` gives family ``高橋`` and ``"山田 エミ"`` family ``山田`` where 1.x read both the other way round, and a lone such token (``"高橋みなみ"``, ``"みなみ"``) lands in ``family``/``last`` where 1.x put it in ``given``/``first``. The reasoning is the one hangul already uses: hiragana never transcribes a foreign name, and a transcription is kana ALONE, so kanji-plus-kana is a Japanese person's name written in Japanese order. A name written **wholly in katakana** is deliberately excluded and stays positional — it is predominantly a transcribed foreign name (``"マイケル ジャクソン"``) already in given-first order. **Default-on: changes parse output for kana-bearing Japanese names**, through ``HumanName`` as well as the 2.0 API, and the spaced forms change what the name renders as (``str(HumanName("高橋 みなみ"))`` is now ``"みなみ 高橋"``). ``Policy(script_orders={})`` clears this entry along with the Han and Hangul ones (#272) + - Fix names containing 〆 (U+3006, the shime mark that opens Japanese surnames like 〆木 and 〆谷) parsing given-first: the script classifier now counts 〆 as Han, extending the 々 entry the table already carries, and going a step further than it — 々 is Script=Han and merely outside the ideograph blocks, while 〆 is Script=Common — so these names take the East Asian family-first reading like any other wholly-Han name. ``Policy(script_orders={})`` restores the positional reading for these names exactly as for other wholly-Han names - Fix the katakana middle dot ``・`` (U+30FB, and its halfwidth twin U+FF65) being read as part of a name rather than as the divider it is: it now separates tokens exactly as a space does. A transcribed foreign name therefore divides into its parts and, being wholly katakana, keeps its source order — ``"マイケル・ジャクソン"`` gives given ``マイケル``, family ``ジャクソン``, where 1.x left the whole string in ``first`` — while a kanji pair written the same way takes the family-first rule (``"高橋・一郎"`` → family ``高橋``). Native Japanese names never contain this character and no other script's names use it, so the separation is unconditional, which also means it is **not** covered by the two policy opt-outs. Rendering follows, the way the Korean split's does: the dot comes back as a space, so ``str(HumanName("マイケル・ジャクソン"))`` is now ``"マイケル ジャクソン"``, and that reaches delimited content too — the nickname in ``"山田 太郎 (マイケル・ジャクソン)"`` renders ``"マイケル ジャクソン"``. The Chinese interpunct ``·`` is deliberately NOT included: U+00B7 is also the Catalan punt volat and appears inside legitimate names (``Gal·la``) (#272) - Fix NFD-decomposed input missing the East Asian defaults entirely: script classification now normalizes to NFC before deciding, so a Korean or Japanese name typed on macOS — where decomposed text is routine — gets the same order rule as its composed twin, which it silently did not before. Segmentation MATCHING deliberately stays raw, so an unspaced NFD hangul name is ordered correctly but not split, rather than being split in the wrong place. One gotcha worth stating plainly: parse output preserves the encoding it was given, so for NFD input ``name.family == "김"`` is ``False`` even though it is the same name — compare NFC-normalized text when comparing across encodings (#272) diff --git a/nameparser/_policy.py b/nameparser/_policy.py index b0af1ac..084c7a9 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -66,7 +66,8 @@ class Script(StrEnum): # so the packs' predicates build on it too, through _script_matcher # below (the hand-copies this replaced dated from when the table # lived in the pipeline, which packs must not import). -# HAN: the ideographic iteration mark U+3005, the URO plus Extension +# HAN: the ideographic iteration mark U+3005 and the shime mark +# U+3006, the URO plus Extension # A, the compatibility block, and the supplementary-plane block # (Ext B-I + CJK Compat Ideographs Supplement, 0x20000-0x323AF) -- # rare surnames are the biggest real source of supplementary-plane @@ -76,13 +77,21 @@ class Script(StrEnum): # U+3005 々 is the block-vs-Script case, running the OPPOSITE way to # U+30FB below: 々 already IS Script=Han under UAX #24 (Scripts.txt # reads `3005 ; Han`), but it sits in CJK Symbols and Punctuation, -# outside every CJK ideograph block this table spans -- so the -# singleton entry is what a BLOCK table needs to reach a character the -# Script property would have classified correctly for free. It earns -# the reach: 々 repeats the preceding kanji and appears only inside -# Han-written names -- 佐々木 (Sasaki, a top-20 Japanese surname), -# 野々村, 奈々. Omitting it made 佐々木 a mixed-script token: the name -# reversed and never gated into segmentation. +# outside every CJK ideograph block this table spans -- so a +# singleton entry was what a BLOCK table needed to reach a character +# the Script property would have classified correctly for free. It +# earns the reach: 々 repeats the preceding kanji and appears only +# inside Han-written names -- 佐々木 (Sasaki, a top-20 Japanese +# surname), 野々村, 奈々. Omitting it made 佐々木 a mixed-script token: +# the name reversed and never gated into segmentation. +# U+3006 〆 (the shime mark) extends that singleton to a two-codepoint +# span on a DIFFERENT justification: unlike 々, 〆 is Script=Common +# under UAX #24, so this is the table deliberately reaching PAST the +# Script property, not around a block boundary -- justified because 〆 +# functions solely as a component of Japanese surnames (〆木 Shimeki, +# 〆谷 Shimetani, 〆野) and appears in no other script's names. +# Leaving it out made 〆木 a mixed-script token: the name reversed and +# never gated into segmentation. # HANGUL: precomposed syllables only -- modern Korean # text never writes names as bare jamo. # HIRAGANA/KATAKANA (#272): the two kana blocks, each in full. There @@ -120,7 +129,7 @@ class Script(StrEnum): # future script would make the result order-dependent instead of # well-defined. _SCRIPT_RANGES: dict[Script, tuple[tuple[int, int], ...]] = { - Script.HAN: ((0x3005, 0x3005), (0x3400, 0x4DBF), (0x4E00, 0x9FFF), + Script.HAN: ((0x3005, 0x3006), (0x3400, 0x4DBF), (0x4E00, 0x9FFF), (0xF900, 0xFAFF), (0x20000, 0x323AF)), Script.HANGUL: ((0xAC00, 0xD7A3),), Script.HIRAGANA: ((0x3040, 0x309F),), diff --git a/nameparser/locales/ja.py b/nameparser/locales/ja.py index 6169fb0..ad667d9 100644 --- a/nameparser/locales/ja.py +++ b/nameparser/locales/ja.py @@ -127,8 +127,10 @@ def ja_segmenter(*, gbdt: bool = False) -> Segmenter: matters for offline, sandboxed or air-gapped deployments. The adapter declines -- returns None, leaving the token whole -- - for text outside the Japanese repertoire, for text too short to - divide, for any answer that fails to reconstruct its input, and for + for text outside the Japanese repertoire, for text bearing the + shime mark 〆 (namedivider 0.4.x's rule path cuts it in the wrong + place), for text too short to divide, for any answer that fails to + reconstruct its input, and for any score outside [0, 1] by more than float noise (a divider scoring 87.0 is broken, and its division is worth no more than its score). An @@ -168,6 +170,15 @@ def segment(text: str) -> Segmentation | None: # answer for it regardless -- the same union DEVIATES declares. if not _wholly_japanese(text): return None + # namedivider 0.4.x has no knowledge of the shime mark: its + # rule path cuts 〆木太郎 to 〆 | 木太郎 at confidence 1.0 + # (measured -- wrong for every attested 〆 surname: 〆木, 〆谷, + # 〆野), so a wrong family would arrive with no SEGMENTATION + # report. Decline instead: the family-first ORDER fix for 〆 + # stands regardless, and division can return when a divider + # knows the mark. + if "〆" in text: + return None # namedivider RAISES below two characters ("Name length needs # at least 2 chars"), and a segmenter's exceptions propagate by # contract, so a one-character token -- routine in spaced input diff --git a/tests/v2/cases.py b/tests/v2/cases.py index db0d77a..cc3d689 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -619,10 +619,25 @@ def __post_init__(self) -> None: notes="々 (U+3005, the ideographic iteration mark) repeats " "the preceding kanji and is Script=Han under UAX #24, " "but sits outside every CJK ideograph BLOCK -- and the " - "classifier is a block table, so it needs a singleton " + "classifier is a block table, so it needs its own " "entry to count as Han; without one 佐々木 -- a top-20 " "Japanese surname -- would be a mixed-script token and " "reverse"), + Case("ja_shime_mark_is_han", "〆木 太郎", + {"family": "〆木", "given": "太郎"}, + classification="fix(shime-mark)", + notes="〆 (U+3006, the shime mark) opens real Japanese " + "surnames -- 〆木 Shimeki, 〆谷 Shimetani, 〆野 -- but " + "carries Script=Common under UAX #24; the table counts " + "it as Han anyway (a deliberate step PAST the Script " + "property, unlike 々's), else 〆木 is a mixed-script " + "token and the name reverses"), + Case("ja_shime_lone_token_takes_family", "〆木太郎", + {"family": "〆木太郎"}, + classification="fix(shime-mark)", + notes="wholly-Han lone token under the family-first entry: " + "the whole token lands in family, unsegmented by " + "default"), Case("ja_nakaguro_inside_a_nickname", "山田 太郎 (マイケル・ジャクソン)", {"family": "山田", "given": "太郎", diff --git a/tests/v2/pipeline/test_vocab.py b/tests/v2/pipeline/test_vocab.py index 52dc96e..b680eb8 100644 --- a/tests/v2/pipeline/test_vocab.py +++ b/tests/v2/pipeline/test_vocab.py @@ -183,7 +183,7 @@ def test_script_ranges_are_pairwise_disjoint() -> None: # every entry of every script rather than script by script, # because the adjacent blocks are exactly where a future edit # would go wrong -- HIRAGANA ends at U+309F and KATAKANA opens at - # U+30A0 with no gap at all, and HAN's singleton U+3005 sits just + # U+30A0 with no gap at all, and HAN's U+3005-U+3006 span sits just # below both. spans = [(lo, hi, script) for script, ranges in _SCRIPT_RANGES.items() diff --git a/tests/v2/test_locales.py b/tests/v2/test_locales.py index c57e28f..b54d772 100644 --- a/tests/v2/test_locales.py +++ b/tests/v2/test_locales.py @@ -269,6 +269,26 @@ def test_ja_adapter_guard_stack_against_a_stub( assert seg("林") is None # namedivider raises below 2 +def test_ja_adapter_declines_shime_tokens( + monkeypatch: pytest.MonkeyPatch) -> None: + # Since U+3006 classifies as Han, 〆木太郎 is wholly Japanese and + # reaches the adapter -- but namedivider 0.4.x has no knowledge of + # the shime mark: measured, its rule path cuts every attested 〆 + # surname at offset 1 (〆木太郎 -> 〆 | 木太郎, score 1.0, + # algorithm='rule'), a wrong family arriving with NO SEGMENTATION + # report because 1.0 clears the floor. The adapter declines these + # tokens before the divider is ever consulted. + calls: list[str] = [] + + def divide(text: str) -> _FakeDivided: + calls.append(text) + return _FakeDivided(text[:1], text[1:], 1.0) + + _fake_namedivider(monkeypatch, divide) + assert locales.ja_segmenter()("〆木太郎") is None + assert calls == [] # the decline sits before divide_name + + def test_ja_adapter_defensive_branches_against_a_stub( monkeypatch: pytest.MonkeyPatch) -> None: # reconstruction failure -> decline; an empty side -> the stated diff --git a/tools/differential/expected_changes.toml b/tools/differential/expected_changes.toml index 0711762..8a72918 100644 --- a/tools/differential/expected_changes.toml +++ b/tools/differential/expected_changes.toml @@ -50,9 +50,10 @@ issue = "fix(#271/#272) native-script CJK: family-first order, hangul segmentati # tracked as #272's and #271's shared gap in issue #295, which also # records the provenance decision a fix needs; until it is closed the # behavioral guarantee lives in the fix(#271)/fix(#272) rows of -# tests/v2/cases.py, and this rule is kept ready for the moment a CJK -# name lands in a corpus. -name_regex = "[\\u3005-\\u3005\\u3040-\\u309F\\u30A0-\\u30FF\\u3400-\\u4DBF\\u4E00-\\u9FFF\\uF900-\\uFAFF\\uAC00-\\uD7A3\\uFF65-\\uFF65]" +# tests/v2/cases.py and the fix(shime-mark) rows, whose 〆 span rides +# in on the same HAN entry, and this rule is kept ready for the moment +# a CJK name lands in a corpus. +name_regex = "[\\u3005-\\u3006\\u3040-\\u309F\\u30A0-\\u30FF\\u3400-\\u4DBF\\u4E00-\\u9FFF\\uF900-\\uFAFF\\uAC00-\\uD7A3\\uFF65-\\uFF65]" fields = ["first", "middle", "last"] [[change]] From 995bb409b25964723b65c0f3f3c382c57424ac30 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Wed, 29 Jul 2026 23:12:56 -0700 Subject: [PATCH 08/11] Sweep docs for the relocated script table Closing sweep for the _SCRIPT_RANGES move to _policy.py. Two stale spots, both corrected: - nameparser/_policy.py: _script_matcher's docstring claimed the closure keeps derived matchers out of test_regex_sync.py's hand-copy completeness sweep as if that sweep scanned everywhere; it scans a fixed set of pipeline modules (_assign, _post_rules, _render, _tokenize, _vocab), which covers _vocab but never _policy or the packs. Reworded to claim only what holds: the closure spares _vocab's derived matchers a declaration row in that sweep. - docs/locales.rst: the contributing contract told authors to declare DEVIATES but never said how a range-declaring pack builds one, while test_locales.py now reads any module-level re.Pattern as a marker regex needing branch coverage and fails a pack importing re without one. Added the mechanism: build the predicate with _policy's _script_matcher, the way zh and ja do, never by compiling private character ranges. Verified rather than edited: AGENTS.md's test_regex_sync.py line ("wherever the copy lives -- including copies outside the package") stays true because the differential toml copy in tools/ remains and is still pinned; its layering line already lists _policy among allowed pack imports. Acceptance greps: 0x4E00 appears in the package only in _policy.py's table; the dead symbols (_HAN_RANGES, _JA_RANGES, _in_repertoire, _SCRIPT_PATTERNS, _JA_PATTERN) are gone from nameparser/, tests/, and tools/; no pack references _vocab. Co-Authored-By: Claude Fable 5 --- docs/locales.rst | 9 ++++++++- nameparser/_policy.py | 7 ++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/locales.rst b/docs/locales.rst index 300a267..e57a847 100644 --- a/docs/locales.rst +++ b/docs/locales.rst @@ -280,7 +280,14 @@ by ``tests/v2/test_locales.py``: #. Declare a module-level ``DEVIATES(name)`` predicate: given a name string, return whether *this pack alone* might parse it differently from the default parser. Over-declaring is safe; under-declaring is - not — when in doubt, ``DEVIATES`` should say yes. + not — when in doubt, ``DEVIATES`` should say yes. A pack whose + scope is a *script* builds the predicate with ``_script_matcher`` + from ``nameparser/_policy.py``, the way ``zh`` and ``ja`` do — + never by compiling its own character ranges: the table in + ``_policy.py`` is the single source for script codepoint spans, and + the contract test reads any module-level ``re.Pattern`` in a pack + as a marker regex needing branch coverage (a pack that imports + ``re`` without exposing one fails outright). #. Add a rotator list to ``tests/v2/test_locales.py``. Every pack needs one, but what it has to contain follows from how the pack declares its scope. A pack declaring by *marker regex* (``ru``, ``tr_az``) diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 084c7a9..428e693 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -155,9 +155,10 @@ def _script_matcher(*scripts: Script, so much as IMPORTS re without exposing such a pattern fails "imports re but exposes no module-level pattern" -- so a pack must not import re at all; predicates built here keep the packs - invisible to that sweep by construction (and keep derived - matchers out of tests/v2/test_regex_sync.py's hand-copy - completeness sweep, which scans for module-level patterns).""" + invisible to that sweep by construction (and spare _vocab's + derived matchers a declaration row in tests/v2/test_regex_sync.py's + completeness sweep, which scans the pipeline modules for + module-level patterns).""" if not scripts: raise ValueError("_script_matcher needs at least one Script") cls = "".join(f"\\U{lo:08x}-\\U{hi:08x}" From fbb32ab023ba756df6d376c975aca3350934365c Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Wed, 29 Jul 2026 23:25:29 -0700 Subject: [PATCH 09/11] Reference PR #303 from the shime-mark release-log bullet Co-Authored-By: Claude Fable 5 --- docs/release_log.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/release_log.rst b/docs/release_log.rst index 31446fa..50a39ad 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -22,7 +22,7 @@ Release Log - Fix names written wholly in Han or Hangul parsing given-first: native-script CJK now reads family-first by default, through the new ``Policy.script_orders`` table, so ``"毛 泽东"`` gives family ``毛`` where 1.x gave family ``泽东``. A name that is a single unspaced token moves the same way — ``"毛泽东"`` and ``"山田太郎"`` now land in ``family``/``last`` where 1.x put them in ``given``/``first``, with the string itself untouched, which makes it the easiest form of this change to miss — a lone token renders the same whichever field holds it, whereas the spaced form does show up in the output (``str(HumanName("毛 泽东"))`` is now ``"泽东 毛"``). No language detection is involved: Chinese and Japanese both write the family name first in native script, so the script settles the order without anyone having to know which language it is. Latin-script and mixed-script names are never affected, and an explicit comma still wins. **Default-on: changes parse output for wholly-CJK names**, through ``HumanName`` as well as the 2.0 API (closes #271) - Fix unspaced Korean names not splitting: the census surname list now ships as default vocabulary (``Lexicon.surnames``) with hangul segmentation on by default (``Policy.segment_scripts``), so ``"김민준"`` parses family ``김``, given ``민준`` where 1.x returned the whole string as ``first``. Rendering follows the split, so ``str(HumanName("김민준"))`` is now ``"민준 김"`` where 1.x echoed the input back unchanged. Nothing but Korean is written in hangul and its surnames are a closed census set, which is what makes the split safe as a default rather than a pack. **Default-on**, and it reaches ``HumanName`` too. ``Policy(segment_scripts=())`` turns the split off; ``Policy(script_orders={})`` separately restores the positional reading; clearing both restores 2.0 behavior exactly (#271) - Fix Japanese names carrying kana parsing given-first: the family-first rule above extends to any name whose characters stay within kanji and kana while carrying at least one kana character, so ``"高橋 みなみ"`` gives family ``高橋`` and ``"山田 エミ"`` family ``山田`` where 1.x read both the other way round, and a lone such token (``"高橋みなみ"``, ``"みなみ"``) lands in ``family``/``last`` where 1.x put it in ``given``/``first``. The reasoning is the one hangul already uses: hiragana never transcribes a foreign name, and a transcription is kana ALONE, so kanji-plus-kana is a Japanese person's name written in Japanese order. A name written **wholly in katakana** is deliberately excluded and stays positional — it is predominantly a transcribed foreign name (``"マイケル ジャクソン"``) already in given-first order. **Default-on: changes parse output for kana-bearing Japanese names**, through ``HumanName`` as well as the 2.0 API, and the spaced forms change what the name renders as (``str(HumanName("高橋 みなみ"))`` is now ``"みなみ 高橋"``). ``Policy(script_orders={})`` clears this entry along with the Han and Hangul ones (#272) - - Fix names containing 〆 (U+3006, the shime mark that opens Japanese surnames like 〆木 and 〆谷) parsing given-first: the script classifier now counts 〆 as Han, extending the 々 entry the table already carries, and going a step further than it — 々 is Script=Han and merely outside the ideograph blocks, while 〆 is Script=Common — so these names take the East Asian family-first reading like any other wholly-Han name. ``Policy(script_orders={})`` restores the positional reading for these names exactly as for other wholly-Han names + - Fix names containing 〆 (U+3006, the shime mark that opens Japanese surnames like 〆木 and 〆谷) parsing given-first: the script classifier now counts 〆 as Han, extending the 々 entry the table already carries, and going a step further than it — 々 is Script=Han and merely outside the ideograph blocks, while 〆 is Script=Common — so these names take the East Asian family-first reading like any other wholly-Han name. ``Policy(script_orders={})`` restores the positional reading for these names exactly as for other wholly-Han names (#303) - Fix the katakana middle dot ``・`` (U+30FB, and its halfwidth twin U+FF65) being read as part of a name rather than as the divider it is: it now separates tokens exactly as a space does. A transcribed foreign name therefore divides into its parts and, being wholly katakana, keeps its source order — ``"マイケル・ジャクソン"`` gives given ``マイケル``, family ``ジャクソン``, where 1.x left the whole string in ``first`` — while a kanji pair written the same way takes the family-first rule (``"高橋・一郎"`` → family ``高橋``). Native Japanese names never contain this character and no other script's names use it, so the separation is unconditional, which also means it is **not** covered by the two policy opt-outs. Rendering follows, the way the Korean split's does: the dot comes back as a space, so ``str(HumanName("マイケル・ジャクソン"))`` is now ``"マイケル ジャクソン"``, and that reaches delimited content too — the nickname in ``"山田 太郎 (マイケル・ジャクソン)"`` renders ``"マイケル ジャクソン"``. The Chinese interpunct ``·`` is deliberately NOT included: U+00B7 is also the Catalan punt volat and appears inside legitimate names (``Gal·la``) (#272) - Fix NFD-decomposed input missing the East Asian defaults entirely: script classification now normalizes to NFC before deciding, so a Korean or Japanese name typed on macOS — where decomposed text is routine — gets the same order rule as its composed twin, which it silently did not before. Segmentation MATCHING deliberately stays raw, so an unspaced NFD hangul name is ordered correctly but not split, rather than being split in the wrong place. One gotcha worth stating plainly: parse output preserves the encoding it was given, so for NFD input ``name.family == "김"`` is ``False`` even though it is the same name — compare NFC-normalized text when comparing across encodings (#272) From dbd1675eacdfe833dbd50eb5ce6dfa68e2b1df81 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Wed, 29 Jul 2026 23:45:51 -0700 Subject: [PATCH 10/11] Tighten comments and pin review-found gaps (PR #303 review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Comment corrections: - _policy.py: scope the 〆 justification to personal names -- the mark is a general-purpose Japanese symbol (〆切, the envelope closing mark); only its within-names use is surname-only - _policy.py: "a pack must not import re" -> "a range-declaring pack" (marker packs like ru/tr_az import re legitimately, by exposing module-level patterns) - _policy.py: the regex-sync sweep scans the pipeline modules PLUS _render, and only private (underscore-prefixed) module-level patterns -- say so - _policy.py: replace the hand-copy PR-history parenthetical with the durable constraint (the table lives here because packs must not import the pipeline) - locales/ja.py: reflow two ragged comment wraps (wording unchanged) - locales/ja.py: acknowledge the blanket 〆 decline's deliberate breadth -- mid-token 〆 (山〆太郎) measures 0.375 on the kanji_feature path and WOULD have carried a SEGMENTATION report; swept in anyway because decline is the safe direction New pins: - test_locales.py guard-stack stub test: assert seg("Yamada太郎") is None -- kills a mutation dropping whole=True from _wholly_japanese, which survived the whole suite - test_locales.py pack-contents tests: _zh/_ja.DEVIATES("𠮷田") -- pins the astral extent of the pack predicates, which a BMP-only hand-rolled replacement would otherwise survive - cases.py ja_shime_with_kana_given ("〆木 ひろ") -- the kana license composing with the widened Han span, previously untested together - test_policy.py test_every_script_member_has_a_range_entry -- a future Script member without a _SCRIPT_RANGES entry is accepted by Policy validation yet never classified; this turns that silent no-op into a failure - test_locales.py test_namedivider_still_miscuts_shime_names -- the self-expiring canary behind the adapter's 〆 decline; when namedivider learns the mark this fails and the decline gets revisited instead of lasting forever Co-Authored-By: Claude Fable 5 --- nameparser/_policy.py | 25 ++++++++++++++----------- nameparser/locales/ja.py | 34 ++++++++++++++++++---------------- tests/v2/cases.py | 5 +++++ tests/v2/test_locales.py | 16 ++++++++++++++++ tests/v2/test_policy.py | 9 ++++++++- 5 files changed, 61 insertions(+), 28 deletions(-) diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 428e693..1828637 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -64,8 +64,8 @@ class Script(StrEnum): # it -- _pipeline/_vocab.py compiles its per-script patterns from it, # and it is importable from the pipeline and the locale packs alike, # so the packs' predicates build on it too, through _script_matcher -# below (the hand-copies this replaced dated from when the table -# lived in the pipeline, which packs must not import). +# below (the table lives here rather than in the pipeline because +# packs must not import the pipeline). # HAN: the ideographic iteration mark U+3005 and the shime mark # U+3006, the URO plus Extension # A, the compatibility block, and the supplementary-plane block @@ -87,9 +87,11 @@ class Script(StrEnum): # U+3006 〆 (the shime mark) extends that singleton to a two-codepoint # span on a DIFFERENT justification: unlike 々, 〆 is Script=Common # under UAX #24, so this is the table deliberately reaching PAST the -# Script property, not around a block boundary -- justified because 〆 -# functions solely as a component of Japanese surnames (〆木 Shimeki, -# 〆谷 Shimetani, 〆野) and appears in no other script's names. +# Script property, not around a block boundary -- justified because +# within personal names 〆 appears solely in Japanese surnames (〆木 +# Shimeki, 〆谷 Shimetani, 〆野) -- its other uses (the envelope +# closing mark, 〆切) never reach a name parser -- and it appears in +# no other script's names. # Leaving it out made 〆木 a mixed-script token: the name reversed and # never gated into segmentation. # HANGUL: precomposed syllables only -- modern Korean @@ -153,12 +155,13 @@ def _script_matcher(*scripts: Script, holding a module-level re.Pattern as a marker pack needing rotator branch coverage, and its registry gate goes further -- a pack that so much as IMPORTS re without exposing such a pattern fails - "imports re but exposes no module-level pattern" -- so a pack must - not import re at all; predicates built here keep the packs - invisible to that sweep by construction (and spare _vocab's - derived matchers a declaration row in tests/v2/test_regex_sync.py's - completeness sweep, which scans the pipeline modules for - module-level patterns).""" + "imports re but exposes no module-level pattern" -- so a + range-declaring pack must not import re at all; predicates built + here keep the packs invisible to that sweep by construction (and + spare _vocab's derived matchers a declaration row in + tests/v2/test_regex_sync.py's completeness sweep, which scans the + pipeline modules (plus _render) for private module-level + patterns).""" if not scripts: raise ValueError("_script_matcher needs at least one Script") cls = "".join(f"\\U{lo:08x}-\\U{hi:08x}" diff --git a/nameparser/locales/ja.py b/nameparser/locales/ja.py index ad667d9..4113e9e 100644 --- a/nameparser/locales/ja.py +++ b/nameparser/locales/ja.py @@ -76,12 +76,11 @@ #: here even though the pack never ACTIVATES it: a katakana-bearing #: token can still be a kana-licensed composite the pack acts on #: (山田エミ) -- DEVIATES must declare it, and the adapter must not -#: decline it. A -#: pure-katakana token never reaches THE ADAPTER: KATAKANA is in no -#: activation set, so the stage's own gate stops it before the -#: segmenter is consulted. DEVIATES still declares one, and must: it -#: is a predicate over any name at all, called before any gate runs, -#: and over-declaring is its safe direction by design. +#: decline it. A pure-katakana token never reaches THE ADAPTER: +#: KATAKANA is in no activation set, so the stage's own gate stops it +#: before the segmenter is consulted. DEVIATES still declares one, +#: and must: it is a predicate over any name at all, called before +#: any gate runs, and over-declaring is its safe direction by design. _JA_SCRIPTS = (Script.HAN, Script.HIRAGANA, Script.KATAKANA) # Both compiled by _policy's factory from the shared codepoint table, @@ -129,15 +128,13 @@ def ja_segmenter(*, gbdt: bool = False) -> Segmenter: The adapter declines -- returns None, leaving the token whole -- for text outside the Japanese repertoire, for text bearing the shime mark 〆 (namedivider 0.4.x's rule path cuts it in the wrong - place), for text too short to divide, for any answer that fails to - reconstruct its input, and for - any score outside [0, 1] by more than float noise (a divider - scoring 87.0 is broken, and its division is worth no more than its - score). An - answer that puts the whole token on one side comes back as - "confidently one token" rather than a decline: the divider read it - and found nothing to cut, which is not the same as having no - opinion. + place), for text too short to divide, for any answer that fails + to reconstruct its input, and for any score outside [0, 1] by + more than float noise (a divider scoring 87.0 is broken, and its + division is worth no more than its score). An answer that puts + the whole token on one side comes back as "confidently one token" + rather than a decline: the divider read it and found nothing to + cut, which is not the same as having no opinion. Answers arrive with namedivider's own confidence, which the parsing stage compares against its floor to decide whether to report a SEGMENTATION ambiguity: namedivider scores a rule-based division @@ -176,7 +173,12 @@ def segment(text: str) -> Segmentation | None: # 〆野), so a wrong family would arrive with no SEGMENTATION # report. Decline instead: the family-first ORDER fix for 〆 # stands regardless, and division can return when a divider - # knows the mark. + # knows the mark. The blanket check is deliberately broader + # than that rule-path failure: mid-token 〆 (山〆太郎) comes + # back 山 | 〆太郎 at 0.375 on the kanji_feature path + # (measured), a low-confidence answer that WOULD have carried + # a SEGMENTATION report -- swept in anyway, because decline is + # the safe direction and mid-token 〆 is vanishingly rare. if "〆" in text: return None # namedivider RAISES below two characters ("Name length needs diff --git a/tests/v2/cases.py b/tests/v2/cases.py index cc3d689..62f7ced 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -638,6 +638,11 @@ def __post_init__(self) -> None: notes="wholly-Han lone token under the family-first entry: " "the whole token lands in family, unsegmented by " "default"), + Case("ja_shime_with_kana_given", "〆木 ひろ", + {"family": "〆木", "given": "ひろ"}, + classification="fix(shime-mark)", + notes="the kana license composes with the widened Han span: " + "〆木 reads as kanji beside a kana given name"), Case("ja_nakaguro_inside_a_nickname", "山田 太郎 (マイケル・ジャクソン)", {"family": "山田", "given": "太郎", diff --git a/tests/v2/test_locales.py b/tests/v2/test_locales.py index b54d772..695be01 100644 --- a/tests/v2/test_locales.py +++ b/tests/v2/test_locales.py @@ -125,6 +125,7 @@ def test_zh_pack_contents() -> None: assert "欧阳" in locales.ZH.lexicon.surnames # compound assert "歐陽" in locales.ZH.lexicon.surnames # traditional assert _zh.DEVIATES("毛泽东") + assert _zh.DEVIATES("𠮷田") # astral Han declared assert not _zh.DEVIATES("John Smith") assert not _zh.DEVIATES("김민준") # HAN alone, not a wider union @@ -205,6 +206,7 @@ def test_ja_pack_contents() -> None: assert locales.JA.policy.script_orders is UNSET assert locales.JA.lexicon == Lexicon.empty() assert _ja.DEVIATES("山田太郎") + assert _ja.DEVIATES("𠮷田") # astral Han declared # katakana declared (see the pack's repertoire note) assert _ja.DEVIATES("マイケル") assert _ja.DEVIATES("みなみ") # hiragana declared @@ -266,6 +268,8 @@ def test_ja_adapter_guard_stack_against_a_stub( assert answer is not None assert answer.splits == (2,) and answer.confidence == 0.5 assert seg("김민준") is None # outside the repertoire + # contains JA chars but is not WHOLLY JA + assert seg("Yamada太郎") is None assert seg("林") is None # namedivider raises below 2 @@ -485,6 +489,18 @@ def test_ja_divides_an_astral_kanji_name() -> None: assert [a.kind for a in n.ambiguities] == [AmbiguityKind.SEGMENTATION] +@_needs_ja +def test_namedivider_still_miscuts_shime_names() -> None: + # The canary behind the adapter's 〆 decline: namedivider 0.4.x's + # rule path cuts every attested 〆 surname at offset 1 with + # confidence 1.0. When this fails, namedivider learned the mark -- + # revisit the decline in ja.py rather than deleting this test. + import namedivider + result = namedivider.BasicNameDivider().divide_name("〆木太郎") + assert (result.family, result.given) == ("〆", "木太郎") + assert float(result.score) == 1.0 + + @_needs_ja def test_ja_stacked_on_zh_pays_the_zh_packs_documented_cost() -> None: # Stacking composes MECHANICALLY (the test above) but not for free, diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index 023b3cd..b37aef6 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -6,7 +6,7 @@ from nameparser._policy import ( DEFAULT_SCRIPT_ORDERS, FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST, GIVEN_FIRST, PatronymicRule, Policy, PolicyPatch, Script, UNSET, - _script_matcher, apply_patch, + _SCRIPT_RANGES, _script_matcher, apply_patch, ) from nameparser._types import Role @@ -100,6 +100,13 @@ def test_script_matcher_reaches_astral_han() -> None: assert _script_matcher(Script.HAN, whole=True)("𠮷田") +def test_every_script_member_has_a_range_entry() -> None: + # a member added to the enum alone would be accepted by + # Policy(segment_scripts=...) yet never classified by + # single_script -- a silent no-op until this fails + assert set(_SCRIPT_RANGES) == set(Script) + + def test_patronymic_rules_coerce_and_reject() -> None: p = Policy(patronymic_rules=frozenset({"east-slavic"})) # type: ignore[arg-type] assert p.patronymic_rules == frozenset({PatronymicRule.EAST_SLAVIC}) From 65b6093c067ded02ff78eb86a59c83c4977a0306 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Wed, 29 Jul 2026 23:57:18 -0700 Subject: [PATCH 11/11] Simplify: hoist the JA union, merge factory closures, trim duplicated prose MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Hoist _JA_SCRIPTS into _policy.py beside _SCRIPT_RANGES: the union was defined twice (_pipeline/_vocab.py frozenset, locales/ja.py tuple) with nothing pinning their membership equality -- the same drift class this branch already deleted at span level. One tuple now serves the kana license, the ja pack's DEVIATES, and the adapter's repertoire guard; the katakana rationale moves onto the ja predicates it explains, and _vocab's naming-defense comment (guarding a constraint that never existed) goes away. * Merge _script_matcher's twin closures: one compiled [cls]+ pattern with the bound method chosen by `whole` -- search over [cls]+ is exactly contains-any, so the second pattern and closure bought nothing. * Reorder ja adapter guards: the O(1) length check now leads the chain, so routine one-char tokens from spaced input ("林 太郎") no longer pay the regex fullmatch and the 〆 scan (measured 4x) before being declined. All guards return the same None, so behavior is unchanged. * Trim four prose duplications: the 〆 paragraph's copy of the 々 closer in _policy's table comment (plus an orphan line wrap), the triple-stated ordering point on _SCRIPT_MATCHERS, locales.rst's restatement of the marker-classification contract the factory docstring already carries, and the twice-narrated namedivider mis-cut measurement in test_locales.py (the authoritative account stays at ja.py's decline comment). Co-Authored-By: Claude Fable 5 --- docs/locales.rst | 7 ++---- nameparser/_pipeline/_vocab.py | 24 ++++++------------- nameparser/_policy.py | 31 +++++++++++++------------ nameparser/locales/ja.py | 42 ++++++++++++++++------------------ tests/v2/test_locales.py | 16 +++++-------- 5 files changed, 51 insertions(+), 69 deletions(-) diff --git a/docs/locales.rst b/docs/locales.rst index e57a847..3ad31fd 100644 --- a/docs/locales.rst +++ b/docs/locales.rst @@ -283,11 +283,8 @@ by ``tests/v2/test_locales.py``: not — when in doubt, ``DEVIATES`` should say yes. A pack whose scope is a *script* builds the predicate with ``_script_matcher`` from ``nameparser/_policy.py``, the way ``zh`` and ``ja`` do — - never by compiling its own character ranges: the table in - ``_policy.py`` is the single source for script codepoint spans, and - the contract test reads any module-level ``re.Pattern`` in a pack - as a marker regex needing branch coverage (a pack that imports - ``re`` without exposing one fails outright). + never by compiling its own character ranges: the factory's + docstring explains how the pack-contract test enforces this. #. Add a rotator list to ``tests/v2/test_locales.py``. Every pack needs one, but what it has to contain follows from how the pack declares its scope. A pack declaring by *marker regex* (``ru``, ``tr_az``) diff --git a/nameparser/_pipeline/_vocab.py b/nameparser/_pipeline/_vocab.py index 88b911a..d0e9b31 100644 --- a/nameparser/_pipeline/_vocab.py +++ b/nameparser/_pipeline/_vocab.py @@ -13,7 +13,8 @@ from collections.abc import Callable, Iterable from nameparser._lexicon import Lexicon, _normalize -from nameparser._policy import Script, _SCRIPT_RANGES, _script_matcher +from nameparser._policy import (Script, _JA_SCRIPTS, _SCRIPT_RANGES, + _script_matcher) # Ported verbatim from v1 (nameparser/config/regexes.py "initial") minus # its empty-string alternative -- WorkToken text is never empty. Kept in @@ -39,26 +40,15 @@ # compiled regex wins by 3-9x, and by roughly 70x on long tokens.) # Derived, never hand-written -- even the class construction goes # through the shared factory: one wholly-of predicate per script, in -# the table's own key order (a dict comprehension over _SCRIPT_RANGES -# preserves it, so single_script's FIRST-covering-entry rule still -# describes what it does; the disjointness requirement that makes -# that well-defined is stated with the table in _policy). +# the table's key order -- the FIRST-covering-entry rule _classify +# documents. _SCRIPT_MATCHERS: dict[Script, Callable[[str], bool]] = { script: _script_matcher(script, whole=True) for script in _SCRIPT_RANGES } -#: The Japanese repertoire: the union effective_script's kana license -#: quantifies over -- HAN, HIRAGANA, KATAKANA (HANGUL simply omitted). -#: A frozenset, not the tuple this started as: resolve_script_set -#: below is the "later task that needs membership" the tuple's -#: original comment anticipated. Membership doesn't care about order, -#: and the matcher built below doesn't either (a regex character -#: class matches the same set regardless of the order its ranges are -#: written in). -_JA_SCRIPTS = frozenset({Script.HAN, Script.HIRAGANA, Script.KATAKANA}) -# Not _wholly_japanese: that name belongs to the ja pack's own -# predicate over the same union (nameparser/locales/ja.py). +# The whole-token matcher over _policy's _JA_SCRIPTS union, backing +# effective_script's kana license. _wholly_ja = _script_matcher(*_JA_SCRIPTS, whole=True) @@ -261,6 +251,6 @@ def resolve_script_set(scripts: Iterable[Script]) -> Script | None: found = frozenset(scripts) if len(found) <= 1: return next(iter(found), None) - if found <= _JA_SCRIPTS: + if found.issubset(_JA_SCRIPTS): return Script.HIRAGANA return None diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 1828637..e8b92bd 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -67,8 +67,8 @@ class Script(StrEnum): # below (the table lives here rather than in the pipeline because # packs must not import the pipeline). # HAN: the ideographic iteration mark U+3005 and the shime mark -# U+3006, the URO plus Extension -# A, the compatibility block, and the supplementary-plane block +# U+3006, the URO plus Extension A, the compatibility block, and the +# supplementary-plane block # (Ext B-I + CJK Compat Ideographs Supplement, 0x20000-0x323AF) -- # rare surnames are the biggest real source of supplementary-plane # hanzi in personal names (e.g. 𠮷田's 𠮷, U+20BB7), so leaving them @@ -92,8 +92,6 @@ class Script(StrEnum): # Shimeki, 〆谷 Shimetani, 〆野) -- its other uses (the envelope # closing mark, 〆切) never reach a name parser -- and it appears in # no other script's names. -# Leaving it out made 〆木 a mixed-script token: the name reversed and -# never gated into segmentation. # HANGUL: precomposed syllables only -- modern Korean # text never writes names as bare jamo. # HIRAGANA/KATAKANA (#272): the two kana blocks, each in full. There @@ -138,6 +136,12 @@ class Script(StrEnum): Script.KATAKANA: ((0x30A0, 0x30FF),), } +#: The Japanese repertoire: the three scripts Japanese names draw on. +#: The kana license (_pipeline/_vocab.py's effective_script), the ja +#: pack's DEVIATES, and the segmenter adapter's repertoire guard all +#: quantify over this one union (HANGUL simply omitted). +_JA_SCRIPTS = (Script.HAN, Script.HIRAGANA, Script.KATAKANA) + def _script_matcher(*scripts: Script, whole: bool = False) -> Callable[[str], bool]: @@ -167,17 +171,14 @@ def _script_matcher(*scripts: Script, cls = "".join(f"\\U{lo:08x}-\\U{hi:08x}" for script in scripts for lo, hi in _SCRIPT_RANGES[script]) - if whole: - pattern = re.compile(f"[{cls}]+") - - def wholly(text: str) -> bool: - return pattern.fullmatch(text) is not None - return wholly - single = re.compile(f"[{cls}]") - - def contains(text: str) -> bool: - return single.search(text) is not None - return contains + # one pattern serves both modes: fullmatch of [cls]+ is wholly-of, + # and search over [cls]+ is exactly contains-any + pattern = re.compile(f"[{cls}]+") + match = pattern.fullmatch if whole else pattern.search + + def matcher(text: str) -> bool: + return match(text) is not None + return matcher # Order-spec constants (#270). Each reads as its contents because roles diff --git a/nameparser/locales/ja.py b/nameparser/locales/ja.py index 4113e9e..d0c3ca9 100644 --- a/nameparser/locales/ja.py +++ b/nameparser/locales/ja.py @@ -59,7 +59,8 @@ from nameparser._lexicon import Lexicon from nameparser._locale import Locale -from nameparser._policy import PolicyPatch, Script, _script_matcher +from nameparser._policy import (PolicyPatch, Script, _JA_SCRIPTS, + _script_matcher) from nameparser._types import Segmentation if TYPE_CHECKING: @@ -72,21 +73,18 @@ segment_scripts=frozenset({Script.HAN, Script.HIRAGANA})), ) -#: The scripts DEVIATES and the adapter quantify over. KATAKANA is -#: here even though the pack never ACTIVATES it: a katakana-bearing -#: token can still be a kana-licensed composite the pack acts on -#: (山田エミ) -- DEVIATES must declare it, and the adapter must not -#: decline it. A pure-katakana token never reaches THE ADAPTER: -#: KATAKANA is in no activation set, so the stage's own gate stops it -#: before the segmenter is consulted. DEVIATES still declares one, -#: and must: it is a predicate over any name at all, called before -#: any gate runs, and over-declaring is its safe direction by design. -_JA_SCRIPTS = (Script.HAN, Script.HIRAGANA, Script.KATAKANA) - -# Both compiled by _policy's factory from the shared codepoint table, -# so the pack carries no spans of its own to drift out of sync: the -# contains-any form backs DEVIATES, the whole-token form is the -# adapter's repertoire guard. +# Both compiled by _policy's factory from the shared codepoint table +# and its _JA_SCRIPTS union, so the pack carries no spans of its own +# to drift out of sync: the contains-any form backs DEVIATES, the +# whole-token form is the adapter's repertoire guard. Both match +# KATAKANA even though the pack never ACTIVATES it: a katakana-bearing +# token can still be a kana-licensed composite the pack acts on +# (山田エミ) -- DEVIATES must declare it, and the adapter must not +# decline it. A pure-katakana token never reaches THE ADAPTER: +# KATAKANA is in no activation set, so the stage's own gate stops it +# before the segmenter is consulted. DEVIATES still declares one, +# and must: it is a predicate over any name at all, called before +# any gate runs, and over-declaring is its safe direction by design. _has_japanese = _script_matcher(*_JA_SCRIPTS) _wholly_japanese = _script_matcher(*_JA_SCRIPTS, whole=True) @@ -160,6 +158,12 @@ def ja_segmenter(*, gbdt: bool = False) -> Segmenter: else namedivider.BasicNameDivider()) def segment(text: str) -> Segmentation | None: + # namedivider RAISES below two characters ("Name length needs + # at least 2 chars"), and a segmenter's exceptions propagate by + # contract, so a one-character token -- routine in spaced input + # like "林 太郎" -- must be declined here, not passed on. + if len(text) < 2: + return None # segment_scripts UNIONS under pack composition, so this can # receive tokens of any other activated script (hangul, under # the default policy). Decline anything outside the Japanese @@ -181,12 +185,6 @@ def segment(text: str) -> Segmentation | None: # the safe direction and mid-token 〆 is vanishingly rare. if "〆" in text: return None - # namedivider RAISES below two characters ("Name length needs - # at least 2 chars"), and a segmenter's exceptions propagate by - # contract, so a one-character token -- routine in spaced input - # like "林 太郎" -- must be declined here, not passed on. - if len(text) < 2: - return None result = divider.divide_name(text) if result.family + result.given != text: return None # defensive: never trust reconstruction diff --git a/tests/v2/test_locales.py b/tests/v2/test_locales.py index 695be01..98cf5aa 100644 --- a/tests/v2/test_locales.py +++ b/tests/v2/test_locales.py @@ -276,12 +276,9 @@ def test_ja_adapter_guard_stack_against_a_stub( def test_ja_adapter_declines_shime_tokens( monkeypatch: pytest.MonkeyPatch) -> None: # Since U+3006 classifies as Han, 〆木太郎 is wholly Japanese and - # reaches the adapter -- but namedivider 0.4.x has no knowledge of - # the shime mark: measured, its rule path cuts every attested 〆 - # surname at offset 1 (〆木太郎 -> 〆 | 木太郎, score 1.0, - # algorithm='rule'), a wrong family arriving with NO SEGMENTATION - # report because 1.0 clears the floor. The adapter declines these - # tokens before the divider is ever consulted. + # reaches the adapter, but the divider is never consulted for 〆 + # tokens -- see the decline comment in ja.py for the measured + # mis-cut it prevents. calls: list[str] = [] def divide(text: str) -> _FakeDivided: @@ -491,10 +488,9 @@ def test_ja_divides_an_astral_kanji_name() -> None: @_needs_ja def test_namedivider_still_miscuts_shime_names() -> None: - # The canary behind the adapter's 〆 decline: namedivider 0.4.x's - # rule path cuts every attested 〆 surname at offset 1 with - # confidence 1.0. When this fails, namedivider learned the mark -- - # revisit the decline in ja.py rather than deleting this test. + # The canary behind the adapter's 〆 decline: when this fails, + # namedivider learned the mark -- revisit the decline in ja.py + # rather than deleting this test. import namedivider result = namedivider.BasicNameDivider().divide_name("〆木太郎") assert (result.family, result.given) == ("〆", "木太郎")