Vocabulary build: type-design follow-ups and silent-pass refusals from #737's review

AGENTS.md

@@ -271,7 +271,7 @@ Add a dedicated `copy.deepcopy()` round-trip test for it too (see `test_regexes_
 
 **Before adding a short/common word to `PARTICLES` globally**, test it mid-string against realistic 3-token names, not just check for English-word collisions: Korean/Vietnamese given names put a short syllable in the middle slot (`Park In Hwan`, `Nguyen To Nga`), and Western names put a bare initial there (`John V. Smith`). A word that looks safe ("nobody is named 'to'") can still swallow a real middle name/initial into the last name once it's a global prefix — confirmed regressions for `to`/`in`/`an`/`ten`/`then` and bare `v` this way (PR #191).
 
-**Adding a curated sub-set of an existing vocabulary set** (a marker over a base set, as `GIVEN_NAME_TITLES` is over `TITLES`) — it is a behavioral PROPERTY of a reading, not a second word list: a row in `tools/vocab/build.py`'s `BEHAVIORAL` table, a branch in `compile_pool` adding the words whose reading carries it, a constant in its `CONSTANTS` roster, and the `NAME = _compiled.NAME` assignment in the config module. The subset relation then holds by construction, since the property cannot exist without its reading, so it needs no check. A relation construction does not give (`∩ == ∅` with a set the sub-set cannot logically overlap; a sub-set member that is also a title is silently inert, title handling consuming it first) goes in `check_pool`, with a mutation in `tests/v2/test_vocab.py`'s table. Do **not** forbid `titles ∩ particles` — that overlap is intentional (`freiherr`, `st`). On the v1 side the sub-set is a `_SetManagerAttribute` on `Constants` (like `first_name_titles` — the v1 ATTRIBUTE names did not move in #293, only the module constants did), **not** a `_CachedUnionMember` — only `prefixes`/`suffix_acronyms`/`suffix_not_acronyms`/`titles` feed the `_pst` hot-path cache, so a sub-set costs nothing at runtime and stays out of `is_rootname`. A runtime `.add()` to the sub-set's `SetManager` does **not** propagate to the parent's (same as `first_name_titles`→`titles`), so a caller adding a brand-new word adds it to both.
+**Adding a curated sub-set of an existing vocabulary set** (a marker over a base set, as `GIVEN_NAME_TITLES` is over `TITLES`) — it is a behavioral PROPERTY of a reading, not a second word list: a row in `tools/vocab/build.py`'s `BEHAVIORAL` table, a branch in `compile_pool` adding the words whose reading carries it, a constant in its `CONSTANTS` roster and a key of the same name in its `Compiled` TypedDict (an import-time assert holds the two to one list), and the `NAME = _compiled.NAME` assignment in the config module. The subset relation then holds by construction, since the property cannot exist without its reading, so it needs no check. A relation construction does not give (`∩ == ∅` with a set the sub-set cannot logically overlap; a sub-set member that is also a title is silently inert, title handling consuming it first) goes in `check_pool`, with a mutation in `tests/v2/test_vocab.py`'s table. Do **not** forbid `titles ∩ particles` — that overlap is intentional (`freiherr`, `st`). On the v1 side the sub-set is a `_SetManagerAttribute` on `Constants` (like `first_name_titles` — the v1 ATTRIBUTE names did not move in #293, only the module constants did), **not** a `_CachedUnionMember` — only `prefixes`/`suffix_acronyms`/`suffix_not_acronyms`/`titles` feed the `_pst` hot-path cache, so a sub-set costs nothing at runtime and stays out of `is_rootname`. A runtime `.add()` to the sub-set's `SetManager` does **not** propagate to the parent's (same as `first_name_titles`→`titles`), so a caller adding a brand-new word adds it to both.
 
 **Adding a flag-gated post-parse transform** (reorder/adjust) — add a `Constants` boolean (default `False`), implement a `handle_*()` method, and call it in `post_process()` after `handle_firstnames()` and before `handle_capitalization()`, gated on the flag. Default-off keeps existing parses byte-for-byte unchanged. Two shipped examples: `patronymic_name_order` gates both `handle_east_slavic_patronymic_name_order()` (#85) and `handle_turkic_patronymic_name_order()` (#185) — one flag driving two independent handlers, added in the same `post_process()` slot; `middle_name_as_last` gates `handle_middle_name_as_last()` (#133), which folds `middle_list` into `last_list`.
 

tests/v2/test_vocab.py

@@ -149,6 +149,7 @@ def _w(body: str) -> str:
     ("reserved-top-level-key", {"und.toml": "default = false\n" + OK}, "reserved for phase 3"),
     ("file-name-not-a-tag", {"english.toml": OK}, "BCP 47 tag"),
     ("file-name-not-canonical-case", {"pt-br.toml": OK}, "BCP 47 tag"),
+    ("file-with-no-entries", {"en.toml": OK, "und.toml": "schema = 1\n"}, "no \\[\\[word\\]\\] entries"),
     ("reserved-reading-property", {"und.toml": _w('[[word]]\ntext = "mba"\ncredential = { bare_only = true }')}, "reserved for phase 2"),
     ("unknown-reading-property", {"und.toml": _w('[[word]]\ntext = "mba"\ncredential = { acronymm = true }')}, "unknown key 'acronymm'"),
     ("unknown-word-key", {"und.toml": _w('[[word]]\ntext = "jr"\ngeneration = {}\nlang = "en"')}, "unknown key 'lang'"),
@@ -157,6 +158,17 @@ def _w(body: str) -> str:
     ("text-not-normalized", {"en.toml": _w('[[word]]\ntext = "Jnr."\ngeneration = {}')}, "normalized form"),
     ("text-double-spaced", {"und.toml": _w('[[word]]\ntext = "z  domu"\nbirth_marker = {}')}, "normalized form"),
     ("text-missing", {"und.toml": _w("[[word]]\ngeneration = {}")}, "non-empty `text`"),
+    # a text no token can equal: the tokenizer divides at a comma and a
+    # nakaguro, strips a bidi mark, and keeps an invisible character the
+    # file does not show
+    ("text-with-a-comma", {"en.toml": _w('[[word]]\ntext = "jr,"\ngeneration = {}')}, "divides a name at"),
+    ("text-with-a-nakaguro", {"ja.toml": _w('[[word]]\ntext = "さん・"\nhonorific = {}')}, "divides a name at"),
+    ("text-with-a-bidi-mark", {"he.toml": _w('[[word]]\ntext = "dr\\u200f"\ntitle = {}')}, "invisible character U\\+200F"),
+    ("text-with-a-zero-width-space", {"en.toml": _w('[[word]]\ntext = "dr\\u200b"\ntitle = {}')}, "invisible character U\\+200B"),
+    ("text-with-a-control-character", {"en.toml": _w('[[word]]\ntext = "dr\\u0007"\ntitle = {}')}, "invisible character U\\+0007"),
+    ("text-with-an-emoji", {"en.toml": _w('[[word]]\ntext = "dr★"\ntitle = {}')}, "strips by default"),
+    ("text-with-an-east-asian-interpunct", {"zh.toml": _w('[[word]]\ntext = "先·生"\nhonorific = {}')}, "rules/T3"),
+    ("phrase-word-with-an-edge-period", {"de.toml": _w('[[word]]\ntext = "dr. med"\ntitle = { addresses = "given" }')}, "write 'dr med'"),
     ("text-twice-in-a-file", {"en.toml": _w('[[word]]\ntext = "jr"\ngeneration = {}\n\n[[word]]\ntext = "jr"\nstatus = {}')}, "listed twice"),
     # 3: kinds, properties and values
     ("unregistered-kind", {"hr.toml": _w('[[word]]\ntext = "mlađi"\ngeneraton = {}')}, "unknown key 'generaton'"),
@@ -188,6 +200,10 @@ def _w(body: str) -> str:
     ("flag-not-a-bool", {"und.toml": _w('[[word]]\ntext = "van"\nparticle = { may_be_given = "yes" }')}, "true or false"),
     ("issue-not-a-number", {"en.toml": _w('[[word]]\ntext = "jr"\ngeneration = { issue = "661" }')}, "issue number"),
     ("decided-not-a-date", {"en.toml": _w('[[word]]\ntext = "jr"\ngeneration = { decided = "2026-10-18" }')}, "TOML date"),
+    # tomllib reads a date-time as a datetime, which isinstance calls a date
+    ("decided-a-date-time", {"en.toml": _w('[[word]]\ntext = "jr"\ngeneration = { decided = 2026-10-18T09:00:00 }')}, "not a string or a date-time"),
+    ("as-of-a-date-time", {"fr.toml": _w('[[word]]\ntext = "fils"\nfamily = { us_census_2010 = 767, as_of = 2026-10-16T00:00:00Z }\nexcluded = { generation = "x" }')}, "as_of must be a TOML date"),
+    ("as-of-without-a-count", {"fr.toml": _w('[[word]]\ntext = "fils"\nfamily = { as_of = 2026-10-16 }\nexcluded = { generation = "x" }')}, "as_of dates a count"),
     ("gender-out-of-range", {"en.toml": _w('[[word]]\ntext = "jr"\ngeneration = { gender = "male" }')}, "gender must be"),
     ("count-without-as-of", {"fr.toml": _w('[[word]]\ntext = "fils"\nfamily = { us_census_2010 = 767 }\nexcluded = { generation = "x" }')}, "needs `as_of`"),
     ("count-source-typo", {"fr.toml": _w('[[word]]\ntext = "fils"\nfamily = { us_census = 767, as_of = 2026-10-16 }\nexcluded = { generation = "x" }')}, "unknown key 'us_census'"),
@@ -212,15 +228,19 @@ def _w(body: str) -> str:
     ("excluded-kind-sidestepped-by-a-sibling-kind", {"fr.toml": _w('[[word]]\ntext = "fils"\nexcluded = { generation = "Marie Fils" }'), "en.toml": _w('[[word]]\ntext = "fils"\nstatus = {}')}, "Marie Fils"),
     ("excluded-credential-sidestepped-by-an-acronym", {"und.toml": _w('[[word]]\ntext = "rai"\nexcluded = { credential = "a surname" }'), "en.toml": _w('[[word]]\ntext = "rai"\ncourtesy = { acronym = true }')}, "a surname"),
     ("excluded-category-in-another-file", {"und.toml": _w('[[word]]\ntext = "rai"\nexcluded = { suffix = "a surname" }'), "en.toml": _w('[[word]]\ntext = "rai"\ncredential = {}')}, "a surname"),
+    # a reason written as a table reaches the refusal too
+    ("excluded-kind-with-a-table-reason", {"fr.toml": _w('[[word]]\ntext = "fils"\nexcluded = { generation = { reason = "TABLE REASON", issue = 684 } }'), "en.toml": _w('[[word]]\ntext = "fils"\ngeneration = {}')}, "TABLE REASON"),
     ("excluded-property-in-another-file", {"ja.toml": _w('[[word]]\ntext = "博士"\ncredential = { acronym = false, excluded = { attached = "Hiroshi" } }'), "zh.toml": _w('[[word]]\ntext = "博士"\ncredential = { acronym = false, attached = { side = "after" } }')}, "Hiroshi"),
     # 7: same_as and cased
     ("same-as-missing-target", {"en.toml": _w('[[word]]\ntext = "jnr"\ngeneration = { same_as = "jr" }')}, "same_as 'jr'"),
+    ("same-as-its-own-word", {"en.toml": _w('[[word]]\ntext = "jr"\ngeneration = { same_as = "jr" }')}, "names the word itself"),
     ("same-as-other-category", {"en.toml": _w('[[word]]\ntext = "jr"\ntitle = {}\n\n[[word]]\ntext = "jnr"\ngeneration = { same_as = "jr" }')}, "no suffix reading"),
     ("cased-spells-other-letters", {"und.toml": _w('[[word]]\ntext = "phd"\ncredential = { cased = "PhDr" }')}, "case mask"),
     ("two-cased-values", {"und.toml": _w('[[word]]\ntext = "ó"\nparticle = { may_be_given = false, cased = "Ó" }'), "ga.toml": _w('[[word]]\ntext = "ó"\ntitle = { cased = "ó" }')}, "one cased mask per word"),
     ("cased-disagrees-across-files", {"und.toml": _w('[[word]]\ntext = "ó"\nparticle = { may_be_given = false, cased = "Ó" }'), "ga.toml": _w('[[word]]\ntext = "ó"\nparticle = { may_be_given = false, cased = "ó" }')}, "cased is"),
     # 8: phrases
     ("phrase-title-addressing-the-family", {"und.toml": _w('[[word]]\ntext = "deputy secretary"\ntitle = {}')}, "phrase can never match"),
+    ("phrase-with-cased", {"und.toml": _w('[[word]]\ntext = "sir and dame"\ntitle = { addresses = "given", cased = "Sir and Dame" }')}, "looked up per word"),
     ("phrase-particle", {"und.toml": _w('[[word]]\ntext = "de la"\nparticle = { may_be_given = false }')}, "phrase can never match"),
     # 9: a bound given name that is a particle may be given
     ("bound-given-particle-never-given", {"und.toml": _w('[[word]]\ntext = "abu"\nparticle = { may_be_given = false }\nbound_given = {}')}, "may_be_given = true"),
@@ -250,6 +270,15 @@ def test_every_mutation_starts_from_a_pool_the_checks_accept() -> None:
         {"und.toml": _w('[[word]]\ntext = "d\'"\nparticle = { may_be_given = false }\n\n'
                         '[[word]]\ntext = "d’"\nparticle = { may_be_given = false }')},
         {"und.toml": _w('[[word]]\ntext = "sir and dame"\ntitle = { addresses = "given" }')},
+        # a zero-width joiner or non-joiner is spelling, and a token keeps it
+        {"fa.toml": _w('[[word]]\ntext = "ab\\u200ccd"\ntitle = {}\n\n'
+                       '[[word]]\ntext = "ef\\u200dgh"\ntitle = {}')},
+        # an interpunct is part of the word except between two East Asian
+        # characters, and an East Asian text may end in one
+        {"ca.toml": _w('[[word]]\ntext = "l·l"\ntitle = {}'),
+         "zh.toml": _w('[[word]]\ntext = "先·"\nhonorific = {}')},
+        # a phrase word may hold an interior period; a date is a date
+        {"und.toml": _w('[[word]]\ntext = "lt.col and dame"\ntitle = { addresses = "given", decided = 2026-10-18 }')},
         {"fr.toml": _w('[[word]]\ntext = "fils"\nfamily = { us_census_2010 = 767, as_of = 2026-10-16 }\n'
                        'excluded = { generation = { reason = "Marie Fils", issue = 684 } }')},
         # the name kinds refuse only themselves: refused as a given name, a
@@ -274,6 +303,36 @@ def test_every_mutation_starts_from_a_pool_the_checks_accept() -> None:
         _compile(files)
 
 
+def test_every_character_the_tokenizer_ignores_is_refused() -> None:
+    # check 2's hand copies are held to the pipeline, and the sweep is
+    # driven by the tokenizer's own predicate, so a separator or strip
+    # class added there fails here until the build refuses it too
+    from nameparser._pipeline import _tokenize
+    from nameparser._pipeline._state import COMMA_CHARS, ParseState
+    from nameparser._policy import Policy, _SCRIPT_RANGES
+    assert build.DIVIDERS == COMMA_CHARS | _tokenize._NAME_DOT_SEPARATORS
+    assert build.EMOJI_RANGES == _tokenize._EMOJI_RANGES
+    assert build.EAST_ASIAN_RANGES == tuple(
+        span for spans in _SCRIPT_RANGES.values() for span in spans)
+    assert build.INTERPUNCT == _tokenize._INTERPUNCT
+    state = ParseState(original="", lexicon=Lexicon.default(), policy=Policy(),
+                       segmenter=None)
+    ignored = [chr(c) for c in range(0x110000)
+               # a space is the one ignored character a text holds: the
+               # separator between a phrase's words
+               if not 0xD800 <= c <= 0xDFFF and c != 0x20
+               and (_tokenize._ignorable(chr(c), state) or chr(c) in COMMA_CHARS)]
+    assert len(ignored) > 100, len(ignored)  # whitespace, bidi, emoji, dots, commas
+    for ch in ignored:
+        escaped = f"\\U{ord(ch):08X}"  # a TOML escape: no raw control in the file
+        # refused by check 2 itself, not by whichever later check happens
+        # to trip, which would leave a new class guarded by accident
+        with pytest.raises(build.VocabError,
+                           match="divides a name|strips by default|rules/T3|"
+                                 "invisible character|not in normalized form"):
+            build.load({"und.toml": _w(f'[[word]]\ntext = "dr{escaped}x"\ntitle = {{}}')})
+
+
 def test_a_name_reading_on_nothing_warns() -> None:
     files = {"en.toml": _w('[[word]]\ntext = "smith"\nfamily = { us_census_2010 = 2442977, as_of = 2026-10-16 }')}
     with pytest.warns(build.VocabWarning, match="inert"):
@@ -306,6 +365,9 @@ def test_query_tells_a_refused_reading_from_a_refused_property() -> None:
     assert [r.text for r in query.select(pool, excluded="honorific")] == ["君"]
     assert [r.text for r in query.select(pool, excluded="attached")] == ["殿"]
     assert [r.text for r in query.select(pool, excluded="suffix")] == ["君"]
+    # the label is for display only, built from the same fields
+    assert sorted(r.label for r in query.rows(pool)) == [
+        "excluded:honorific", "honorific", "honorific excluded:attached"]
 
 
 def _load_query() -> ModuleType:
@@ -334,6 +396,7 @@ def _load_query() -> ModuleType:
     ({"kind": "suffix"}, ["さん", "殿", "jr"]),       # a category matches its kinds
     ({"kind": "honorific", "excluded": "attached"}, ["殿"]),
     ({"excluded": "suffix"}, ["senior"]),
+    ({"excluded": "attach"}, []),                  # a property by its whole name only
     ({"file": "und.toml"}, ["dr"]),
     ({"word": "jr", "kind": "title"}, []),
 ])