Count a word by box and country in the fielded local corpora

AGENTS.md

@@ -261,7 +261,7 @@ The 2.0 rewrite lands as underscore-private modules alongside the v1 code. These
 
 ## Extension Patterns
 
-**Adding a word to the vocabulary** — the steps are in `nameparser/vocab/README.md`, and the traps they have run into (other readings that clash, reading the real-name counts, the predicates that test a member's spelling, user-doc examples that use the word, recording the vetting on the word) are in `nameparser/vocab/AGENTS.md`, which loads when a session reads a vocabulary TOML file. First check the word's OTHER readings (`uv run python tools/vocab/query.py --word <word>`), then its bearers (`uv run python tools/name_usage.py <word>`). For a title that is also a name (rules/H9's batches, #913), `uv run python tools/vocab/word_report.py <word>... --base <checkout>` prints the H9 shape grid, the word's incidence in every fetched local corpus and its usage rows in one run, and `--rank` orders candidates before a batch is cut; `uv run python tools/differential/measure.py --base <checkout>` then runs every corpus check of the change, one after another, the local ones against the base.
+**Adding a word to the vocabulary** — the steps are in `nameparser/vocab/README.md`, and the traps they have run into (other readings that clash, reading the real-name counts, the predicates that test a member's spelling, user-doc examples that use the word, recording the vetting on the word) are in `nameparser/vocab/AGENTS.md`, which loads when a session reads a vocabulary TOML file. First check the word's OTHER readings (`uv run python tools/vocab/query.py --word <word>`), then its bearers (`uv run python tools/name_usage.py <word>`, with `--boxes` adding rows for the fielded local corpora that count the word by box and country, and `--country CC` the country a pack is for). For a title that is also a name (rules/H9's batches, #913), `uv run python tools/vocab/word_report.py <word>... --base <checkout>` prints the H9 shape grid, the word's incidence in every fetched local corpus and its usage rows in one run, and `--rank` orders candidates before a batch is cut; `uv run python tools/differential/measure.py --base <checkout>` then runs every corpus check of the change, one after another, the local ones against the base.
 
 **Before adding a short/common word to `PARTICLES` globally**, test it mid-string against realistic 3-token names, not just check for English-word collisions: Korean/Vietnamese given names put a short syllable in the middle slot (`Park In Hwan`, `Nguyen To Nga`), and Western names put a bare initial there (`John V. Smith`). A word that looks safe ("nobody is named 'to'") can still swallow a real middle name/initial into the last name once it's a global prefix — confirmed regressions for `to`/`in`/`an`/`ten`/`then` and bare `v` this way (PR #191).
 

tests/v2/test_local_corpora.py

@@ -4500,3 +4500,139 @@ def test_alike_against_a_base_counts_each_direction_apart() -> None:
     assert local.format_alike(nepal, {"0": True}) == ["1 of 1 pairs read alike"]
     with pytest.raises(SystemExit, match="other persons"):
         local.format_alike(nepal, {"0": True, "1": True}, {"0": True})
+
+
+# -- the box index ------------------------------------------------------------------
+
+def _extract(path: Path, header: dict[str, object], lines: list[dict[str, object]]) -> Path:
+    """An extract written as the fetch writes one: a header line, then a
+    record a line, gzipped."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    text = "".join(json.dumps(x) + "\n" for x in (header, *lines))
+    path.write_bytes(gzip.compress(text.encode(), mtime=0))
+    return path
+
+
+def _pv_extract(*lines: dict[str, object]) -> Path:
+    return _extract(patentsview.extract_path(), {"release": patentsview.PV_RELEASE},
+                    list(lines))
+
+
+_PV_LINES = (
+    {"given": "Ana María", "family": "del Río", "country": "ES", "people": 3},
+    {"given": "Juan del", "family": "del Castillo del Río", "country": "MX", "people": 2},
+    {"given": "Ann", "family": "Lee", "country": "US", "people": 10},
+    {"given": "Bo", "family": "DEL.", "people": 1})
+
+
+def test_the_box_index_counts_people_by_box_and_country_once_a_record() -> None:
+    index = local.box_index(patentsview.pv_boxes(_pv_extract(*_PV_LINES)))
+    assert index.totals == {"ES": 3, "MX": 2, "US": 10, "": 1}
+    # 'del Castillo del Río' counts del once; 'DEL.' folds to it as the word
+    # index folds it; the country it is a share of is every record's.
+    assert index.family["del"] == {"ES": 3, "MX": 2, "": 1}
+    assert index.given["del"] == {"MX": 2}
+    assert index.given["maría"] == {"ES": 3} and "lee" not in index.given
+
+
+def test_an_orcid_holder_weighs_one(_own_cache: Path) -> None:
+    path = _extract(orcid.extract_path(), {"file": orcid.ORCID_FILE}, [
+        {"given": "Ann", "family": "Lee", "country": "NZ"}, {"given": "Ann"}])
+    index = local.box_index(orcid.orcid_boxes(path))
+    assert index.totals == {"NZ": 1, "": 1} and index.given["ann"] == {"NZ": 1, "": 1}
+
+
+@pytest.mark.parametrize(("reader", "path", "header"), [
+    ("orcid", "orcid", {"file": "ORCID_2024_10_summaries.tar.gz"}),
+    ("patentsview", "patentsview", {"release": "2024-12-31"})])
+def test_an_extract_of_another_release_is_refused_by_its_reader(
+        reader: str, path: str, header: dict[str, object]) -> None:
+    module = {"orcid": orcid, "patentsview": patentsview}[reader]
+    boxes = module.orcid_boxes if reader == "orcid" else module.pv_boxes
+    written = _extract(module.extract_path(), header, [])
+    with pytest.raises(ValueError, match="is not an extract of"):
+        list(boxes(written))
+
+
+def test_the_cached_box_index_is_read_back_under_its_key_and_rebuilt_off_it(
+        capsys: pytest.CaptureFixture[str]) -> None:
+    _pv_extract(*_PV_LINES)
+    built = local.cached_box_index(local.PV)
+    cache = local.directory(local.PV) / local.BOX_INDEX
+    assert "building the box index" in capsys.readouterr().err
+    assert local.cached_box_index(local.PV) == built
+    assert capsys.readouterr().err == ""  # read back, not built again
+    # A refetched extract is other bytes: the index is built from them.
+    _pv_extract(*_PV_LINES[:2])
+    assert local.cached_box_index(local.PV).totals == {"ES": 3, "MX": 2}
+    assert "rebuilding the box index" in capsys.readouterr().err
+    assert json.loads(gzip.decompress(cache.read_bytes()))["totals"] == {"ES": 3, "MX": 2}
+
+
+@pytest.mark.parametrize("damage", [b"not gzip", gzip.compress(b"{", mtime=0),
+                                    gzip.compress(b'{"key": 1}', mtime=0)],
+                         ids=["not gzip", "not JSON", "no counts"])
+def test_a_cached_box_index_that_cannot_be_read_is_built_over(
+        capsys: pytest.CaptureFixture[str], damage: bytes) -> None:
+    _pv_extract(*_PV_LINES)
+    cache = local.directory(local.PV) / local.BOX_INDEX
+    cache.write_bytes(damage)
+    assert local.cached_box_index(local.PV).totals["US"] == 10
+    assert "rebuilding the box index" in capsys.readouterr().err
+
+
+def test_a_box_index_without_its_extract_names_the_refresh() -> None:
+    local.directory(local.PV)
+    with pytest.raises(ValueError, match="is missing.*fetch patentsview-inventors --refresh"):
+        local.cached_box_index(local.PV)
+
+
+def test_a_change_to_the_box_reader_moves_the_key(monkeypatch: pytest.MonkeyPatch) -> None:
+    before = local.box_fingerprint(local.PV)
+    monkeypatch.setitem(local.SOURCES, local.PV, dataclasses.replace(
+        local.SOURCES[local.PV], boxes=lambda path: iter(())))
+    assert local.box_fingerprint(local.PV) != before
+
+
+def test_the_box_fingerprint_lists_everything_box_index_calls() -> None:
+    # box_fingerprint's list is kept by hand, as _FOLD_CODE is: a helper
+    # box_index starts calling, left off it, leaves the key unmoved.
+    import inspect
+    module = vars(local)
+    listed = {"box_index", "words", "word_key", "_strippable", "_keyed",
+              *local._FOLD_CONSTANTS, "BoxIndex", "Counter"}
+    read: set[str] = set()
+    for name in ("box_index", "words", "word_key", "_keyed"):
+        read |= _globals_read(inspect.unwrap(module[name]).__code__)
+    called = {n for n in read & module.keys()
+              if not isinstance(module[n], types.ModuleType)}
+    assert called - listed == set()
+
+
+@pytest.mark.parametrize("damage", ["truncated", "corrupt", "empty", "no people"])
+def test_an_extract_that_cannot_be_read_fails_as_a_value_error(damage: str) -> None:
+    # _attempt in name_usage catches ValueError and OSError: anything else
+    # gzip, json or the reader raised took every row of the run down.
+    path = _pv_extract(*_PV_LINES)
+    body = path.read_bytes()
+    if damage == "truncated":
+        path.write_bytes(body[: len(body) // 2])
+    elif damage == "corrupt":
+        path.write_bytes(body[:12] + bytes(b ^ 0xFF for b in body[12:40]) + body[40:])
+    elif damage == "empty":
+        path.write_bytes(gzip.compress(b"", mtime=0))
+    else:
+        _pv_extract({"given": "Ann", "family": "Lee", "country": "US"})
+    with pytest.raises(ValueError, match="could not be read.*--refresh"):
+        local.cached_box_index(local.PV)
+
+
+def test_a_cached_box_index_whose_deflate_is_damaged_is_built_over(
+        capsys: pytest.CaptureFixture[str]) -> None:
+    _pv_extract(*_PV_LINES)
+    local.cached_box_index(local.PV)
+    cache = local.directory(local.PV) / local.BOX_INDEX
+    body = cache.read_bytes()
+    cache.write_bytes(body[:12] + bytes(b ^ 0xFF for b in body[12:40]) + body[40:])
+    assert local.cached_box_index(local.PV).totals["US"] == 10
+    assert "rebuilding the box index" in capsys.readouterr().err

tests/v2/test_name_usage_tool.py

@@ -2295,3 +2295,103 @@ def test_a_damaged_local_corpus_fails_its_row_not_the_run(
     out = capsys.readouterr().out
     assert "ctgov-officials (local corpus)" in out and "failed; CorpusDamaged" in out
     assert "US Census 2010 surnames" in out
+
+
+# -- box indexes -----------------------------------------------------------------
+
+def _boxes() -> object:
+    corpora = tool._local_corpora()
+    return corpora.BoxIndex(
+        totals={"ES": 400, "MX": 100, "US": 5000, "": 9000},
+        given={"del": {"MX": 30, "ES": 20, "": 50}},
+        family={"del": {"ES": 60, "MX": 10, "US": 5, "": 80}})
+
+
+def test_a_box_row_counts_each_box_and_gives_each_country_its_share() -> None:
+    row = tool.box_usage("del", "patentsview-inventors", _boxes(), "", 2, [])
+    assert (row.given, row.family) == (100, 155)
+    # Ranked by people writing it, the records stating no country last:
+    # they would lead every list and say nothing of where a name is borne.
+    assert row.given_shares == [("MX", 30, 100), ("ES", 20, 400), ("--", 50, 9000)]
+    assert row.family_shares == [("ES", 60, 400), ("MX", 10, 100), ("--", 80, 9000)]
+    assert "14,500 people in 3 countries" in row.note
+
+
+def test_a_country_asked_for_is_listed_outside_the_top() -> None:
+    row = tool.box_usage("del", "patentsview-inventors", _boxes(), "", 1, ["US", "PH"])
+    assert row.family_shares == [("ES", 60, 400), ("US", 5, 5000), ("PH", 0, 0),
+                                 ("--", 80, 9000)]
+    rendered = tool.render("del", [row])
+    assert "family box by country: ES 60 (15.00% of 400), US 5 (0.10% of 5,000), " \
+           "PH 0 (- of 0), -- 80 (0.89% of 9,000)" in rendered
+
+
+@pytest.mark.parametrize(("word", "says"), [
+    ("de la", "a spaced word is not looked up"),
+    ("..", "nothing in it is left")])
+def test_a_box_row_answers_single_words_only(word: str, says: str) -> None:
+    row = tool.box_usage(word, "patentsview-inventors", _boxes(), "", 5, [])
+    assert row.status == "unrepresentable" and says in row.note
+
+
+def test_a_box_row_not_fetched_or_failed_says_so() -> None:
+    missing = tool.box_usage("del", "patentsview-inventors", None, "", 5, [])
+    assert missing.status == "not downloaded"
+    assert "local_corpora.py fetch patentsview-inventors" in missing.note
+    failed = tool.box_usage("del", "patentsview-inventors", None, "ValueError: gone", 5, [])
+    assert (failed.status, failed.note) == ("failed", "ValueError: gone")
+
+
+def test_only_the_sources_with_boxes_get_a_box_row(
+        monkeypatch: pytest.MonkeyPatch) -> None:
+    built: list[str] = []
+    module = tool._local_corpora()
+    def built_for(source: str) -> object:
+        built.append(source)
+        return _boxes()
+
+    monkeypatch.setattr(module, "cached_box_index", built_for)
+    locals_ = [(source, object(), "") for source in sorted(module.SOURCES)]
+    found = tool.load_box_indexes(locals_)  # type: ignore[arg-type]
+    assert [s for s, _, _ in found] == built == sorted(
+        s for s, src in module.SOURCES.items() if src.boxes is not None)
+    assert {"orcid-researchers", "patentsview-inventors"} <= set(built)
+
+
+def test_the_country_option_reaches_the_box_rows(
+        monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None:
+    monkeypatch.setattr(tool, "load_locals", lambda: [])
+    monkeypatch.setattr(tool, "load_box_indexes", lambda locals_: [
+        ("patentsview-inventors", _boxes(), "")])
+    assert tool.main(["--no-wikidata", "--countries", "1", "--country", "us", "del"]) == 0
+    out = capsys.readouterr().out
+    assert "patentsview-inventors (boxes by country)" in out
+    assert "family box by country: ES 60 (15.00% of 400), US 5 (0.10% of 5,000)" in out
+
+
+def test_the_box_rows_are_built_only_when_asked_for(
+        monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None:
+    # Building one takes minutes and reading one seconds: a plain lookup
+    # must not pay for rows nobody asked for.
+    asked: list[object] = []
+    monkeypatch.setattr(tool, "load_locals", lambda: [])
+    def loaded(locals_: object) -> list[tuple[str, object, str]]:
+        asked.append(1)
+        return [("patentsview-inventors", _boxes(), "")]
+
+    monkeypatch.setattr(tool, "load_box_indexes", loaded)
+    assert tool.main(["--no-wikidata", "del"]) == 0
+    assert asked == [] and "boxes by country" not in capsys.readouterr().out
+    assert tool.main(["--no-wikidata", "--boxes", "del"]) == 0
+    assert asked == [1] and "boxes by country" in capsys.readouterr().out
+
+
+def test_a_country_asked_for_twice_or_blank_is_listed_once(
+        monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None:
+    monkeypatch.setattr(tool, "load_locals", lambda: [])
+    monkeypatch.setattr(tool, "load_box_indexes", lambda locals_: [
+        ("patentsview-inventors", _boxes(), "")])
+    assert tool.main(["--no-wikidata", "--countries", "1", "--country", "us",
+                      "--country", "US", "--country", " ", "del"]) == 0
+    line = next(l for l in capsys.readouterr().out.splitlines() if "family box" in l)
+    assert line.count("US 5") == 1 and line.count("--") == 1