feat(vocabulary): split, classify, weight (M19a)

CLAUDE.md

@@ -66,6 +66,10 @@ packages/navigator/  @codegraph/navigator — the navigator MODEL: the browsable
                      tree (modules → types → operations/attributes) plus one
                      classified dependency row per base edge, carrying member,
                      provenance and anchor included. Pure computation.
+packages/vocabulary/ @codegraph/vocabulary — the words of the model's NAMES
+                     (never source): the universal splitter driven by each
+                     profile's lexicon, the word classes (domain, technical,
+                     keyword, short) and the role weights. Pure computation.
 packages/insights/   @codegraph/insights — the bottom-up EXPLANATION walk: units
                      (operations → types → modules), SCC-condensed walk order,
                      context packs, prompts, Merkle fingerprints, plan/run, the
@@ -107,7 +111,7 @@ fixtures/            Reference corpora + expected model.jsonl snapshots.
 
 ### Hard boundaries (convenience does not override architecture)
 
-- **`core`, `analyzer`, `city` and `navigator` never import Three.js or React**
+- **`core`, `analyzer`, `city`, `navigator` and `vocabulary` never import Three.js or React**
   — they must run in Node with no DOM. `viz` reads the city model and
   `navigator-ui` the navigator model; neither mutates its model and neither
   re-derives graph facts the analyzer already computes.

PLAN.md

@@ -4951,54 +4951,76 @@ lexicon)`, in six steps, each pinned by a table of hand cases per idiom:
    universal, no data: `@total` → `total`, `valid?` → `valid`, `save!`,
    `name=`, `Order::Line`, `max_lines`, `acme-report`, `Repository\`1` →
    `repository`, `1`;
-3. camel-case boundaries with the acronym rule — lower→Upper, and an
-   Upper run followed by Upper + lower splits before its last letter
+3. camel-case boundaries with the acronym rule — lower→Upper, digit→Upper,
+   and an Upper run followed by Upper + lower splits before its last letter
    (`HTTPServer` → `http`, `server`; `getURLString` → `get`, `url`,
    `string`; `XMLHttpRequest` → `xml`, `http`, `request`) — on Unicode
    letter categories, not ASCII (`fixtures/unicode` is the test: `ＡFullwidth`
    and `𠀀Supplementary` must split and lowercase without a replacement
-   character);
-4. a digit run attaches to the token before it (`utf8`, `md5`, `v2`); a
-   token that is only digits is dropped and counted (`skipped.numeric`);
+   character). A caseless letter (`𠀀`, `注`) counts as lower; a mark takes
+   the case of the letter it sits on, so a decomposed `é` never splits;
+4. a digit run stays with the letters around it — only an Upper after it
+   breaks (`utf8`, `md5`, `v2`, `i18n`, `http2`; `Utf8Encoder` → `utf8`,
+   `encoder`); a token that is only digits is dropped and counted
+   (`skipped.numeric`);
 5. lowercase (`toLowerCase`, locale-independent: a Turkish-locale machine
    must write the same bytes);
 6. a `conventional` identifier is skipped whole BEFORE step 1 and counted
    (`skipped.conventional`): `toString` must not contribute `string`,
    `handle_call` must not contribute `call`.
 
-Properties (fast-check over identifiers the grammar admits): no empty token;
-every token lowercase; every token is, case-insensitively, a substring of
-the input with the affixes removed; the same input gives the same tokens;
-`split(a + sep + b) = split(a) ++ split(b)` for any lexicon separator.
+Properties (fast-check over identifiers the grammar admits): every token is
+a non-empty run of letters, marks and digits, lowercase, never digits only,
+and a slice of the lowered name; the same input gives the same tokens;
+`split(a + sep + b) = split(a) ++ split(b)` for any character that is not a
+letter, a mark or a digit — without a lexicon, since the file kinds, the
+interface prefix and a conventional name read the WHOLE name. A Σ is left
+out of the generator: its lower case depends on what follows it, so a
+token's lower case and its slice of the lowered name may differ (`ΟΔΟΣ` →
+`οδος`, pinned by hand).
 
 **The classes** (`classify.ts`), a closed set `TERM_CLASSES = ["domain",
 "technical", "keyword", "short"]`, decided per TOKEN in this order:
 `keyword` when the lowercased token equals a lowercased lexicon keyword
-(`type`, `default`, `case`, `new`, `end`); `short` when it is one character
-(`i`, `e`, `x` — length 2 stays: `id`, `db`, `io` are words); `technical`
-when it is in the shared list ∪ the profile's (`DEFAULT_TECHNICAL`, ~80
-words in the package: `get set is has to from of with by and or not new
-create build make init impl abstract base default util utils helper manager
-service repository controller handler factory provider dto dao entity model
-data info config exception error test mock spec com org net io src lib main
-app`…, overridable whole with `--technical FILE`, its digest recorded in the
-artifact); else `domain`. The list is contestable by construction — which is
-why it is data in one place and the artifact says which one it used.
+(`type`, `default`, `case`, `new`, `end`) — a keyword the splitter never
+yields whole (`defined?`, `__FILE__`, `set!`) classifies nothing, since the
+`file` of `upload_file` is not `__FILE__`; `short` when it is one code point
+that is not a mark (`i`, `e`, `x`, `𠀀` — length 2 stays: `id`, `db`, `io`
+are words); `technical` when it is in the shared list ∪ the profile's
+(`DEFAULT_TECHNICAL`, 125 words in the package: `get set is has to from of
+with by and or not new create build make init impl abstract base default
+util utils helper manager service repository controller handler factory
+provider dto dao entity model data info config exception error test mock
+spec com org net io src lib main app`, the CRUD verbs, the web and format
+words…, overridable whole with `--technical FILE`, its digest recorded in
+the artifact — the profile's own words apply either way); else `domain`. The
+list is contestable by construction — which is why it is data in one place
+and the artifact says which one it used. It holds what every language
+shares: a word two lexicons listed (`args`, `opts`, `ctx`, `err`,
+`callback`, `str`, `mutex`, `mixin`, `async`) moved into it, and a test
+keeps it disjoint from every profile's `technical`.
 
 **Roles and weights** (`weight.ts`). The role of the entity carrying a name
-comes from its traits, never its kind (the navigator's rule): `module`
-(TModule), `type` (TType), `operation` (TInvocable), `parameter` / `local`
-(TStructural listed in the parent's `parameters[]` / `localVariables[]`),
-`attribute` (every other TStructural: fields, constants, properties, module
-attributes). Default weights, recorded in the artifact and overridable with
-`--weights role=n,…`:
+comes from its traits, never its kind (the navigator's rule), first match
+winning: `module` (TModule), `type` (TType — a C# delegate, TType +
+TInvocable, is a type), `operation` (TInvocable — a Clojure fn-var too),
+`parameter` / `local` (TStructural listed in some entity's `parameters[]` /
+`localVariables[]`), `attribute` (every other TStructural: fields,
+constants, properties, module attributes), and `other` — a named entity no
+role trait describes (TypeScript's `namespace`, PHP's `codeFile`, Python's
+`decorator`), whose words still count (principle 1). Built on the fixtures,
+one parameter is listed nowhere: the C# indexer's, a child of its
+`property`, which the C# profile gives no `TWithParameters`; it weighs as
+an attribute until the model lists it, pinned by id. Default weights,
+recorded in the artifact and overridable with `--weights role=n,…`:
 
 | role | weight | why |
 |---|---|---|
 | `type`, `module` | 8 | a concept the corpus names once and refers to everywhere |
 | `attribute` | 4 | a thing the type holds — nouns, usually domain |
 | `operation` | 3 | something it does; the prefix is usually technical (`get`, `find`, `build`) |
 | `parameter`, `local` | 1 | incidental: the "variable buried deep in a method" of the issue |
+| `other` | 1 | no trait says what it is |
 
 An occurrence is one token in one name; a name carrying a word twice counts
 twice. Both the weighted sum and the raw occurrence count are kept — one is
@@ -5256,12 +5278,13 @@ never looks like a fact.
       specification; conventional names from its runtime), `codegraph
       profiles --lang` printing it; the lexicon's validation codes and tests; `schemas/`
       unchanged, proven by the drift check.
-- [ ] `packages/vocabulary/` (`@codegraph/vocabulary`, depends on core and
+- [x] `packages/vocabulary/` (`@codegraph/vocabulary`, depends on core and
       analyzer only): `split.ts`, `classify.ts` with `DEFAULT_TECHNICAL`,
-      `weight.ts`, `aggregate.ts` on `containingType`/`containingModule`,
-      `artifact.ts` (`codegraph.vocabulary/1`), `drift.ts`; the hand tables
-      per idiom and the properties of §20.1; the unicode fixture through the
-      splitter.
+      `weight.ts`; the hand tables per idiom and the properties of §20.1;
+      the unicode fixture through the splitter; the shared words out of
+      the profiles.
+- [ ] `aggregate.ts` on `containingType`/`containingModule`, `artifact.ts`
+      (`codegraph.vocabulary/1`), `drift.ts`.
 - [ ] `codegraph vocabulary` (§20.1): the report, `--level`, `--scope`,
       `--class`, `--weights`, `--technical`, `--drift`, `--out`, `--json`,
       the view flags; usage errors for an unknown scope, role or class.

eslint.config.js

@@ -58,6 +58,7 @@ export default tseslint.config(
       "packages/analyzer/**/*.ts",
       "packages/city/**/*.ts",
       "packages/navigator/**/*.ts",
+      "packages/vocabulary/**/*.ts",
       "packages/insights/**/*.ts",
       "packages/cli/**/*.ts",
     ],

extractors/typescript/vitest.config.ts

@@ -13,4 +13,7 @@ export default defineConfig({
       "@codegraph/core": fileURLToPath(new URL("../../packages/core/src/index.ts", import.meta.url)),
     },
   },
+  // An extraction builds a TypeScript program: 0.2–0.5 s here, 2–5 s on a CI runner
+  // under `pnpm -r test`, where the 5 s default timed out a test that extracts twice.
+  test: { testTimeout: 30_000 },
 });

packages/core/src/profiles/clojure.ts

@@ -139,7 +139,7 @@ export const clojureProfile: Profile = {
       "-main", "-init", "-post-init", "-toString", "-equals", "-hashCode",
     ],
     technical: [
-      "coll", "ctx", "kw", "ns", "opts", "seq", "sym", "xf", "xform",
+      "coll", "kw", "ns", "seq", "sym", "xf", "xform",
     ],
   },
 };

packages/core/src/profiles/csharp.ts

@@ -246,7 +246,7 @@ export const csharpProfile: Profile = {
       "op_LeftShift", "op_RightShift", "op_UnsignedRightShift",
     ],
     technical: [
-      "args", "async", "attribute", "enumerable", "enumerator", "extension", "extensions", "linq",
+      "attribute", "enumerable", "enumerator", "extension", "extensions", "linq",
     ],
     interfacePrefix: true,
   },

packages/core/src/profiles/ecmascript.ts

@@ -24,6 +24,6 @@ export const ECMASCRIPT_LEXICON = {
     "[Symbol.toStringTag]", "[Symbol.dispose]", "[Symbol.asyncDispose]",
   ],
   technical: [
-    "args", "async", "callback", "cb", "fn", "json", "opts", "promise", "prototype",
+    "cb", "fn", "promise", "prototype",
   ],
 } as const satisfies Lexicon;

packages/core/src/profiles/elixir.ts

@@ -155,9 +155,8 @@ export const elixirProfile: Profile = {
       "@fallback_to_any", "@file",
     ],
     technical: [
-      "attrs", "behaviour", "callback", "changeset", "conn", "ecto", "ets", "gen", "live", "mix",
-      "opts", "otp", "phoenix", "pid", "plug", "repo", "schema", "server", "socket", "sup",
-      "supervisor", "web",
+      "attrs", "behaviour", "changeset", "conn", "ecto", "ets", "gen", "live", "mix", "otp",
+      "phoenix", "pid", "plug", "schema", "server", "socket", "sup", "supervisor", "web",
     ],
     fileKinds: ["file"],
   },

packages/core/src/profiles/go.ts

@@ -125,7 +125,7 @@ export const goProfile: Profile = {
       "MarshalJSON", "UnmarshalJSON", "MarshalText", "UnmarshalText", "ServeHTTP",
     ],
     technical: [
-      "cfg", "cmd", "ctx", "err", "errs", "internal", "mu", "mutex", "pkg", "srv", "svc", "wg",
+      "cfg", "cmd", "errs", "internal", "mu", "pkg", "srv", "svc", "wg",
     ],
   },
 };

packages/core/src/profiles/php.ts

@@ -148,7 +148,7 @@ export const phpProfile: Profile = {
       "__set_state", "__clone", "__debugInfo",
     ],
     technical: [
-      "arr", "closure", "facade", "middleware", "php", "str",
+      "arr", "closure", "facade", "middleware", "php",
     ],
     fileKinds: ["codeFile"],
   },

packages/core/src/profiles/python.ts

@@ -131,7 +131,7 @@ export const pythonProfile: Profile = {
       "__doc__", "__dict__",
     ],
     technical: [
-      "args", "dict", "kwargs", "metaclass", "mixin", "py", "str", "tuple",
+      "dict", "kwargs", "metaclass", "py", "tuple",
     ],
   },
 };

packages/core/src/profiles/ruby.ts

@@ -133,7 +133,7 @@ export const rubyProfile: Profile = {
     ],
     technical: [
       "accessor", "attr", "concern", "concerns", "ext", "gem", "gemspec", "klass", "lambda",
-      "mixin", "proc", "rake", "rakefile",
+      "proc", "rake", "rakefile",
     ],
     fileKinds: ["file"],
   },

packages/core/src/profiles/rust.ts

@@ -124,8 +124,8 @@ export const rustProfile: Profile = {
       "borrow_mut", "index_mut", "default", "poll",
     ],
     technical: [
-      "arc", "buf", "cell", "ctx", "err", "ffi", "iter", "mutex", "ok", "opt", "rc", "refcell",
-      "rs", "rx", "sys", "tx", "vec",
+      "arc", "buf", "cell", "ffi", "iter", "ok", "opt", "rc", "refcell", "rs", "rx", "sys",
+      "tx", "vec",
     ],
   },
 };

packages/insights/test/store.test.ts

@@ -90,15 +90,20 @@ const blockArb = {
 };
 const strip = <T extends object>(o: T): T => Object.fromEntries(Object.entries(o).filter(([, v]) => v !== undefined)) as T;
 
+// Built once, never per draw: a regex arbitrary costs a parse, and drawing a fresh one per record timed this suite out on CI.
+const fingerprintArb = fc.stringMatching(/^[0-9a-f]{64}$/u);
+const envelopeArb = (level: "operation" | "type" | "module") =>
+  fc.record({
+    key: maybe(keyArb), kind: text, name: maybe(text), file: maybe(text), scc: maybe(fc.array(text, { maxLength: 3 })),
+    origin: fc.constantFrom("llm", "template"), block: blockArb[level as "operation"], fingerprint: fingerprintArb,
+    model: maybe(text), usage: maybe(usageArb), metadata: maybe(fc.dictionary(text, text, { maxKeys: 3 })),
+  });
+const ENVELOPES = { operation: envelopeArb("operation"), type: envelopeArb("type"), module: envelopeArb("module") };
 const recordArb = (id: string): fc.Arbitrary<InsightRecord> =>
-  fc.constantFrom("operation", "type", "module").chain((level) =>
-    fc
-      .record({
-        key: maybe(keyArb), kind: text, name: maybe(text), file: maybe(text), scc: maybe(fc.array(text, { maxLength: 3 })),
-        origin: fc.constantFrom("llm", "template"), block: blockArb[level as "operation"], fingerprint: fc.stringMatching(/^[0-9a-f]{64}$/u),
-        model: maybe(text), usage: maybe(usageArb), metadata: maybe(fc.dictionary(text, text, { maxKeys: 3 })),
-      })
-      .map((r) => strip({ t: "i", id, level, ...r }) as unknown as InsightRecord),
+  fc.oneof(
+    ...(["operation", "type", "module"] as const).map((level) =>
+      ENVELOPES[level].map((r) => strip({ t: "i", id, level, ...r }) as unknown as InsightRecord),
+    ),
   );
 
 const failureArb = (id: string): fc.Arbitrary<FailureRecord> =>

packages/vocabulary/package.json

@@ -0,0 +1,31 @@
+{
+  "name": "@codegraph/vocabulary",
+  "version": "0.2.0",
+  "license": "MIT",
+  "type": "module",
+  "description": "Codegraph vocabulary: the words of a model's names, classified and weighted by the role of what carries them",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "files": ["dist"],
+  "scripts": {
+    "build": "tsup",
+    "typecheck": "tsc -p tsconfig.json --noEmit",
+    "test": "vitest run"
+  },
+  "dependencies": {
+    "@codegraph/core": "workspace:*"
+  },
+  "devDependencies": {
+    "@types/node": "catalog:",
+    "fast-check": "catalog:",
+    "tsup": "catalog:",
+    "typescript": "catalog:",
+    "vitest": "catalog:"
+  }
+}

packages/vocabulary/src/classify.ts

@@ -0,0 +1,66 @@
+import type { Lexicon } from "@codegraph/core";
+
+/**
+ * The classes (PLAN.md §20.1): every word is kept and tagged, never filtered —
+ * drift needs the technical words too, and "not shown" must stay
+ * distinguishable from "not there".
+ */
+
+export const TERM_CLASSES = ["domain", "technical", "keyword", "short"] as const;
+export type TermClass = (typeof TERM_CLASSES)[number];
+
+/**
+ * Plumbing every language shares. Contestable by construction, which is why
+ * it is data in one place, overridable whole, and the artifact names the list
+ * it used. A language's own plumbing (Java's `bean`, Ruby's `concern`) is its
+ * profile's `lexicon.technical`, never repeated here.
+ */
+export const DEFAULT_TECHNICAL: readonly string[] = Object.freeze([
+  "abstract", "adapter", "add", "and", "api", "app", "arg", "args", "as", "async",
+  "at", "base", "build", "builder", "by", "callback", "can", "cli", "com", "common",
+  "config", "configuration", "context", "controller", "core", "create", "ctx", "dao", "data", "default",
+  "delete", "dto", "entity", "env", "err", "error", "errors", "exception", "factory", "fake",
+  "fetch", "find", "fixture", "for", "from", "generic", "get", "handle", "handler", "has",
+  "helper", "helpers", "http", "id", "impl", "in", "info", "init", "into", "io",
+  "is", "json", "lib", "listener", "load", "main", "make", "manager", "misc", "mixin",
+  "mock", "model", "mutex", "net", "new", "not", "obj", "of", "on", "options",
+  "opts", "or", "org", "param", "params", "provider", "proxy", "remove", "repo", "repository",
+  "req", "request", "res", "resp", "response", "result", "run", "save", "service", "set",
+  "settings", "setup", "shared", "should", "spec", "src", "str", "stub", "teardown", "temp",
+  "test", "tests", "tmp", "to", "update", "uri", "url", "util", "utils", "uuid",
+  "val", "visitor", "with", "wrapper", "xml",
+]);
+
+export type Classify = (token: string) => TermClass;
+
+const WORD = /^[\p{L}\p{M}\p{N}]+$/u;
+const MARK = /\p{M}/u;
+
+/**
+ * Decides, in this order: `keyword` (the lexicon reserves the word), `short`
+ * (one letter or digit), `technical` (the shared list ∪ the profile's), else
+ * `domain`. A keyword the splitter never yields whole (`defined?`, `__FILE__`)
+ * classifies nothing: the `file` of `upload_file` is not `__FILE__`.
+ */
+export function createClassifier(
+  lexicon: Lexicon | undefined,
+  technical: readonly string[] = DEFAULT_TECHNICAL,
+): Classify {
+  const keywords = new Set(
+    (lexicon?.keywords ?? []).map((keyword) => keyword.toLowerCase()).filter((keyword) => WORD.test(keyword)),
+  );
+  const plumbing = new Set([...technical, ...(lexicon?.technical ?? [])]);
+  return (token) => {
+    if (keywords.has(token)) return "keyword";
+    if (isShort(token)) return "short";
+    if (plumbing.has(token)) return "technical";
+    return "domain";
+  };
+}
+
+/** One code point that is not a mark: `x`, `𠀀`, a decomposed `é`. */
+function isShort(token: string): boolean {
+  let letters = 0;
+  for (const char of token) if (!MARK.test(char) && ++letters > 1) return false;
+  return true;
+}

packages/vocabulary/src/index.ts

@@ -0,0 +1,11 @@
+/**
+ * `@codegraph/vocabulary` — the words a corpus speaks (PLAN.md §20): every
+ * name the model carries, split into words, each tagged with a class and
+ * weighted by the role of the entity that carries it.
+ *
+ * PURE COMPUTATION, like `@codegraph/city`: reads names and a profile's
+ * lexicon, never source, and touches no DOM.
+ */
+export * from "./split.js";
+export * from "./classify.js";
+export * from "./weight.js";