Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions architecture/evaluator.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,13 @@ ensemble of two or more must span at least two architecturally diverse model
families, validated when the config is built and again at run time, so an
ensemble never rests on one model family. A single judge is allowed.

Each judge answers within its model's `max_tokens`, scaled up for locales whose
script tokenizes densely, as the simulator's calls are; a judge whose model
sets none gets 4,096 tokens, or 8,192 in those locales. A judge that reasons
before answering can need more, which `max_tokens` on its row in the models
file gives it. An answer that doesn't parse leaves the row without a judge
verdict, and the next `usersim eval` of the run judges that row again.

Every row is judged on the axes every probe shares. A built-in probe whose
rows need axes of their own declares them in its package's `axes.py`, as
`JUDGE_AXES` keyed by the probe's label, which is the row's `probe_family`.
Expand Down
14 changes: 12 additions & 2 deletions src/usersim/engine/evaluator/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -151,8 +151,18 @@ class TrajectoryEvaluatorConfig(SingleColumnConfig):
)

# ── Operational knobs ───────────────────────────────────────────
max_judge_tokens: int = 4096
max_judge_tokens_non_ascii: int = 8192
max_judge_tokens: int = Field(
default=4096,
description=(
"Answer budget for a judge whose model sets no max_tokens. A judge "
"model's own max_tokens takes precedence, scaled up for locales "
"whose script tokenizes densely."
),
)
max_judge_tokens_non_ascii: int = Field(
default=8192,
description="max_judge_tokens for locales whose script tokenizes densely.",
)
verbosity: int = 1

# ── Validators ─────────────────────────────────────────────────
Expand Down
18 changes: 14 additions & 4 deletions src/usersim/engine/evaluator/generator.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@
)

from usersim.engine.core.identity import resolve_model_name
from usersim.engine.core.llm import acall_llm
from usersim.engine.core.llm import NON_ASCII_TOKEN_SCALE, acall_llm, scaled_max_tokens
from usersim.engine.core.messages import format_conversation_history_for_prompt
from usersim.engine.core.missing import none_for_missing
from usersim.engine.core.persona import format_persona_for_prompt
Expand Down Expand Up @@ -302,7 +302,6 @@ async def agenerate(self, data: dict) -> dict:
rubric=render_rubric(applicable_axes),
)

max_tokens = cfg.max_judge_tokens_non_ascii if locale in _NON_ASCII_LOCALES else cfg.max_judge_tokens
schema_model = _build_schema_for(applicable_axes)

for spec in judge_specs if applicable_axes else ():
Expand All @@ -312,7 +311,7 @@ async def agenerate(self, data: dict) -> dict:
judge_alias=spec.alias,
prompt=prompt,
schema_model=schema_model,
max_tokens=max_tokens,
max_tokens=self._judge_max_tokens(models.get(spec.alias), locale in _NON_ASCII_LOCALES),
)
judge_elapsed = time.monotonic() - t_judge
logger.debug(
Expand Down Expand Up @@ -392,6 +391,13 @@ def get_models(self) -> dict[str, Any]:
logger.warning(f" |-- evaluator: judge alias {j.alias!r} not resolvable in model registry: {e}")
return out

def _judge_max_tokens(self, facade: Any, non_ascii: bool) -> int:
"""A judge's answer budget: its model's ``max_tokens``, scaled up for a dense script, else the config's."""
configured = scaled_max_tokens(facade, NON_ASCII_TOKEN_SCALE if non_ascii else 1.0).get("max_tokens")
if configured:
return configured
return self.config.max_judge_tokens_non_ascii if non_ascii else self.config.max_judge_tokens

async def _call_judge(
self,
models: dict[str, Any],
Expand Down Expand Up @@ -426,7 +432,11 @@ async def _call_judge(
try:
parsed = json.loads(content)
except (json.JSONDecodeError, TypeError):
logger.warning(f" |-- evaluator: judge {judge_alias!r} produced unparseable structured output")
logger.warning(
f" |-- evaluator: judge {judge_alias!r} produced unparseable structured output within "
f"max_tokens={max_tokens}; a judge that reasons before answering may need a larger max_tokens "
"on its model's row"
)
return {}
return parsed if isinstance(parsed, dict) else {}

Expand Down
36 changes: 36 additions & 0 deletions tests/engine/evaluator/test_evaluator_generator.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ def _build_generator(
facades = {j.alias: MagicMock(name=f"facade_{j.alias}") for j in cfg.judges}
for alias, facade in facades.items():
facade.model_config.model = f"{alias}-model"
facade._model_config.inference_parameters.max_tokens = None
gen.get_model = lambda alias: facades.get(alias) # type: ignore[method-assign]
gen.get_models = lambda: facades # type: ignore[method-assign]

Expand Down Expand Up @@ -166,6 +167,41 @@ async def _capture(models, judge_alias, prompt, schema_model, max_tokens):

assert ("TAIL_FIGURE" in prompts[0]) is sees_tail

@pytest.mark.parametrize(
("locale", "configured", "expected"),
[
("en_US", {"judge_a": 16384, "judge_b": 2048}, {"judge_a": 16384, "judge_b": 2048}),
("ja_JP", {"judge_a": 16384, "judge_b": 2048}, {"judge_a": 32768, "judge_b": 4096}),
("en_US", {"judge_a": None, "judge_b": None}, {"judge_a": 4096, "judge_b": 4096}),
("ja_JP", {"judge_a": None, "judge_b": None}, {"judge_a": 8192, "judge_b": 8192}),
],
)
async def test_each_judge_answers_within_its_models_max_tokens(
self, locale: str, configured: dict[str, int | None], expected: dict[str, int]
) -> None:
gen = _build_generator(TrajectoryEvaluatorConfig(name="eval_v2", judges=_ok_judges()))
for alias, max_tokens in configured.items():
gen.get_models()[alias]._model_config.inference_parameters.max_tokens = max_tokens
budgets: dict[str, int] = {}

async def _capture(models, judge_alias, prompt, schema_model, max_tokens):
budgets[judge_alias] = max_tokens
return {}

gen._call_judge = _capture # type: ignore[method-assign]
await gen.agenerate(
{
"conversation_messages": json.dumps(
[{"role": "user", "content": "hi"}, {"role": "assistant", "content": "hello"}]
),
"persona": json.dumps({"first_name": "A", "last_name": "B"}),
"probe_family": "tool_calling",
"locale": locale,
}
)

assert budgets == expected

async def test_a_row_with_no_family_is_judged_on_the_universal_axes_alone(self) -> None:
gen = _build_generator(TrajectoryEvaluatorConfig(name="eval_v2", judges=_ok_judges()))
row = {
Expand Down
Loading