Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion architecture/probes.md
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ its label.
| `verbatim_field` | `"verbatim_text"` | `BankVerbatimMixin`: the task field turn 1 reads |
| `turn1_instruction_template` | `""` | `CustomTurn1InstructionMixin`: the instruction turn 1 is written from |
| `tool_loop_mode` | `"single"` | `ToolExecutionMixin`: one round of tool calls per turn, or with `"multi"` rounds until the assistant stops calling tools |
| `tool_max_calls_per_turn` | `8` | `ToolExecutionMixin`: the most tool calls it runs in one turn. A call past it gets a reply saying it was not run; after such a reply, the turn ends with a reply made without tools |
| `tool_max_calls_per_turn` | `8` | `ToolExecutionMixin`: the most tool calls it runs in one turn. A call past it gets a reply saying it was not run and to answer the user with the results the assistant has; after such a reply, the turn ends with a reply made without tools |

## Mixins

Expand Down
8 changes: 5 additions & 3 deletions src/usersim/engine/core/probes.py
Original file line number Diff line number Diff line change
Expand Up @@ -1271,8 +1271,9 @@ class ToolExecutionMixin:
across several assistant<->tool round-trips).

In either shape, a call beyond the turn's cap is not run: it gets a
``role:"tool"`` reply saying so, so every call an assistant message makes
has a reply, and the assistant sees why before it answers.
``role:"tool"`` reply saying so and asking for an answer with the results
in hand, so every call an assistant message makes has a reply, and the
assistant sees why before it answers.

Hooks a probe implements:

Expand Down Expand Up @@ -1379,7 +1380,8 @@ async def after_assistant_turn(
{
"error": (
"Not run: this turn reached its limit of "
f"{self.tool_max_calls_per_turn} tool calls."
f"{self.tool_max_calls_per_turn} tool calls. "
"Answer the user now with the results you have."
),
"status": 429,
}
Expand Down
15 changes: 12 additions & 3 deletions src/usersim/engine/core/simulation.py
Original file line number Diff line number Diff line change
Expand Up @@ -440,12 +440,17 @@ def _is_assistant_refusal_echo(text: str, allowed: frozenset[str] = frozenset())
"Does this user message end the conversation: the user is finished, satisfied "
"or not, and the message asks the assistant to do nothing further? A message "
"that also makes a request, gives an instruction or asks a question, such as "
"asking it to book, change or check something, does not end it. Consider the "
"language and cultural context.\n"
"asking it to book, change or check something, does not end it. Nor does one "
"that answers the assistant's question or gives it the details, a choice or a "
"correction it asked for, even when it also thanks the assistant or says that "
"is all it needs. Consider the language and cultural context.\n"
"Reply with ONLY 'yes' or 'no'.\n\n"
"User message: {follow_up}"
)

# A question mark in Latin, full-width (CJK) or Arabic script.
_QUESTION_MARK = re.compile(r"[??؟]")


# Tool-call syntax that reached the reply as text instead of as a tool call.
_TOOL_CALL_MARKUP = re.compile(r"<tool_call>|<function=[\w.\-]+>|\[TOOL_CALLS\]")
Expand Down Expand Up @@ -477,8 +482,12 @@ async def _check_conversation_complete(models: dict, follow_up: str) -> bool:
"""Ask the summary_model whether the user is signaling conversation completion.

Language-agnostic: works for any locale without per-language regex patterns.
Uses the cheapest model (summary_model) with a short yes/no prompt.
Uses the cheapest model (summary_model) with a short yes/no prompt. A message
holding a question mark asks for something, so it never ends the
conversation and needs no model call.
"""
if _QUESTION_MARK.search(follow_up or ""):
return False
prompt = _EARLY_STOP_PROMPT.format(follow_up=follow_up)
msgs = [{"role": "user", "content": prompt}]
resp = await acall_llm(models, MODEL_SUMMARY, msgs)
Expand Down
18 changes: 18 additions & 0 deletions tests/engine/core/test_engine_e2e.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
from usersim.engine.core.outcomes import FailureAttribution, FailureClass, OutcomeStatus
from usersim.engine.core.probes import BaseProbe
from usersim.engine.core.simulation import (
_EARLY_STOP_PROMPT,
ConversationLoop,
_check_conversation_complete,
_poor_assistant_response_reason,
Expand Down Expand Up @@ -155,6 +156,23 @@ async def test_llm_based_handles_whitespace(self):
models = {"summary_model": StubFacade([" Yes \n"])}
assert await _check_conversation_complete(models, "Perfect, thanks!")

@pytest.mark.parametrize(
"follow_up",
[
"That cutoff's already passed, so can you find the soonest option I can actually book?",
"Merci ! Et pour dimanche, tu regardes aussi ?",
"ありがとう。で、来週の分は?",
],
)
async def test_a_message_with_a_question_never_ends_the_conversation(self, follow_up):
summary = StubFacade(["yes"])
assert not await _check_conversation_complete({"summary_model": summary}, follow_up)
assert summary._idx == 0 # decided without a model call

def test_the_question_covers_a_message_that_answers_the_assistant(self):
assert "answers the assistant's question or gives it the details" in _EARLY_STOP_PROMPT
assert "even when it also thanks the assistant or says that is all it needs" in _EARLY_STOP_PROMPT


# ---------------------------------------------------------------------------
# Tests: assistant quality check
Expand Down
2 changes: 1 addition & 1 deletion tests/engine/core/test_episode_runtime_parity.py
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ async def acompletion(self, messages: Sequence[Any], **kwargs: Any) -> SimpleNam
"assistant": "Scripted assistant response.",
"judge": "<explanation>valid scripted turn</explanation><rating>success</rating>",
"summary": "yes",
"user": "Could you explain that a little more?",
"user": "Please explain that a little more.",
}[self.role]
return SimpleNamespace(
message=SimpleNamespace(
Expand Down
18 changes: 18 additions & 0 deletions tests/engine/core/test_probes_substrate.py
Original file line number Diff line number Diff line change
Expand Up @@ -660,6 +660,24 @@ async def test_a_call_past_the_cap_is_answered_but_not_run(self, mode: str) -> N
assert replies["c2"]["status"] == 429
assert _unanswered_call_ids(state.messages) == set()

async def test_a_call_past_the_cap_tells_the_assistant_to_answer_with_what_it_has(self) -> None:
probe = _ToolExecProbe(mode="multi", cap=1)
state = _StubState()
with patch(
"usersim.engine.core.llm.acall_llm",
return_value={"content": "capped synth", "tool_calls": None},
):
await probe.after_assistant_turn(
{},
state,
{"content": "", "tool_calls": [_tc(cid="c1"), _tc(cid="c2")]},
object(),
)
refused = next(json.loads(m["content"]) for m in state.messages if m.get("tool_call_id") == "c2")
assert refused["error"] == (
"Not run: this turn reached its limit of 1 tool calls. Answer the user now with the results you have."
)

async def test_multi_mode_turn_at_the_cap_is_told_before_its_reply_without_tools(self) -> None:
probe = _ToolExecProbe(mode="multi", cap=2)
state = _StubState()
Expand Down
2 changes: 1 addition & 1 deletion tests/probes/general_open_ended/test_probe.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ class KyotoTrip(ScriptedModel):

CONTENT = {
**ScriptedModel.CONTENT,
"user": "週末に京都へ行くなら、どのお寺を見ればいいですか?",
"user": "週末に京都へ行くので、見るべきお寺を教えてください。",
"assistant": "清水寺と伏見稲荷大社がおすすめです。",
"summary": "no",
}
Expand Down
Loading