Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions crates/xgen-cli/src/driver.rs
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,11 @@ pub enum DriverProgressControl {
Cancel,
}

/// Consecutive model responses the driver may discard before it stops the Run (ADR-0061).
///
/// Each retry is a new journaled model call and spends the Run's model-call budget.
pub const MAX_REJECTED_RESPONSE_RETRIES: u32 = 2;

/// Provider-neutral bounded coordinator used by CLI composition roots and hermetic tests.
#[derive(Debug, Clone)]
pub struct RunDriver {
Expand Down Expand Up @@ -312,6 +317,7 @@ impl RunDriver {
AP: ApprovalPort,
O: FnMut(DriverProgress) -> DriverProgressControl,
{
let mut rejected_responses_in_a_row = 0_u32;
for _ in 0..self.max_ticks.get() {
if model_call_may_be_next(store)? {
if !model_egress_allowed {
Expand All @@ -335,6 +341,7 @@ impl RunDriver {
| AgentLoopTick::ModelCallLifecycleConfigured { .. }
| AgentLoopTick::ModelCallAbandoned { .. } => {}
AgentLoopTick::PlanAccepted { .. } => {
rejected_responses_in_a_row = 0;
if observer(DriverProgress::PlanCommitted) == DriverProgressControl::Cancel {
return Ok(DriverOutcome::Cancelled);
}
Expand Down Expand Up @@ -487,9 +494,28 @@ impl RunDriver {
return Ok(DriverOutcome::Quiescent(reason));
}
AgentLoopTick::PlannerUnavailable { failure, .. } => {
// ADR-0061: the rejected call is already settled in the journal, so the next
// tick reserves a fresh call exactly like `/resume` would.
if failure == PlannerPortFailure::InvalidResponse
&& rejected_responses_in_a_row < MAX_REJECTED_RESPONSE_RETRIES
{
rejected_responses_in_a_row += 1;
continue;
}
return Ok(DriverOutcome::PlannerUnavailable(failure));
}
AgentLoopTick::ProposalRejected { reason, .. } => {
// A spent budget rejects every later proposal the same way.
let budget_spent = matches!(
reason,
ProposalRejection::ToolCallBudgetExhausted
| ProposalRejection::PlannedStepBudgetExceeded
);
if !budget_spent && rejected_responses_in_a_row < MAX_REJECTED_RESPONSE_RETRIES
{
rejected_responses_in_a_row += 1;
continue;
}
return Ok(DriverOutcome::ProposalRejected(reason));
}
AgentLoopTick::MaterializerUnavailable { failure, .. } => {
Expand Down
96 changes: 82 additions & 14 deletions crates/xgen-cli/tests/interactive_repl.rs
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,12 @@ fn changed_safe_environment_is_named_and_blocks_resume_until_restored() {
provider_response(
&json!({"formatVersion":1,"kind":"completion_candidate","steps":[],"summary":""}),
),
provider_response(
&json!({"formatVersion":1,"kind":"completion_candidate","steps":[],"summary":""}),
),
provider_response(
&json!({"formatVersion":1,"kind":"completion_candidate","steps":[],"summary":""}),
),
completion_response(),
]);
let command = |value| {
Expand Down Expand Up @@ -345,7 +351,7 @@ fn rejected_final_response_resumes_after_restart_without_replaying_writes_or_fai
&json!({"executable": "git", "args": ["--definitely-invalid-option"],
"cwd": ".", "env": {}, "timeoutMs": 30000, "maxOutputBytes": 32768}),
));
responses.push(provider_response(&invalid));
responses.extend(rejected_responses(&invalid));
responses.push(provider_response(&json!({"formatVersion": 1,
"kind": "completion_candidate", "steps": [], "summary": "Files saved; command failed."})));
let server = SequentialServer::with_responses(responses);
Expand Down Expand Up @@ -402,11 +408,11 @@ fn rejected_final_response_resumes_after_restart_without_replaying_writes_or_fai
for name in ["first.txt", "second.txt"] {
assert_eq!(fs::read_to_string(workspace.join(name)).unwrap(), name);
}
let requests: Vec<_> = (0..5)
let requests: Vec<_> = (0..7)
.map(|_| server.requests.recv_timeout(TEST_TIMEOUT).unwrap())
.collect();
server.handle.join().unwrap();
let context = planning_context(&requests[4]);
let context = planning_context(&requests[6]);
let outputs = context["toolOutputs"].as_array().unwrap();
assert_eq!(outputs.len(), 3);
let process = outputs
Expand All @@ -420,9 +426,10 @@ fn rejected_final_response_resumes_after_restart_without_replaying_writes_or_fai
#[test]
fn headless_run_keeps_the_legacy_contract_and_rejects_conversation_proposals() {
let fixture = tempdir().unwrap();
let server = SequentialServer::with_responses(vec![provider_response(&json!({
// The driver discards up to MAX_REJECTED_RESPONSE_RETRIES invalid responses (ADR-0061).
let server = SequentialServer::with_responses(rejected_responses(&json!({
"formatVersion": 1, "kind": "response_candidate", "steps": [], "summary": "not a task completion"
}))]);
})));
let output = Command::new(env!("CARGO_BIN_EXE_xgen"))
.args([
"run",
Expand Down Expand Up @@ -454,6 +461,54 @@ fn headless_run_keeps_the_legacy_contract_and_rejects_conversation_proposals() {
);
}

#[test]
fn one_invalid_response_is_retried_and_the_run_completes_without_resume() {
let fixture = tempdir().unwrap();
fs::write(fixture.path().join("README.md"), "retry fixture\n").unwrap();
let server = SequentialServer::with_responses(vec![
plan_response(
"read",
"Read README",
"xgeny.fs/read-text",
&json!({"path":"README.md"}),
),
provider_response(
&json!({"formatVersion":1,"kind":"completion_candidate","steps":[],"summary":""}),
),
completion_response(),
]);
let output = Command::new(env!("CARGO_BIN_EXE_xgen"))
.args([
"run",
"Read README",
"--allow-dir",
".",
"--allow-remote-model-egress",
"--allow-read",
])
.current_dir(fixture.path())
.env("XGEN_STATE_HOME", fixture.path().join("state"))
.env("XGEN_CONFIG_HOME", fixture.path().join("config"))
.env("XGEN_OPENAI_BASE_URL", &server.base_url)
.env("XGEN_OPENAI_MODEL", MODEL)
.env("XGEN_OPENAI_TOKENIZER", TOKENIZER)
.env("XGEN_OPENAI_RESPONSE_FORMAT", "json_object")
.env_remove("XGEN_OPENAI_API_KEY")
.output()
.unwrap();
assert_eq!(output.status.code(), Some(0), "{}", stderr(&output));
// plan, discarded invalid completion, accepted completion
for _ in 0..3 {
server.requests.recv_timeout(TEST_TIMEOUT).unwrap();
}
server.handle.join().unwrap();
assert!(
!stderr(&output).contains("model_rejected"),
"{}",
stderr(&output)
);
}

#[test]
fn trust_flag_runs_a_tool_goal_without_any_approval_prompt() {
let fixture = tempdir().unwrap();
Expand Down Expand Up @@ -760,6 +815,13 @@ fn completion_response() -> Vec<u8> {
}))
}

/// One invalid model response per call the driver makes before it stops the Run.
fn rejected_responses(content: &Value) -> Vec<Vec<u8>> {
(0..=xgen_cli::MAX_REJECTED_RESPONSE_RETRIES)
.map(|_| provider_response(content))
.collect()
}

fn provider_response(content: &Value) -> Vec<u8> {
serde_json::to_vec(&json!({
"id": "repl-test-response",
Expand Down Expand Up @@ -793,9 +855,15 @@ fn usage_counts_rejected_responses_and_survives_resume_without_recounting_replay
provider_response(
&json!({"formatVersion":1,"kind":"completion_candidate","steps":[],"summary":""}),
),
provider_response(
&json!({"formatVersion":1,"kind":"completion_candidate","steps":[],"summary":""}),
),
provider_response(
&json!({"formatVersion":1,"kind":"completion_candidate","steps":[],"summary":""}),
),
completion_response(),
];
for (index, response) in responses.iter_mut().take(2).enumerate() {
for (index, response) in responses.iter_mut().take(4).enumerate() {
let mut envelope: Value = serde_json::from_slice(response).unwrap();
let input = if index == 0 { 100 } else { 200 };
let output = if index == 0 { 20 } else { 30 };
Expand Down Expand Up @@ -825,7 +893,7 @@ fn usage_counts_rejected_responses_and_survives_resume_without_recounting_replay
let run_id = extract_run_id(&stderr(&initial));
let stdout = String::from_utf8_lossy(&initial.stdout);
assert!(
stdout.contains("usage: calls=2/2 input=300 output=50 cached=180 partial=false"),
stdout.contains("usage: calls=4/4 input=700 output=110 cached=420 partial=false"),
"{stdout}"
);
assert!(stdout.contains("model_rejected.planner_invalid_response"));
Expand All @@ -848,10 +916,10 @@ fn usage_counts_rejected_responses_and_survives_resume_without_recounting_replay
let stdout = String::from_utf8_lossy(&resumed.stdout);
assert!(stdout.contains(COMPLETION));
assert!(
stdout.contains("usage: calls=3/3 input=300 output=50 cached=unknown partial=true"),
stdout.contains("usage: calls=5/5 input=700 output=110 cached=unknown partial=true"),
"{stdout}"
);
for _ in 0..3 {
for _ in 0..5 {
server.requests.recv_timeout(TEST_TIMEOUT).unwrap();
}
server.handle.join().unwrap();
Expand All @@ -868,11 +936,11 @@ fn usage_counts_rejected_responses_and_survives_resume_without_recounting_replay
.unwrap();
assert!(offline.status.success(), "{}", stderr(&offline));
let report: Value = serde_json::from_slice(&offline.stdout).unwrap();
assert_eq!(report["observedCalls"], 3);
assert_eq!(report["callsWithTokenUsage"], 2);
assert_eq!(report["tokenSubtotal"]["totalTokens"], 350);
assert_eq!(report["observedCalls"], 5);
assert_eq!(report["callsWithTokenUsage"], 4);
assert_eq!(report["tokenSubtotal"]["totalTokens"], 810);
assert_eq!(report["calls"][1]["outcome"], "response_rejected");
assert_eq!(report["calls"][2]["usage"], Value::Null);
assert_eq!(report["calls"][4]["usage"], Value::Null);
let priced = Command::new(env!("CARGO_BIN_EXE_xgen"))
.args([
"usage",
Expand All @@ -892,7 +960,7 @@ fn usage_counts_rejected_responses_and_survives_resume_without_recounting_replay
let priced: Value = serde_json::from_slice(&priced.stdout).unwrap();
assert_eq!(
priced["costEstimate"]["estimatedUsdSubtotal"],
"0.000238000"
"0.000542000"
);
assert_eq!(priced["costEstimate"]["partial"], true);
assert_eq!(
Expand Down
41 changes: 28 additions & 13 deletions crates/xgen-cli/tests/workspace_discovery.rs
Original file line number Diff line number Diff line change
Expand Up @@ -59,12 +59,19 @@ fn invocation_diagnostics_distinguish_schema_and_resource_failures_without_value
let state_root = fixture.path().join("state");
let workspace = fixture.path().join("workspace");
fs::create_dir(&workspace).unwrap();
let server = SequentialServer::spawn_responses(vec![plan_response(
"write",
"Write fixture",
"xgeny.fs/write-atomic",
&arguments,
)]);
// Each rejected proposal is retried up to MAX_REJECTED_RESPONSE_RETRIES times (ADR-0061).
let server = SequentialServer::spawn_responses(
(0..=xgen_cli::MAX_REJECTED_RESPONSE_RETRIES)
.map(|_| {
plan_response(
"write",
"Write fixture",
"xgeny.fs/write-atomic",
&arguments,
)
})
.collect(),
);
let output = bounded_output(xgen(&state_root).args([
"run",
"--workspace",
Expand Down Expand Up @@ -97,7 +104,9 @@ fn invocation_diagnostics_distinguish_schema_and_resource_failures_without_value
let db = state_root.join("runs").join(run_id).join("run.sqlite3");
let store = SqliteRunStore::open_existing(db).unwrap();
assert!(store.load_execution_receipts().unwrap().is_empty());
server.requests.recv_timeout(TEST_TIMEOUT).unwrap();
for _ in 0..=xgen_cli::MAX_REJECTED_RESPONSE_RETRIES {
server.requests.recv_timeout(TEST_TIMEOUT).unwrap();
}
server.handle.join().unwrap();
assert!(server.requests.try_recv().is_err());
}
Expand Down Expand Up @@ -1828,12 +1837,18 @@ fn tool_discovery_rejects_unknown_collections_and_candidate_execution() {
let runtime = fixture.path().join("runtime");
fs::create_dir(&workspace).unwrap();
discovery_runtime(&state, &runtime);
let server = SequentialServer::spawn_responses(vec![plan_response(
"invalid",
"Attempt unavailable operation",
capability_id,
&input,
)]);
let server = SequentialServer::spawn_responses(
(0..=xgen_cli::MAX_REJECTED_RESPONSE_RETRIES)
.map(|_| {
plan_response(
"invalid",
"Attempt unavailable operation",
capability_id,
&input,
)
})
.collect(),
);
let result = bounded_output(xgen(&state).env("PATH", runtime).args([
"run",
"--workspace",
Expand Down
54 changes: 54 additions & 0 deletions docs/adr/0061-rejected-response-retry.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
# ADR-0061: 거절된 모델 응답의 제한된 재시도

- 상태: Accepted
- 기준일: 2026-10-26
- 변경 분류: 범용 driver 정책. 특정 모델·작업·업무 규칙 없음.
- 기존 경계: ADR-0038(invocation 진단), ADR-0057, receipt completion check
- protocol·SQLite schema·journal 변경: 없음. 재시도는 기존 model call 예약·settlement를 그대로 쓴다.

## 문제

모델 응답 하나가 거절되면 Run이 바로 멈췄다. 거절은 두 갈래다.

- `model_rejected.planner_invalid_response`: 응답이 형식 계약이나 completion check를 통과하지 못했다.
- `proposal_rejected.*`: Core가 제안을 받아들이지 않았다(scope 밖 인자, 스키마 위반, 없는 capability).

두 경우 모두 거절된 call은 journal에 settle되어 있어서 `/resume`하면 새 call을 예약해 이어 간다.
사용자가 손으로 `/resume`을 입력해야 했을 뿐이다.

Claude Code와 같은 작업 6개를 비교한 실험(devhub-social 사본, deepseek flash, `--trust`)에서 rename
작업(137개 참조, 17개 파일)이 2회 모두 실패했다. 2회 모두 파일 4개를 고친 뒤 응답 하나가
`planner_invalid_response`로 거절되어 멈췄다. 전체 934개 call 가운데 거절된 응답은 3개였고 그중 2개가
이 작업에서 나왔다. 응답 하나가 틀렸다고 이미 진행한 작업을 버리기에는 거절이 드물다.

## 결정

1. `RunDriver`는 거절된 응답을 연속 `MAX_REJECTED_RESPONSE_RETRIES`(2)번까지 버리고 다음 tick을 진행한다.
대상은 `PlannerUnavailable(InvalidResponse)`와 `ProposalRejected`다.
2. 제안이 받아들여지면(`PlanAccepted`) 연속 횟수를 0으로 되돌린다. 세 번 연속 거절되면 지금처럼 Run을
멈추고 같은 reason을 보고한다.
3. 재시도마다 새 model call이다. journal에 예약·settle되고 Run의 model call 예산(turn 수의 2배)과
usage에 그대로 잡힌다. 별도 예산을 두지 않는다.
4. Timeout과 transport 실패는 재시도하지 않는다. 결과가 불확정이라 기존처럼 recovery 경계로 남긴다.
provider 한도·4xx·rate limit, 그리고 예산 소진 거절(`tool_call_budget_exhausted`,
`planned_step_budget_exceeded`)도 다시 보내도 결과가 같으므로 재시도하지 않는다.
5. 재시도할 때 거절 이유를 모델 context에 넣지 않는다. 지금 `/resume`과 같은 재표본이다. 이유를 넣으려면
planning context 계약을 바꿔야 하므로 별도 결정으로 미룬다.

`xgen run`(headless)도 같은 driver를 쓰므로 거절 exit code 20은 세 번 연속 거절된 뒤에 나온다.

## 대안

- **재시도 없이 사용자에게 `/resume` 안내:** 지금 동작이다. 거절이 드문데 사람이 매번 개입해야 한다.
- **무제한 재시도:** 같은 실수를 반복하는 모델이 예산을 다 쓸 때까지 멈추지 않는다.
- **거절 이유를 context에 넣고 재시도:** 수정 가능성은 높지만 context 계약 변경이다. 재표본만으로 충분한지
먼저 확인한다.

## 검증

- 단위·e2e: 잘못된 응답 하나 뒤의 정상 completion으로 headless Run이 `/resume` 없이 완료된다. 거절·resume
경로를 검사하던 기존 테스트는 잘못된 응답을 3번 주어 그대로 거절 경로를 검사한다. usage 테스트는
재시도 call도 집계됨을 확인한다.
- 실사용: 같은 rename 작업을 새 binary로 2회 실행했다. 2회 모두 거절된 응답 2개를 버리고 이어 가서
정상 제안 24개를 처리했다(call 26개). 더는 거절로 멈추지 않았고, 24 turn 예산을 다 써서 멈췄다
(참조 137개 중 86개 남음). 예산은 별도 문제다.
Loading