Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions scripts/core-boundaries/rules/source/required-rules.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -2273,7 +2273,7 @@ export const requiredContentRules = [
{
path: 'src/crates/execution/agent-runtime/tests/agent_session_contracts/scheduled_job_contracts.rs',
reason:
'agent-runtime scheduled-job owner must keep behavior-equivalence contracts for wire shape, retry, coalescing, one-shot, missing-session, and restart recovery semantics',
'agent-runtime scheduled-job owner must keep contracts for wire shape, pending wakeups, failed-trigger consumption, coalescing, one-shot, missing-session, and restart recovery semantics',
patterns: [
{
regex: /\bmanual_trigger_coalesces_existing_pending_run\b/,
Expand All @@ -2296,9 +2296,17 @@ export const requiredContentRules = [
message: 'missing enqueue success one-shot regression',
},
{
regex: /\benqueue_failure_preserves_retry_and_missing_session_disable_semantics\b/,
regex: /\benqueue_failure_consumes_trigger_and_preserves_missing_session_disable_semantics\b/,
message: 'missing enqueue failure regression',
},
{
regex: /\bfailed_trigger_waits_for_the_next_schedule_instead_of_retrying\b/,
message: 'missing failed trigger scheduling regression',
},
{
regex: /\bfailed_one_shot_trigger_stops_until_an_explicit_manual_run\b/,
message: 'missing failed one-shot manual run regression',
},
{
regex: /\brestart_recovery_marks_active_turn_error\b/,
message: 'missing restart recovery regression',
Expand Down
4 changes: 3 additions & 1 deletion scripts/core-boundaries/self-test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -3305,7 +3305,9 @@ export function runManifestParserSelfTest({
'pending_wakeup_prefers_retry_time_when_present',
'disabled_and_config_clear_remove_pending_retry_without_touching_history',
'enqueue_success_sets_active_turn_and_disables_one_shot_next_run',
'enqueue_failure_preserves_retry_and_missing_session_disable_semantics',
'enqueue_failure_consumes_trigger_and_preserves_missing_session_disable_semantics',
'failed_trigger_waits_for_the_next_schedule_instead_of_retrying',
'failed_one_shot_trigger_stops_until_an_explicit_manual_run',
'restart_recovery_marks_active_turn_error',
'serde_shape_preserves_legacy_cron_state_wire_contract',
],
Expand Down
146 changes: 136 additions & 10 deletions src/crates/assembly/core/src/service/cron/service.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,9 @@ use super::schedule::{
};
use super::store::CronJobStore;
use super::types::{
CreateCronJobRequest, CronJob, CronJobPayload, CronJobTarget, CronJobTargetKind, CronJobsFile,
CronLaunchSpec, CronSchedule, CronWorkspaceRef, UpdateCronJobRequest, DEFAULT_RETRY_DELAY_MS,
CreateCronJobRequest, CronJob, CronJobPayload, CronJobRunStatus, CronJobTarget,
CronJobTargetKind, CronJobsFile, CronLaunchSpec, CronSchedule, CronWorkspaceRef,
UpdateCronJobRequest,
};
use super::{CronJobsChangedEvent, CronJobsChangedReason, CRON_JOBS_CHANGED_EVENT};
use crate::agentic::coordination::scheduler::DIALOG_TURN_ID_ALREADY_SETTLED_MESSAGE;
Expand Down Expand Up @@ -702,19 +703,21 @@ impl CronService {
Err(error) => {
let missing_session = matches!(job.target_kind(), CronJobTargetKind::Session)
&& cron_enqueue_error_is_missing_session(&error);
let failure_action = job.state.mark_enqueue_failed(
now_after_submit,
error.clone(),
DEFAULT_RETRY_DELAY_MS,
missing_session,
);
let failure_action =
job.state
.mark_enqueue_failed(now_after_submit, error.clone(), missing_session);
job.updated_at_ms = now_after_submit;

if matches!(
let disable_missing_session = matches!(
failure_action,
ScheduledJobEnqueueFailureAction::DisableMissingSession
) {
);
if disable_missing_session || job.is_one_shot() {
job.enabled = false;
job.state.mark_disabled();
}

if disable_missing_session {
info!(
"Scheduled job auto-disabled (session no longer exists): job_id={}, session_id={}",
job.id,
Expand Down Expand Up @@ -1050,6 +1053,21 @@ fn reconcile_loaded_job(job: &mut CronJob, now_ms: i64) -> OpenBitFunResult<bool
job.updated_at_ms = now_ms;
}

// Older versions kept failed submissions pending and retried every five
// seconds, creating another workspace session on each attempt.
if job.state.last_run_status == Some(CronJobRunStatus::Error)
&& matches!(
(job.state.pending_trigger_at_ms, job.state.last_run_finished_at_ms),
(Some(pending_at), Some(finished_at)) if pending_at <= finished_at
)
{
job.state.clear_pending_trigger();
if job.is_one_shot() {
job.enabled = false;
}
job.updated_at_ms = now_ms;
}

if !job.enabled {
job.state.mark_disabled();
} else if job.state.pending_trigger_at_ms.is_some() {
Expand Down Expand Up @@ -1403,6 +1421,114 @@ mod tests {
));
}

#[test]
fn restart_discards_legacy_failed_retry_and_keeps_the_next_daily_run() {
let scheduled_at_ms = Utc
.with_ymd_and_hms(2026, 11, 15, 22, 0, 0)
.single()
.unwrap()
.timestamp_millis();
let next_run_at_ms = scheduled_at_ms + 86_400_000;
let mut job = sample_job(CronSchedule::Cron {
expr: "0 6 * * *".to_string(),
tz: Some("Asia/Shanghai".to_string()),
});
job.target = CronJobTarget::Workspace {
workspace: job.workspace().clone(),
launch: CronLaunchSpec::default(),
};
let mut legacy = serde_json::to_value(&job).unwrap();
legacy["state"] = serde_json::json!({
"nextRunAtMs": next_run_at_ms,
"pendingTriggerAtMs": scheduled_at_ms,
"retryAtMs": scheduled_at_ms + 5_000,
"lastRunStatus": "error",
"lastError": "Model API unavailable",
"lastRunFinishedAtMs": scheduled_at_ms + 1,
"consecutiveFailures": 400
});
job = serde_json::from_value(legacy).unwrap();

assert!(reconcile_loaded_job(&mut job, scheduled_at_ms + 60_000).unwrap());
assert!(job.enabled);
assert_eq!(job.state.pending_trigger_at_ms, None);
assert_eq!(job.state.retry_at_ms, None);
assert_eq!(next_wakeup_for_job(&job), Some(next_run_at_ms));
assert_eq!(
job.state.last_error.as_deref(),
Some("Model API unavailable")
);
assert_eq!(job.state.last_run_finished_at_ms, Some(scheduled_at_ms + 1));
assert_eq!(job.state.consecutive_failures, 400);
assert_eq!(
compute_next_run_after_ms(&job.schedule, job.created_at_ms, scheduled_at_ms).unwrap(),
Some(next_run_at_ms)
);

let mut restored = serde_json::from_str(&serde_json::to_string(&job).unwrap()).unwrap();
assert!(!reconcile_loaded_job(&mut restored, scheduled_at_ms + 120_000).unwrap());
assert_eq!(restored, job);
}

#[test]
fn restart_disables_legacy_failed_one_shot_without_rescheduling_it() {
let mut job = sample_job(CronSchedule::At {
at: "2026-11-16T06:00:00+08:00".to_string(),
});
job.state = CronJobState {
pending_trigger_at_ms: Some(100),
retry_at_ms: Some(5_100),
last_run_status: Some(CronJobRunStatus::Error),
last_error: Some("Model API unavailable".to_string()),
last_run_finished_at_ms: Some(200),
consecutive_failures: 2,
..Default::default()
};

assert!(reconcile_loaded_job(&mut job, 10_000).unwrap());
assert!(!job.enabled);
assert_eq!(next_wakeup_for_job(&job), None);
assert_eq!(job.state.pending_trigger_at_ms, None);
assert_eq!(job.state.retry_at_ms, None);
assert_eq!(job.state.consecutive_failures, 2);
assert!(!reconcile_loaded_job(&mut job, 20_000).unwrap());
}

#[test]
fn restart_preserves_a_trigger_that_has_not_been_attempted() {
let mut job = sample_job(CronSchedule::Every {
every_ms: 60_000,
anchor_ms: Some(0),
});
job.state.next_run_at_ms = Some(60_000);
job.state.pending_trigger_at_ms = Some(100);

reconcile_loaded_job(&mut job, 1_000).unwrap();

assert!(job.enabled);
assert_eq!(job.state.pending_trigger_at_ms, Some(100));
assert_eq!(job.state.retry_at_ms, Some(1_000));
assert_eq!(next_wakeup_for_job(&job), Some(1_000));
}

#[test]
fn restart_preserves_a_new_manual_trigger_after_an_earlier_failure() {
let mut job = sample_job(CronSchedule::Every {
every_ms: 60_000,
anchor_ms: Some(0),
});
job.state.next_run_at_ms = Some(60_000);
job.state
.mark_enqueue_failed(100, "Model API unavailable".to_string(), false);
job.state.mark_manual_trigger(200);

reconcile_loaded_job(&mut job, 1_000).unwrap();

assert!(job.enabled);
assert_eq!(job.state.pending_trigger_at_ms, Some(200));
assert_eq!(next_wakeup_for_job(&job), Some(1_000));
}

#[tokio::test]
async fn loaded_jobs_are_rewritten_when_they_gain_their_first_anchor() {
let mut job = sample_job(CronSchedule::Every {
Expand Down
17 changes: 6 additions & 11 deletions src/crates/execution/agent-runtime/src/scheduled_job.rs
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ pub enum ScheduledJobTriggerAction {

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ScheduledJobEnqueueFailureAction {
Retry,
SkipTrigger,
DisableMissingSession,
}

Expand Down Expand Up @@ -126,27 +126,22 @@ impl ScheduledJobRuntimeState {
}
}

/// Finish this trigger even when submission failed before a turn started.
/// Model/runtime failures must not create extra runs between scheduled times.
pub fn mark_enqueue_failed(
&mut self,
failed_at_ms: i64,
error: String,
retry_delay_ms: i64,
missing_session: bool,
) -> ScheduledJobEnqueueFailureAction {
self.last_run_status = Some(ScheduledJobRunStatus::Error);
self.last_error = Some(error);
self.last_run_finished_at_ms = Some(failed_at_ms);
self.consecutive_failures = self.consecutive_failures.saturating_add(1);
self.clear_pending_trigger();
self.mark_turn_failed(failed_at_ms, error);

if missing_session {
self.active_turn_id = None;
self.next_run_at_ms = None;
self.pending_trigger_at_ms = None;
self.retry_at_ms = None;
ScheduledJobEnqueueFailureAction::DisableMissingSession
} else {
self.retry_at_ms = Some(failed_at_ms + retry_delay_ms);
ScheduledJobEnqueueFailureAction::Retry
ScheduledJobEnqueueFailureAction::SkipTrigger
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -141,32 +141,34 @@ fn enqueue_success_sets_active_turn_and_disables_one_shot_next_run() {
}

#[test]
fn enqueue_failure_preserves_retry_and_missing_session_disable_semantics() {
let mut retry_state = ScheduledJobRuntimeState {
fn enqueue_failure_consumes_trigger_and_preserves_missing_session_disable_semantics() {
let mut failed_state = ScheduledJobRuntimeState {
next_run_at_ms: Some(60_000),
pending_trigger_at_ms: Some(100),
retry_at_ms: Some(150),
consecutive_failures: 1,
..Default::default()
};

let retry_action = retry_state.mark_enqueue_failed(
200,
"temporary failure".to_string(),
DEFAULT_SCHEDULED_JOB_RETRY_DELAY_MS,
false,
);
let failure_action =
failed_state.mark_enqueue_failed(200, "temporary failure".to_string(), false);

assert_eq!(retry_action, ScheduledJobEnqueueFailureAction::Retry);
assert_eq!(
retry_state.retry_at_ms,
Some(200 + DEFAULT_SCHEDULED_JOB_RETRY_DELAY_MS)
failure_action,
ScheduledJobEnqueueFailureAction::SkipTrigger
);
assert_eq!(retry_state.pending_trigger_at_ms, Some(100));
assert_eq!(retry_state.consecutive_failures, 2);
assert_eq!(failed_state.retry_at_ms, None);
assert_eq!(failed_state.pending_trigger_at_ms, None);
assert_eq!(failed_state.next_run_at_ms, Some(60_000));
assert_eq!(failed_state.consecutive_failures, 2);
assert_eq!(
retry_state.last_run_status,
failed_state.last_run_status,
Some(ScheduledJobRunStatus::Error)
);
assert_eq!(retry_state.last_error.as_deref(), Some("temporary failure"));
assert_eq!(
failed_state.last_error.as_deref(),
Some("temporary failure")
);

let mut missing_session_state = ScheduledJobRuntimeState {
active_turn_id: Some("stale-turn".to_string()),
Expand All @@ -177,12 +179,8 @@ fn enqueue_failure_preserves_retry_and_missing_session_disable_semantics() {
..Default::default()
};

let disable_action = missing_session_state.mark_enqueue_failed(
200,
"session not found".to_string(),
5_000,
true,
);
let disable_action =
missing_session_state.mark_enqueue_failed(200, "session not found".to_string(), true);

assert_eq!(
disable_action,
Expand All @@ -195,6 +193,56 @@ fn enqueue_failure_preserves_retry_and_missing_session_disable_semantics() {
assert_eq!(missing_session_state.consecutive_failures, 3);
}

#[test]
fn failed_trigger_waits_for_the_next_schedule_instead_of_retrying() {
let scheduled_at_ms = 1_000;
let next_run_at_ms = scheduled_at_ms + 86_400_000;
let mut state = ScheduledJobRuntimeState {
next_run_at_ms: Some(scheduled_at_ms),
..Default::default()
};
state.apply_due_scheduled_trigger(scheduled_at_ms, Some(next_run_at_ms));

state.mark_enqueue_failed(
scheduled_at_ms + 1,
"Model API unavailable".to_string(),
false,
);

assert_eq!(state.pending_trigger_at_ms, None);
assert_eq!(state.retry_at_ms, None);
assert_eq!(state.last_run_status, Some(ScheduledJobRunStatus::Error));
assert_eq!(state.last_error.as_deref(), Some("Model API unavailable"));
assert_eq!(state.consecutive_failures, 1);
assert_eq!(state.next_wakeup_at_ms(), Some(next_run_at_ms));
for attempt in 1..=500 {
assert!(
!state.pending_is_due(scheduled_at_ms + attempt * DEFAULT_SCHEDULED_JOB_RETRY_DELAY_MS)
);
}

assert_eq!(
state.apply_due_scheduled_trigger(next_run_at_ms, Some(next_run_at_ms + 86_400_000)),
ScheduledJobTriggerAction::Pending
);
assert!(state.pending_is_due(next_run_at_ms));
}

#[test]
fn failed_one_shot_trigger_stops_until_an_explicit_manual_run() {
let mut state = ScheduledJobRuntimeState {
pending_trigger_at_ms: Some(100),
..Default::default()
};
state.mark_enqueue_failed(200, "Model initialization failed".to_string(), false);

assert_eq!(state.next_wakeup_at_ms(), None);
assert!(!state.pending_is_due(100_000));

state.mark_manual_trigger(100_001);
assert!(state.pending_is_due(100_001));
}

#[test]
fn trigger_delivered_elsewhere_clears_pending_and_stops_retrying() {
let mut state = ScheduledJobRuntimeState {
Expand Down
Loading