Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,18 @@

## Unreleased

### After-Response Tasks (#476)
### After-Response Tasks (#476, #478)

- Cloudflare runs gated `waitUntil` tasks after terminal response egress, with fresh
outbound scopes and effective budgets capped at 30 seconds. Workers lifetime limits
still apply; tasks have no persistence or automatic retries.

- Add `RequestContext::after_response(budget, task)` and the adapter-facing
`edgezero_core::after_response::{AfterResponseScheduler, AfterResponseHandle}`. A task starts
after the response attempt ends and is cancelled at its budget. Adapters without support return
`501 Not Implemented` at registration.
- Add the `after-response-tasks` capability: Axum and Fastly `BoundedCooperative`,
Cloudflare and Spin `Unsupported` (tracked in #478 and #483).
Cloudflare `BoundedCooperative` (#478), and Spin `Unsupported` (tracked in #483).
- Axum production, `run_app` and `AxumDevServer::run` shutdown now waits for registered after-response tasks within the
existing shutdown grace, after connections drain. Tasks still pending at the grace make the
process exit non-zero with `grace exhausted`.
Expand Down
230 changes: 230 additions & 0 deletions crates/edgezero-adapter-cloudflare/src/after_response.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,230 @@
//! Request-local after-response release and bounded Workers tasks.

#[cfg(all(feature = "cloudflare", target_arch = "wasm32"))]
mod platform {
use super::{Gate, LocalBoxFuture, ReleaseGuard, run_bounded};
use std::sync::Arc;
use std::time::Duration;

use edgezero_core::after_response::{AfterResponseHandle, AfterResponseScheduler};
use edgezero_core::error::EdgeError;
use edgezero_core::time::MonotonicClock;
use worker::Context;

use crate::outbound::native_time;

struct Scheduler {
clock: MonotonicClock,
context: Arc<Context>,
gate: Arc<Gate>,
}

impl AfterResponseScheduler for Scheduler {
fn schedule(
&self,
budget: Duration,
task: LocalBoxFuture<'static, ()>,
) -> Result<(), EdgeError> {
let released = self.gate.register()?;
let clock = self.clock.clone();
self.context.wait_until(async move {
let _released = released.await;
run_bounded(task, budget, native_time(clock)).await;
});
Ok(())
}
}

pub(crate) fn arm(
context: Arc<Context>,
clock: MonotonicClock,
) -> (AfterResponseHandle, ReleaseGuard) {
let release = ReleaseGuard::new();
let handle = AfterResponseHandle::new(Scheduler {
context,
gate: Arc::clone(&release.0),
clock,
});
(handle, release)
}
}

#[cfg(all(feature = "cloudflare", target_arch = "wasm32"))]
pub(crate) use platform::arm;

use std::panic::AssertUnwindSafe;
use std::sync::{Arc, Mutex, PoisonError};
use std::time::Duration;

use edgezero_core::error::EdgeError;
use edgezero_core::outbound::{OutboundScopeOwner, OutboundTime};
use futures::channel::oneshot;
use futures_util::future::{Either, FutureExt as _, LocalBoxFuture, select};

const MAX_BUDGET: Duration = Duration::from_secs(30);

#[derive(Default)]
struct Gate {
waiters: Mutex<Option<Vec<oneshot::Sender<()>>>>,
}

impl Gate {
fn register(&self) -> Result<oneshot::Receiver<()>, EdgeError> {
let mut state = self.waiters.lock().unwrap_or_else(PoisonError::into_inner);
let waiters = state.as_mut().ok_or_else(|| {
EdgeError::internal(anyhow::anyhow!(
"after-response tasks can only be registered while the request is handled"
))
})?;
let (sender, receiver) = oneshot::channel();
waiters.push(sender);
Ok(receiver)
}
}

/// Sole release owner, retained through conversion and the terminal response pump.
pub(crate) struct ReleaseGuard(Arc<Gate>);

impl ReleaseGuard {
fn new() -> Self {
Self(Arc::new(Gate {
waiters: Mutex::new(Some(Vec::new())),
}))
}
}

#[expect(
clippy::missing_trait_methods,
reason = "Drop::pin_drop is unstable and conflicts with drop"
)]
impl Drop for ReleaseGuard {
fn drop(&mut self) {
let waiters = self
.0
.waiters
.lock()
.unwrap_or_else(PoisonError::into_inner)
.take();
for sender in waiters.unwrap_or_default() {
let _closed = sender.send(());
}
}
}

async fn run_bounded(task: LocalBoxFuture<'static, ()>, budget: Duration, time: OutboundTime) {
let deadline = time.clock().deadline_after(budget.min(MAX_BUDGET));
let callback = OutboundScopeOwner::callback();
let result = callback
.run(
AssertUnwindSafe(async move {
match select(time.wait_until(deadline), task).await {
Either::Left((Ok(()), _task)) => {
log::warn!("after-response task exceeded its budget and was cancelled");
}
Either::Left((Err(_error), _task)) => {
log::warn!("after-response task timer failed; the task was cancelled");
}
Either::Right(((), _timer)) => {}
}
})
.catch_unwind(),
)
.await;
if result.is_err() {
log::error!("after-response task panicked");
}
}

#[cfg(test)]
mod tests {
use super::{MAX_BUDGET, ReleaseGuard, run_bounded};
use std::cell::Cell;
use std::rc::Rc;
use std::sync::Arc;
use std::time::Duration;

use edgezero_core::error::EdgeError;
use edgezero_core::outbound::OutboundTime;
use edgezero_core::time::MonotonicClock;
use futures::executor::block_on;
use futures::future::{join_all, pending};
use futures_util::FutureExt as _;

#[test]
fn release_is_request_local_and_closes_registration() {
let first = ReleaseGuard::new();
let second = ReleaseGuard::new();
let retained = Arc::clone(&first.0);
let first_receiver = first.0.register().unwrap();
let second_receiver = first.0.register().unwrap();
let mut other = second.0.register().unwrap();
assert!(other.try_recv().unwrap().is_none());
drop(first);
assert_eq!(block_on(first_receiver), Ok(()));
assert_eq!(block_on(second_receiver), Ok(()));
retained.register().unwrap_err();
assert!(other.try_recv().unwrap().is_none());
drop(second);
assert_eq!(block_on(other), Ok(()));
}

#[test]
fn panicking_task_does_not_prevent_a_sibling_from_running() {
let ran = Rc::new(Cell::new(false));
let observed = Rc::clone(&ran);
let time = || OutboundTime::new(MonotonicClock::default(), |_, _| Box::pin(pending()));
let tasks = [
run_bounded(
async { panic!("task failure") }.boxed_local(),
Duration::from_secs(1),
time(),
),
run_bounded(
async move {
observed.set(true);
}
.boxed_local(),
Duration::from_secs(1),
time(),
),
];
block_on(join_all(tasks));
assert!(ran.get());
}

#[test]
fn huge_budget_is_capped_and_timer_failure_drops_local_task_unpolled() {
struct Probe(Rc<Cell<bool>>);
#[expect(clippy::missing_trait_methods, reason = "Drop::pin_drop is unstable")]
impl Drop for Probe {
fn drop(&mut self) {
self.0.set(true);
}
}
let dropped = Rc::new(Cell::new(false));
let polled = Rc::new(Cell::new(false));
let observed = Rc::clone(&polled);
let probe = Probe(Rc::clone(&dropped));
let clock = MonotonicClock::default();
let start = clock.now();
let time = OutboundTime::new(
MonotonicClock::new(move || start),
move |deadline, paired_clock| {
assert_eq!(deadline.remaining_at(paired_clock.now()), Some(MAX_BUDGET));
Box::pin(async { Err(EdgeError::internal(anyhow::anyhow!("timer failure"))) })
},
);
block_on(run_bounded(
async move {
let _probe = probe;
observed.set(true);
pending::<()>().await;
}
.boxed_local(),
Duration::MAX,
time,
));
assert!(dropped.get());
assert!(!polled.get());
}
}
6 changes: 3 additions & 3 deletions crates/edgezero-adapter-cloudflare/src/cli.rs
Original file line number Diff line number Diff line change
Expand Up @@ -160,8 +160,8 @@ impl Adapter for CloudflareCliAdapter {
| Capability::OutboundFlexiblePhaseBudget
| Capability::OutboundHttp
| Capability::OutboundRedirectFollowing => CapabilitySupport::Native,
Capability::AfterResponseTasks
| Capability::ConfigReadAllocationBounds
Capability::AfterResponseTasks => CapabilitySupport::BoundedCooperative,
Capability::ConfigReadAllocationBounds
| Capability::OutboundCompleteResourceAccounting
| Capability::RawIngressFramingValidation
| Capability::RawIngressHeadLimits
Expand Down Expand Up @@ -1528,7 +1528,7 @@ mod tests {
for (capability, support) in [
(
Capability::AfterResponseTasks,
CapabilitySupport::Unsupported,
CapabilitySupport::BoundedCooperative,
),
(
Capability::ConfigReadAllocationBounds,
Expand Down
3 changes: 3 additions & 0 deletions crates/edgezero-adapter-cloudflare/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@
#[cfg(all(feature = "cli", not(target_arch = "wasm32")))]
pub mod cli;

#[cfg(any(test, all(feature = "cloudflare", target_arch = "wasm32")))]
mod after_response;

// `config_store` compiles on host for its `InMemory` test backend; the
// production `Kv` backend is feature-gated internally.
#[cfg(any(test, all(feature = "cloudflare", target_arch = "wasm32")))]
Expand Down
9 changes: 7 additions & 2 deletions crates/edgezero-adapter-cloudflare/src/request.rs
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ use worker::{
Context, Delay, Env, Error as WorkerError, Method, Request as CfRequest, Response as CfResponse,
};

use crate::after_response;
use crate::config_store::CloudflareConfigStore;
use crate::context::CloudflareRequestContext;
use crate::key_value_store::CloudflareKvStore;
Expand Down Expand Up @@ -598,8 +599,11 @@ pub(crate) async fn dispatch_with_handles(
let request_method =
CoreMethod::from_bytes(req.inner().method().as_bytes()).unwrap_or(CoreMethod::GET);
let context = Arc::new(ctx);
let runtime = CloudflareEgressRuntime::new(signal, Arc::clone(&context));
let head_request = match into_core_request_head(&req, env, context, app.monotonic_clock()) {
let (after_response, release) =
after_response::arm(Arc::clone(&context), app.monotonic_clock());
let runtime =
CloudflareEgressRuntime::new(signal, Arc::clone(&context)).with_after_response(release);
let mut head_request = match into_core_request_head(&req, env, context, app.monotonic_clock()) {
Ok(request) => request,
Err(error) => {
drop(req);
Expand All @@ -608,6 +612,7 @@ pub(crate) async fn dispatch_with_handles(
.map_err(|delivery_error| edge_error_to_worker(&delivery_error));
}
};
head_request.extensions_mut().insert(after_response);
dispatch_ingress_stream_and(
app,
head_request,
Expand Down
Loading
Loading