improvement(sandbox): pause idle chat workbenches instead of keeping them running (#9452)

* improvement(sandbox): pause idle chat workbenches instead of keeping them running

- Session sandboxes now pause after a 5-minute idle lease (was: kept running for
  20 minutes, then killed). Resume keeps files, packages and processes and takes
  well under a second; paused sandboxes are not billed and do not count toward
  the provider's concurrency limit.
- A new cleanup-session-sandboxes cron kills workbenches paused for more than 15
  minutes, so a workbench is still discarded about 20 minutes after its last use.
  Each kill runs under the session lock and re-checks the sandbox's own info,
  so a just-resumed workbench is never killed.
- Streamed workbench writes and upload snapshots lease their full transfer
  budget so a machine cannot pause mid-transfer.
- E2B Mothership sandboxes are priced at their template's 2 vCPU / 1 GiB
  instead of the 4 GiB Function profile.

* chore(helm): bump chart version for the session sandbox cleanup cron

* fix(sandbox): bound the session sandbox sweep by a deadline, not a batch size

Author: waleedlatif1

apps/docs/content/docs/platform/self-hosting/background-jobs.mdx

@@ -188,6 +188,7 @@ Create a workflow with a Schedule trigger set to every minute, deploy it, and wa
 | Reconcile billing seats | `/api/cron/reconcile-billing-seats` | `0 * * * *` | Billing only — safe to disable when self-hosted |
 | Reconcile inbox entitlement | `/api/cron/reconcile-inbox-entitlement` | `0 3 * * *` | Inbox access reconciliation |
 | Cleanup sandbox images | `/api/cron/cleanup-sandbox-images` | `30 4 * * *` | Reclaims sandbox images |
+| Cleanup session sandboxes | `/api/cron/cleanup-session-sandboxes` | `*/5 * * * *` | Kills chat workbenches paused past their retention window |
 | Idempotency key cleanup | `/api/webhooks/cleanup/idempotency` | `0 5 * * *` | Deletes deduplication records older than seven days |
 
 Subscription renewal covers Microsoft Teams chat triggers. Their Graph subscriptions last at most three days; rich notifications have a shorter limit. Gmail, Outlook, Drive, Calendar, and Sheets use the per-minute polling jobs instead. See [Microsoft Graph subscription lifetimes](https://learn.microsoft.com/en-us/graph/change-notifications-overview). SCIM reconciliation calls its authenticated endpoint directly each hour; it does not require Trigger.dev.

apps/sim/app/api/cron/cleanup-session-sandboxes/route.ts

@@ -0,0 +1,33 @@
+import { createLogger } from '@sim/logger'
+import { getErrorMessage } from '@sim/utils/errors'
+import { type NextRequest, NextResponse } from 'next/server'
+import { verifyCronAuth } from '@/lib/auth/internal'
+import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
+import { reapPausedSessionSandboxes } from '@/lib/execution/remote-sandbox/session'
+
+export const dynamic = 'force-dynamic'
+export const maxDuration = 60
+
+const logger = createLogger('CleanupSessionSandboxesAPI')
+
+/**
+ * Retention sweep for paused chat workbenches. An idle workbench pauses instead of dying and the
+ * provider keeps it indefinitely, so this run is what bounds how long its files outlive the chat's
+ * last use. A no-op on providers without session sandboxes.
+ */
+export const GET = withRouteHandler(async (request: NextRequest) => {
+  const authError = verifyCronAuth(request, 'session sandbox cleanup')
+  if (authError) return authError
+
+  try {
+    const result = await reapPausedSessionSandboxes()
+    logger.info('Swept paused session sandboxes', result)
+    return NextResponse.json({ success: true, ...result })
+  } catch (error) {
+    logger.error('Failed to sweep paused session sandboxes', { error })
+    return NextResponse.json(
+      { error: getErrorMessage(error, 'Failed to sweep paused session sandboxes') },
+      { status: 500 }
+    )
+  }
+})

apps/sim/lib/billing/sandbox-pricing.test.ts

@@ -17,30 +17,36 @@ describe('sandbox pricing', () => {
     ['e2b', 0.1656],
     ['daytona', 0.16668],
   ] as const)('prices one hour of the Function profile on %s', (provider, expected) => {
-    const pricing = createSandboxPricing(provider, 1)
+    const pricing = createSandboxPricing(provider, 'code', 1)
 
     expect(priceSandboxUsage(pricing, 3_600_000, 3_600_000).rawCost).toBeCloseTo(expected, 8)
   })
 
+  it('prices one hour of the Mothership workbench on its template resources, not the Function profile', () => {
+    const pricing = createSandboxPricing('e2b', 'mothership', 1)
+
+    expect(priceSandboxUsage(pricing, 3_600_000, 3_600_000).rawCost).toBeCloseTo(0.117, 8)
+  })
+
   it('applies the multiplier once and rounds the final cost to eight decimals', () => {
-    const pricing = createSandboxPricing('e2b', 1.75)
+    const pricing = createSandboxPricing('e2b', 'code', 1.75)
 
     expect(priceSandboxUsage(pricing, 1234, 10_000).billedCost).toBe(0.00009934)
   })
 
   it('caps duration at the provider lifetime', () => {
-    const pricing = createSandboxPricing('daytona', 1)
+    const pricing = createSandboxPricing('daytona', 'code', 1)
 
     expect(priceSandboxUsage(pricing, 90_000, 60_000).durationMs).toBe(60_000)
   })
 
   it('allows a zero multiplier and rejects invalid multipliers', () => {
-    const freePricing = createSandboxPricing('e2b', 0)
+    const freePricing = createSandboxPricing('e2b', 'code', 0)
 
     expect(priceSandboxUsage(freePricing, 1000, 1000).billedCost).toBe(0)
-    expect(() => createSandboxPricing('e2b', -1)).toThrow('finite nonnegative')
-    expect(() => createSandboxPricing('e2b', Number.NaN)).toThrow('finite nonnegative')
-    expect(() => createSandboxPricing('e2b', Number.POSITIVE_INFINITY)).toThrow(
+    expect(() => createSandboxPricing('e2b', 'code', -1)).toThrow('finite nonnegative')
+    expect(() => createSandboxPricing('e2b', 'code', Number.NaN)).toThrow('finite nonnegative')
+    expect(() => createSandboxPricing('e2b', 'code', Number.POSITIVE_INFINITY)).toThrow(
       'finite nonnegative'
     )
   })
@@ -49,7 +55,7 @@ describe('sandbox pricing', () => {
     it('coerces the string COST_MULTIPLIER that process.env delivers', () => {
       setEnv({ COST_MULTIPLIER: '1.1' })
 
-      const pricing = createSandboxPricing('e2b')
+      const pricing = createSandboxPricing('e2b', 'code')
 
       expect(pricing.multiplier).toBe(1.1)
       expect(priceSandboxUsage(pricing, 1000, 1000).billedCost).toBeCloseTo(0.0000506, 8)
@@ -58,18 +64,18 @@ describe('sandbox pricing', () => {
     it('falls back to 1 when COST_MULTIPLIER is unset', () => {
       setEnv({ COST_MULTIPLIER: undefined })
 
-      expect(createSandboxPricing('daytona').multiplier).toBe(1)
+      expect(createSandboxPricing('daytona', 'code').multiplier).toBe(1)
     })
 
     it('falls back to 1 instead of throwing when COST_MULTIPLIER is not a nonnegative number', () => {
       setEnv({ COST_MULTIPLIER: 'abc' })
-      expect(createSandboxPricing('e2b').multiplier).toBe(1)
+      expect(createSandboxPricing('e2b', 'code').multiplier).toBe(1)
 
       setEnv({ COST_MULTIPLIER: '-2' })
-      expect(createSandboxPricing('e2b').multiplier).toBe(1)
+      expect(createSandboxPricing('e2b', 'code').multiplier).toBe(1)
 
       setEnv({ COST_MULTIPLIER: '   ' })
-      expect(createSandboxPricing('e2b').multiplier).toBe(1)
+      expect(createSandboxPricing('e2b', 'code').multiplier).toBe(1)
     })
   })
 })

apps/sim/lib/billing/sandbox-pricing.ts

@@ -4,7 +4,7 @@ import {
   FUNCTION_SANDBOX_CPU_COUNT,
   FUNCTION_SANDBOX_MEMORY_GB,
 } from '@/lib/execution/remote-sandbox/function-resources'
-import type { SandboxProviderId } from '@/lib/execution/remote-sandbox/types'
+import type { SandboxKind, SandboxProviderId } from '@/lib/execution/remote-sandbox/types'
 
 const E2B_CPU_USD_PER_VCPU_SECOND = 0.000014
 const E2B_MEMORY_USD_PER_GIB_SECOND = 0.0000045
@@ -37,16 +37,23 @@ export interface PricedSandboxUsage {
   billedCost: number
 }
 
-const PRICING_BY_PROVIDER: Record<
-  SandboxProviderId,
-  Pick<SandboxPricing, 'resources' | 'rates'>
-> = {
+interface ProviderPricing extends Pick<SandboxPricing, 'resources' | 'rates'> {
+  /** Kinds whose image is provisioned differently from the Function profile. */
+  resourcesByKind?: Partial<Record<SandboxKind, SandboxPricing['resources']>>
+}
+
+const PRICING_BY_PROVIDER: Record<SandboxProviderId, ProviderPricing> = {
   e2b: {
     resources: {
       vcpu: FUNCTION_SANDBOX_CPU_COUNT,
       memoryGiB: FUNCTION_SANDBOX_MEMORY_GB,
       diskGiB: 0,
     },
+    resourcesByKind: {
+      // The Mothership template is built without explicit resources, so E2B runs it on its
+      // default 2 vCPU and 1 GiB.
+      mothership: { vcpu: 2, memoryGiB: 1, diskGiB: 0 },
+    },
     rates: {
       cpuUsdPerVcpuSecond: E2B_CPU_USD_PER_VCPU_SECOND,
       memoryUsdPerGiBSecond: E2B_MEMORY_USD_PER_GIB_SECOND,
@@ -69,16 +76,18 @@ const PRICING_BY_PROVIDER: Record<
 
 export function createSandboxPricing(
   provider: SandboxProviderId,
+  kind: SandboxKind,
   multiplier = getCostMultiplier()
 ): SandboxPricing {
   if (!Number.isFinite(multiplier) || multiplier < 0) {
     throw new Error('Sandbox pricing multiplier must be a finite nonnegative number')
   }
   const pricing = PRICING_BY_PROVIDER[provider]
+  const resources = pricing.resourcesByKind?.[kind] ?? pricing.resources
   return {
     provider,
     multiplier,
-    resources: { ...pricing.resources },
+    resources: { ...resources },
     rates: { ...pricing.rates },
   }
 }

apps/sim/lib/execution/remote-sandbox/billing.live.test.ts

@@ -28,7 +28,7 @@ describe.skipIf(!smokeEnabled)('sandbox billing smoke', () => {
   it(
     'bills the sandbox lifetime at the provider rate',
     async () => {
-      const pricing = createSandboxPricing(resolveProvider().id)
+      const pricing = createSandboxPricing(resolveProvider().id, 'code')
       const usdPerSecond =
         pricing.resources.vcpu * pricing.rates.cpuUsdPerVcpuSecond +
         pricing.resources.memoryGiB * pricing.rates.memoryUsdPerGiBSecond +

apps/sim/lib/execution/remote-sandbox/create.ts

@@ -34,7 +34,7 @@ export async function createSandbox(
   if (meterUsage && effectiveLifetimeMs === undefined) {
     throw new Error('Metered sandbox execution requires a provider lifetime')
   }
-  const pricing = meterUsage ? createSandboxPricing(provider.id) : undefined
+  const pricing = meterUsage ? createSandboxPricing(provider.id, kind) : undefined
   let startedAtMs = Date.now()
   const providerOptions = {
     ...options,

apps/sim/lib/execution/remote-sandbox/e2b.ts

@@ -1,5 +1,10 @@
 import { createHash } from 'node:crypto'
-import type { Sandbox as E2BSandbox, Template as E2BTemplate } from '@e2b/code-interpreter'
+import type {
+  Sandbox as E2BSandbox,
+  Template as E2BTemplate,
+  SandboxInfo,
+  SandboxPaginator,
+} from '@e2b/code-interpreter'
 import { createLogger } from '@sim/logger'
 import { getErrorMessage } from '@sim/utils/errors'
 import { generateId, generateShortId } from '@sim/utils/id'
@@ -65,6 +70,7 @@ import {
 } from '@/lib/execution/remote-sandbox/spec'
 import type {
   CreateSandboxOptions,
+  PausedSessionSandbox,
   RunCommandOptions,
   SandboxCodeResult,
   SandboxCommandResult,
@@ -307,6 +313,12 @@ const E2B_SESSION_METADATA_KEY = 'simSessionKey'
 const E2B_SESSION_OWNERSHIP_KEY = 'simSessionOwnership'
 const E2B_SESSION_OWNERSHIP_VERSION = 'tracked-v1'
 
+async function listAllSandboxes(paginator: SandboxPaginator): Promise<SandboxInfo[]> {
+  const sandboxes = await paginator.nextItems()
+  while (paginator.hasNext) sandboxes.push(...(await paginator.nextItems()))
+  return sandboxes
+}
+
 async function stopSessionCommand(
   sandbox: E2BSandbox,
   processId: string,
@@ -1095,6 +1107,9 @@ export const e2bProvider: SandboxProvider = {
               [E2B_SESSION_METADATA_KEY]: options.sessionKey,
               [E2B_SESSION_OWNERSHIP_KEY]: E2B_SESSION_OWNERSHIP_VERSION,
             },
+            // An idle workbench pauses instead of dying. Only a locked reconnect may resume it,
+            // which is what lets the retention sweep kill it safely.
+            lifecycle: { onTimeout: 'pause' as const, autoResume: false },
           }
         : {}),
     }
@@ -1122,12 +1137,12 @@ export const e2bProvider: SandboxProvider = {
     const apiKey = env.E2B_API_KEY
     if (!apiKey) throw new Error('E2B_API_KEY is required when E2B is enabled')
     const { Sandbox } = await import('@e2b/code-interpreter')
-    const paginator = Sandbox.list({
-      apiKey,
-      query: { metadata: { [E2B_SESSION_METADATA_KEY]: key }, state: ['running', 'paused'] },
-    })
-    const candidates = await paginator.nextItems()
-    while (paginator.hasNext) candidates.push(...(await paginator.nextItems()))
+    const candidates = await listAllSandboxes(
+      Sandbox.list({
+        apiKey,
+        query: { metadata: { [E2B_SESSION_METADATA_KEY]: key }, state: ['running', 'paused'] },
+      })
+    )
     if (candidates.length === 0) return null
     candidates.sort((a, b) => a.startedAt.getTime() - b.startedAt.getTime())
     const candidate = candidates[0]
@@ -1172,4 +1187,44 @@ export const e2bProvider: SandboxProvider = {
       options.lifetimeMs === undefined ? undefined : requestedAtMs + timeoutMs
     )
   },
+
+  async listPausedSessionSandboxes(): Promise<PausedSessionSandbox[]> {
+    const apiKey = env.E2B_API_KEY
+    if (!apiKey) return []
+    const { Sandbox } = await import('@e2b/code-interpreter')
+    const paused = await listAllSandboxes(Sandbox.list({ apiKey, query: { state: ['paused'] } }))
+    return paused.flatMap((info) => {
+      const sessionKey = info.metadata[E2B_SESSION_METADATA_KEY]
+      // A paused sandbox reports the moment it paused as its end.
+      return sessionKey
+        ? [{ sandboxId: info.sandboxId, sessionKey, pausedAtMs: info.endAt.getTime() }]
+        : []
+    })
+  },
+
+  async killPausedSessionSandbox(
+    sandbox: PausedSessionSandbox,
+    pausedBeforeMs: number
+  ): Promise<boolean> {
+    const apiKey = env.E2B_API_KEY
+    if (!apiKey) throw new Error('E2B_API_KEY is required when E2B is enabled')
+    const { Sandbox, NotFoundError } = await import('@e2b/code-interpreter')
+    const requestOptions = { apiKey, requestTimeoutMs: 5_000 }
+    // The list can trail a resume, so only the sandbox's own info decides.
+    const info = await Sandbox.getInfo(sandbox.sandboxId, requestOptions).catch(
+      (error: unknown) => {
+        if (error instanceof NotFoundError) return null
+        throw error
+      }
+    )
+    if (
+      !info ||
+      info.state !== 'paused' ||
+      info.metadata[E2B_SESSION_METADATA_KEY] !== sandbox.sessionKey ||
+      info.endAt.getTime() > pausedBeforeMs
+    ) {
+      return false
+    }
+    return Sandbox.kill(sandbox.sandboxId, requestOptions)
+  },
 }

apps/sim/lib/execution/remote-sandbox/index.ts

@@ -105,8 +105,9 @@ interface SandboxLease {
  *
  * A session lease never binds the abort signal to teardown — cancelling one
  * execution must not destroy state the next turn builds on — and is never
- * metered, because session sandboxes are server-owned rather than billed to
- * workspace compute. Providers without session support degrade to one-shot.
+ * metered as workspace compute: the Mothership tool bills each call's execution
+ * time, while the idle time between calls is server-owned. Providers without
+ * session support degrade to one-shot.
  */
 async function leaseSandbox(
   kind: SandboxKind,

apps/sim/lib/execution/remote-sandbox/pi-sandbox-billing.live.test.ts

@@ -29,7 +29,7 @@ describe.skipIf(!smokeEnabled)('pi sandbox billing smoke', () => {
   it(
     'bills the session a Pi sandbox was held for',
     async () => {
-      const pricing = createSandboxPricing(resolveProvider().id)
+      const pricing = createSandboxPricing(resolveProvider().id, 'pi')
       const usdPerBilledSecond =
         (pricing.resources.vcpu * pricing.rates.cpuUsdPerVcpuSecond +
           pricing.resources.memoryGiB * pricing.rates.memoryUsdPerGiBSecond +

apps/sim/lib/execution/remote-sandbox/session-file-snapshot.test.ts

@@ -35,6 +35,7 @@ vi.mock('@/lib/execution/remote-sandbox/session-lock', () => ({
   ) => execute(signal),
 }))
 
+import { SESSION_SANDBOX_IDLE_MS } from '@/lib/execution/remote-sandbox/session'
 import { openSessionFileSnapshot } from '@/lib/execution/remote-sandbox/session-file-snapshot'
 import { fileReadFailure } from '@/lib/mothership/agent-cli/engines/file-view'
 
@@ -104,7 +105,6 @@ it('bounds the remote copy itself, not just the eventual response stream', async
   expect(mocks.read).not.toHaveBeenCalled()
 })
 it('asks the reconnect for the idle lease, which can outlast the runtime cap', async () => {
-  const IDLE_MS = 20 * 60_000
   const capAtMs = Date.now() + 60_000
   let leaseEndAtMs = capAtMs
   const machine = await mocks.find()
@@ -124,7 +124,7 @@ it('asks the reconnect for the idle lease, which can outlast the runtime cap', a
     maxBytes: 100,
   })
   await snapshot.dispose()
-  expect(leaseEndAtMs).toBeGreaterThanOrEqual(Date.now() + IDLE_MS - 1000)
+  expect(leaseEndAtMs).toBeGreaterThanOrEqual(Date.now() + SESSION_SANDBOX_IDLE_MS - 1000)
 })
 it('never creates a replacement machine for a missing read', async () => {
   mocks.find.mockResolvedValue(null)