Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -954,9 +954,9 @@ function withReplayRequestId(
describe("CodexAdapterV2 process exit", () => {
it.effect.each([
{ exit: "numeric", active: false },
{ exit: "SIGKILL", active: false },
{ exit: "unknown exit status", active: false },
{ exit: "numeric", active: true },
{ exit: "SIGKILL", active: true },
{ exit: "unknown exit status", active: true },
] as const)("retires the shared session after $exit (active turn: $active)", ({ exit, active }) =>
Effect.gen(function* () {
const scenario = `process-exit-${exit}-${active}`;
Expand All @@ -982,7 +982,7 @@ describe("CodexAdapterV2 process exit", () => {
_tag: "Unknown",
module: "ChildProcess",
method: "exitCode",
cause: new Error("Process terminated by SIGKILL"),
cause: new Error("Exit status unavailable"),
}),
});
const client = yield* CodexClient.make(
Expand Down Expand Up @@ -1089,11 +1089,11 @@ describe("CodexAdapterV2 process exit", () => {
// Without the fix the stream never ends, so this waits until the project timeout.
const readerExit = yield* Fiber.await(eventReader);
// A failure, not a clean end, carries the Codex error into the session's lastError.
assert.isTrue(
Exit.isFailure(readerExit) &&
isProviderAdapterEventStreamError(Cause.squash(readerExit.cause)),
"the session manager must receive process death so it can retire the shared process",
);
if (Exit.isSuccess(readerExit)) assert.fail("Expected the session event stream to fail");
const failure = Cause.squash(readerExit.cause);
if (!isProviderAdapterEventStreamError(failure))
assert.fail("Expected a provider stream error");
assert.equal(failure.processExited, exit === "numeric");
}).pipe(
Effect.scoped,
Effect.provide(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2251,6 +2251,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
new ProviderAdapter.ProviderAdapterEventStreamError({
driver: CODEX_PROVIDER,
providerSessionId: input.providerSessionId,
processExited: cause._tag === "CodexAppServerProcessExitedError",
cause,
}),
),
Expand Down
126 changes: 75 additions & 51 deletions apps/server/src/orchestration-v2/RunExecutionService.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,10 @@ import * as Queue from "effect/Queue";
import * as Ref from "effect/Ref";
import * as Stream from "effect/Stream";
import * as TestClock from "effect/testing/TestClock";
import { CodexAppServerRequestError } from "effect-codex-app-server/errors";
import {
CodexAppServerProcessExitedError,
CodexAppServerRequestError,
} from "effect-codex-app-server/errors";
import { SqlError, classifySqliteError } from "effect/sql/SqlError";

import { toPersistenceSqlError } from "../persistence/Errors.ts";
Expand Down Expand Up @@ -3618,59 +3621,80 @@ it.effect("does not interrupt an ended provider when its terminal write fails",
}),
);

it.effect("ends a running root command when the provider stream fails mid-turn", () =>
Effect.gen(function* () {
const now = yield* DateTime.now;
const { written, observed } = yield* captureRootRunTermination({
key: "stream-error-running-command",
shouldFinalizeRun: () => Effect.succeed(true),
events: (ids) =>
Stream.make(
{
type: "provider_turn.updated" as const,
driver,
providerTurn: {
id: ids.rootProviderTurnId,
providerThreadId: ids.providerThreadId,
nodeId: ids.rootNodeId,
runAttemptId: ids.attemptId,
nativeTurnRef: null,
ordinal: 1,
status: "running" as const,
startedAt: now,
completedAt: null,
it.effect.each([true, false])(
"ends a running root command after stream failure (confirmed exit: %s)",
(processExited) =>
Effect.gen(function* () {
const now = yield* DateTime.now;
const interrupts = yield* Ref.make(0);
const { written, observed, finalEvents } = yield* captureRootRunTermination({
key: "stream-error-running-command",
shouldFinalizeRun: () => Effect.succeed(true),
events: (ids) =>
Stream.make(
{
type: "provider_turn.updated" as const,
driver,
providerTurn: {
id: ids.rootProviderTurnId,
providerThreadId: ids.providerThreadId,
nodeId: ids.rootNodeId,
runAttemptId: ids.attemptId,
nativeTurnRef: null,
ordinal: 1,
status: "running" as const,
startedAt: now,
completedAt: null,
},
},
},
backgroundTurnItemEvent(ids, "command_execution", "running", 1),
).pipe(
Stream.concat(
Stream.fail(
new ProviderAdapter.ProviderAdapterEventStreamError({
driver,
providerSessionId: ProviderSessionId.make("session:stream-error-running-command"),
cause: "provider process exited",
}),
backgroundTurnItemEvent(ids, "command_execution", "running", 1),
).pipe(
Stream.concat(
Stream.fail(
new ProviderAdapter.ProviderAdapterEventStreamError({
driver,
providerSessionId: ProviderSessionId.make("session:stream-error-running-command"),
processExited,
cause: processExited
? new CodexAppServerProcessExitedError({ code: 0, stderr: "private stderr" })
: "provider stream failed",
}),
),
),
),
),
// The dead process cannot take the interrupt either.
interruptTurn: (input) =>
Effect.fail(
new ProviderAdapter.ProviderAdapterInterruptError({
driver,
providerThreadId: input.providerThread.id,
providerTurnId: input.providerTurnId,
cause: "provider already exited",
}),
),
});
assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
const ids = backgroundScenarioIds("stream-error-running-command");
const endedCommand = written.find(
(item) => item.id === ids.itemId && item.status !== "running",
);
assert.isDefined(endedCommand);
}),
interruptTurn: (input) =>
Ref.update(interrupts, (count) => count + 1).pipe(
Effect.andThen(
Effect.fail(
new ProviderAdapter.ProviderAdapterInterruptError({
driver,
providerThreadId: input.providerThread.id,
providerTurnId: input.providerTurnId,
cause: "interrupt unavailable",
}),
),
),
),
});
assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
const ids = backgroundScenarioIds("stream-error-running-command");
const endedCommand = written.find(
(item) => item.id === ids.itemId && item.status !== "running",
);
assert.isDefined(endedCommand);
assert.equal(yield* Ref.get(interrupts), processExited ? 0 : 1);
const turn = finalEvents.find((event) => event.type === "provider-turn.updated");
if (!processExited) {
assert.isUndefined(turn);
return;
}
assert.equal(turn?.payload.id, ids.rootProviderTurnId);
assert.equal(turn?.payload.status, "interrupted");
assert.deepEqual(turn?.payload.completedAt, now);
const error = written.find((item) => item.type === "error");
assert.include(error?.failure.message ?? "", "Codex app-server exited with code 0");
assert.notInclude(error?.failure.message ?? "", "private stderr");
}),
);

it.effect("keeps a running root command open when a write fails under a live provider", () =>
Expand Down
76 changes: 36 additions & 40 deletions apps/server/src/orchestration-v2/RunExecutionService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2088,12 +2088,16 @@ export const layer: Layer.Layer<
const routing = yield* Ref.get(eventRouting);
const rootProviderTurn = yield* Ref.get(latestRootProviderTurn);
const squashed = Cause.squash(cause);
const streamError =
isRunExecutionIngestError(squashed) &&
isProviderEventStreamError(squashed.cause)
? squashed.cause
: undefined;
// An exclusive session closes when its thread detaches, and no terminal
// follows. The process is gone, so a handoff settles here with no interrupt.
if (
isRunExecutionIngestError(squashed) &&
isProviderEventStreamError(squashed.cause) &&
squashed.cause.detachedThreadId === input.run.threadId &&
streamError !== undefined &&
streamError.detachedThreadId === input.run.threadId &&
(yield* workspaceHandedOff)
) {
const handedOff = yield* finalizeRootRun(
Expand Down Expand Up @@ -2139,50 +2143,42 @@ export const layer: Layer.Layer<
// An adapter fails its stream when its runtime is gone. A failed write, or a session T3
// released or saw end, leaves work that can still report or that the releaser settles.
const runtimeLost =
!detached &&
isRunExecutionIngestError(squashed) &&
isProviderEventStreamError(squashed.cause) &&
squashed.cause.raisedByT3 !== true;
!detached && streamError !== undefined && streamError.raisedByT3 !== true;
if (yield* Ref.get(rootRunFinalized)) {
if (runtimeLost) yield* endWorkLostAfterRootRun(completedAt);
return;
}
const providerThread = yield* Ref.get(latestProviderThread);
// Stop background work too; the failed consumer cannot persist terminal events.
const turnStopped =
routing.rootProviderTurnId !== null &&
!routing.rootTurnEnded &&
((runtimeLost && streamError.processExited === true) ||
(yield* input.session
.interruptTurn({
providerThread,
providerTurnId: routing.rootProviderTurnId,
requestRuntimeRestart: true,
})
.pipe(
Effect.as(true),
Effect.catchCauseIf(
(interruptCause) => !Cause.hasInterruptsOnly(interruptCause),
(interruptCause) =>
Effect.logWarning(
"orchestration V2 failed to interrupt turn after ingestion failure",
{ runId: input.run.id, cause: interruptCause },
).pipe(Effect.as(false)),
),
)));
// A failed interrupt leaves the turn open for recovery. Preserve a terminal
// snapshot if its original write failed.
const abandonedProviderTurn =
routing.rootProviderTurnId !== null && !routing.rootTurnEnded
? yield* input.session
.interruptTurn({
providerThread,
providerTurnId: routing.rootProviderTurnId,
requestRuntimeRestart: true,
})
.pipe(
Effect.as(true),
Effect.catchCauseIf(
(interruptCause) => !Cause.hasInterruptsOnly(interruptCause),
(interruptCause) =>
Effect.logWarning(
"orchestration V2 failed to interrupt turn after ingestion failure",
{ runId: input.run.id, cause: interruptCause },
).pipe(Effect.as(false)),
),
// A failed interrupt leaves the stored turn running so the next start
// retries it. The snapshot is cached before it is published, so it may
// not be stored: replay a terminal one as-is.
Effect.map((interrupted) =>
!interrupted || rootProviderTurn === null
? null
: isTerminalProviderTurnStatus(rootProviderTurn.status)
? rootProviderTurn
: {
...rootProviderTurn,
status: "interrupted" as const,
completedAt,
},
),
)
: null;
!turnStopped || rootProviderTurn === null
? null
: isTerminalProviderTurnStatus(rootProviderTurn.status)
? rootProviderTurn
: { ...rootProviderTurn, status: "interrupted" as const, completedAt };
const latestItemOrdinal = yield* Ref.get(latestTurnItemOrdinal);
const openSubagents = yield* Ref.get(openRunOwnedSubagents);
const lostToolCalls = runtimeLost
Expand Down
Loading
Loading