Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
187 changes: 187 additions & 0 deletions apps/server/src/orchestration-v2/ProjectionStore.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7334,6 +7334,193 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
}),
);

it.effect(
"a turn start reads a long thread's deferred commands, routable tasks and fork by index",
() =>
Effect.gen(function* () {
const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
const sql = yield* SqlClient.SqlClient;
const threadId = yield* addRolledBackRecoveryCandidate("turn-start-commands");
const runId = RunId.make("run:turn-start-commands:rolled-back");
const created = yield* DateTime.now;
const nowIso = DateTime.formatIso(created);
const message = (id: string, role: "user" | "assistant", text: string, minutes: number) =>
projectionStore.apply({
id: EventId.make(`event:${id}`),
type: "message.updated",
threadId,
occurredAt: created,
payload: {
id: MessageId.make(id),
threadId,
runId: null,
nodeId: null,
role,
text,
attachments: [],
streaming: false,
createdBy: "user",
creationSource: "web",
createdAt: DateTime.add(created, { minutes }),
updatedAt: DateTime.add(created, { minutes }),
},
});
yield* message("message:turn-start-commands:rolled-back", "user", "Own prompt", 5);
yield* message("message:turn-start-commands:compact", "user", "\u00a0 /Compact\n", 1);
yield* message("message:turn-start-commands:goal", "user", "/goal ship it", 2);
yield* message("message:turn-start-commands:plain", "user", "Plain prompt", 3);
yield* message("message:turn-start-commands:later", "user", "/compact later", 4);
yield* message("message:turn-start-commands:assistant", "assistant", "/compact", 6);

const otherThreadId = ThreadId.make("thread:other");
const transfer = (id: string, type: "fork" | "merge_back") =>
projectionStore.apply({
id: EventId.make(`event:${id}`),
type: "context-transfer.created",
threadId,
occurredAt: created,
payload: {
id: ContextTransferId.make(id),
type,
sourceThreadId: otherThreadId,
targetThreadId: threadId,
sourcePoint: { threadId: otherThreadId },
basePoint: null,
sourceProviderInstanceId: providerInstanceId,
targetProviderInstanceId: providerInstanceId,
targetRunId: null,
status: "pending",
resolution: null,
createdBy: "user",
error: null,
createdAt: created,
updatedAt: created,
consumedAt: null,
},
});
yield* transfer("transfer:turn-start-commands:fork", "fork");
yield* transfer("transfer:turn-start-commands:referenced", "merge_back");
yield* projectionStore.apply({
id: EventId.make("event:turn-start-commands:handoff"),
type: "context-handoff.updated",
threadId,
occurredAt: created,
payload: {
id: ContextHandoffId.make("handoff:turn-start-commands"),
transferId: ContextTransferId.make("transfer:turn-start-commands:referenced"),
threadId,
targetRunId: runId,
fromProviderThreadIds: [],
toProviderThreadId: ProviderThreadId.make("provider-thread:turn-start-commands"),
coveredRunOrdinals: { from: 1, to: 1 },
strategy: "full_thread_summary",
status: "ready",
summaryMessageId: null,
summaryText: "Summary",
createdByProviderInstanceId: providerInstanceId,
createdAt: created,
updatedAt: created,
},
});
const nativeId = NodeId.make("subagent:turn-start-commands:native");
yield* projectionStore.apply({
id: EventId.make("event:turn-start-commands:native"),
type: "subagent.updated",
threadId,
runId,
nodeId: nativeId,
driver,
providerInstanceId,
occurredAt: created,
payload: {
id: nativeId,
threadId,
runId,
parentNodeId: NodeId.make("node:turn-start-commands:rolled-back"),
status: "completed",
startedAt: created,
completedAt: created,
origin: "provider_native",
createdBy: "agent",
driver,
providerInstanceId,
providerThreadId: null,
childThreadId: null,
nativeTaskRef: null,
prompt: "Do the work",
title: "native",
model: null,
completionDelivery: { state: "delivered", observedByRunId: null },
result: null,
updatedAt: created,
},
});
yield* sql`
INSERT INTO orchestration_v2_projection_subagents (
subagent_id, thread_id, run_id, parent_node_id, provider, origin, status, updated_at,
payload_json
) VALUES
('subagent:turn-start-commands:delegated', ${threadId}, NULL, 'node', 'codex',
'app_owned', 'completed', ${nowIso}, '{}'),
('subagent:turn-start-commands:running', ${threadId}, NULL, 'node', 'codex',
'provider_native', 'running', ${nowIso}, '{}')
`;
yield* sql`
INSERT INTO orchestration_v2_projection_context_transfers (
context_transfer_id, source_thread_id, target_thread_id, type, status, updated_at,
payload_json
) VALUES
('transfer:turn-start-commands:result', 'thread:other', ${threadId},
'subagent_result', 'consumed', ${nowIso}, '{}'),
('transfer:turn-start-commands:other-fork', ${threadId}, 'thread:other',
'fork', 'pending', ${nowIso}, '{}')
`;

const { statements, tracer } = traceSqlStatements();
const context = yield* projectionStore
.getTurnStartContext(threadId, runId)
.pipe(Effect.withTracer(tracer));
assert.deepEqual(
context.messages.map((item) => item.id),
[
"message:turn-start-commands:compact",
"message:turn-start-commands:goal",
"message:turn-start-commands:rolled-back",
],
);
assert.deepEqual(
context.subagents.map((task) => task.id),
[nativeId],
);
assert.deepEqual(
context.contextTransfers.map((item) => item.id),
["transfer:turn-start-commands:fork", "transfer:turn-start-commands:referenced"],
);

const planOf = Effect.fn("planOf")(function* (table: string) {
const statement = statements.find((query) =>
query.includes(`FROM orchestration_v2_projection_${table}`),
)!;
const params = Array.from({ length: statement.split("?").length - 1 }, () => threadId);
return (yield* sql.unsafe<{ readonly detail: string }>(
`EXPLAIN QUERY PLAN ${statement}`,
params,
)).map((row) => row.detail);
});
for (const [table, index] of [
["messages", "orchestration_v2_projection_messages_deferred_command_idx"],
["subagents", "orchestration_v2_projection_subagents_routable_idx"],
["context_transfers", "orchestration_v2_projection_context_transfers_fork_idx"],
] as const) {
const plan = yield* planOf(table);
assert.isTrue(
plan.some((detail) => detail.includes(index)),
`${table}\n${plan.join("\n")}`,
);
}
}),
);

it.effect(
"counts each shell's own runless and live-run items, skipping rolled back and missing runs",
() =>
Expand Down
34 changes: 21 additions & 13 deletions apps/server/src/orchestration-v2/ProjectionStore.ts
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,10 @@ import type * as Statement from "effect/sql/Statement";

import { MCP_APP_OUTPUT_KEY } from "@t3tools/shared/mcpApp";
import { threadHtmlRenderAttachmentIds } from "../attachmentStore.ts";
import {
DEFERRED_COMMAND_MESSAGE_CONDITION,
JAVASCRIPT_TRIM_WHITESPACE,
} from "./deferredCommandMessage.ts";
import { readableSearchText } from "./messageSearchIndex.ts";
import {
isFoldableWorkItem,
Expand Down Expand Up @@ -5384,10 +5388,6 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
Effect.mapError((cause) => new ProjectionStoreSetupError({ cause })),
);

// SQLite trim defaults to ASCII spaces; compact recognition uses JavaScript trim.
const javascriptTrimWhitespace =
" \t\n\r\v\f\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000\ufeff";

const getTurnStartContext: ProjectionStoreV2Shape["getTurnStartContext"] = (threadId, runId) =>
sql
.withTransaction(
Expand All @@ -5407,9 +5407,10 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
INDEXED BY sqlite_autoindex_orchestration_v2_projection_nodes_1
WHERE thread_id = ${threadId} AND node_id IN (SELECT json_extract(payload_json, '$.rootNodeId') FROM orchestration_v2_projection_runs WHERE thread_id = ${threadId} AND run_id = ${runId}) ORDER BY node_id ASC
`.pipe(Effect.flatMap(decodeRows(decodeNodePayload, threadId)));
// Only the completed provider-native ones are routed. A long thread holds thousands of delegated ones.
const subagents = yield* sql<PayloadRow>`
SELECT payload_json FROM orchestration_v2_projection_subagents
WHERE thread_id = ${threadId} AND status NOT IN ('interrupted','failed','cancelled') ORDER BY subagent_id ASC
WHERE thread_id = ${threadId} AND origin = 'provider_native' AND status = 'completed' ORDER BY subagent_id ASC
`.pipe(Effect.flatMap(decodeRows(decodeSubagentPayload, threadId)));
const providerThreads = yield* sql<PayloadRow>`
SELECT payload_json FROM orchestration_v2_projection_provider_threads
Expand All @@ -5427,9 +5428,15 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
SELECT payload_json FROM orchestration_v2_projection_context_handoffs
WHERE thread_id = ${threadId} ORDER BY rowid ASC
`.pipe(Effect.flatMap(decodeRows(decodeContextHandoffPayload, threadId)));
// The fork into this thread and the transfers its handoffs came from, not the thousands of delegated results.
const contextTransfers = yield* sql<PayloadRow>`
SELECT payload_json FROM orchestration_v2_projection_context_transfers
WHERE source_thread_id = ${threadId} OR target_thread_id = ${threadId} ORDER BY rowid ASC
WHERE (target_thread_id = ${threadId} AND type = 'fork')
OR context_transfer_id IN (
SELECT json_extract(payload_json, '$.transferId')
FROM orchestration_v2_projection_context_handoffs WHERE thread_id = ${threadId}
)
ORDER BY rowid ASC
`.pipe(Effect.flatMap(decodeRows(decodeContextTransferPayload, threadId)));
const turnItems = yield* sql<PayloadRow>`
SELECT payload_json FROM orchestration_v2_projection_turn_items
Expand All @@ -5453,20 +5460,21 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
const messageIds = [run, unsentSource].flatMap((source) =>
source === undefined ? [] : [source.userMessageId],
);
// Earlier /compact and /goal turns defer context too; parseGoalCommand narrows the /goal prefix match.
const messages = yield* sql<PayloadRow>`
SELECT payload_json FROM orchestration_v2_projection_messages
WHERE thread_id = ${threadId} AND (
message_id IN ${sql.in(messageIds)}
OR (role = 'user' AND lower(trim(json_extract(payload_json, '$.text'), ${javascriptTrimWhitespace})) = '/compact')
-- Earlier /goal turns defer context too; parseGoalCommand narrows this prefix match.
OR (role = 'user' AND trim(json_extract(payload_json, '$.text'), ${javascriptTrimWhitespace}) GLOB '/goal*')
SELECT payload_json FROM (
SELECT payload_json, created_at, message_id FROM orchestration_v2_projection_messages
WHERE thread_id = ${threadId} AND message_id IN ${sql.in(messageIds)}
UNION
SELECT payload_json, created_at, message_id FROM orchestration_v2_projection_messages
WHERE thread_id = ${threadId} AND ${sql.literal(DEFERRED_COMMAND_MESSAGE_CONDITION)}
) ORDER BY created_at ASC, message_id ASC
`.pipe(Effect.flatMap(decodeRows(decodeMessagePayload, threadId)));
const conversation = yield* sql<{ present: number }>`
SELECT EXISTS(
SELECT 1 FROM orchestration_v2_projection_messages
WHERE thread_id = ${threadId} AND role = 'user'
AND (lower(trim(json_extract(payload_json, '$.text'), ${javascriptTrimWhitespace})) <> '/compact'
AND (lower(trim(json_extract(payload_json, '$.text'), ${JAVASCRIPT_TRIM_WHITESPACE})) <> '/compact'
OR json_array_length(payload_json, '$.attachments') > 0)
) AS present
`;
Expand Down
15 changes: 15 additions & 0 deletions apps/server/src/orchestration-v2/deferredCommandMessage.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
// SQLite's trim needs the characters JavaScript's String.prototype.trim removes spelled out.
export const JAVASCRIPT_TRIM_WHITESPACE =
" \t\n\r\v\f\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000\ufeff";

const text = `json_extract(payload_json, '$.text')`;
const whitespace = `'${JAVASCRIPT_TRIM_WHITESPACE}'`;

/**
* A user message that is a `/compact` or `/goal` command, whose turns defer context to the next
* one. The partial index on it is only used when the read spells this text the same way.
*/
export const DEFERRED_COMMAND_MESSAGE_CONDITION = `role = 'user' AND (
lower(trim(${text}, ${whitespace})) = '/compact'
OR trim(${text}, ${whitespace}) GLOB '/goal*'
)`;
2 changes: 2 additions & 0 deletions apps/server/src/persistence/Migrations.ts
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,7 @@ import Migration0096 from "./Migrations/096_RepairStaleTitleRegeneration.ts";
import Migration0097 from "./Migrations/097_IssueTrackerBindings.ts";
import Migration0098 from "./Migrations/098_ActiveShellIndexIndexes.ts";
import Migration0099 from "./Migrations/099_BackgroundRecoveryIndexes.ts";
import Migration0100 from "./Migrations/100_TurnStartReadIndexes.ts";

const migrationEntries = [
[1, "OrchestrationEvents", Migration0001],
Expand Down Expand Up @@ -209,6 +210,7 @@ const migrationEntries = [
[97, "IssueTrackerBindings", Migration0097],
[98, "ActiveShellIndexIndexes", Migration0098],
[99, "BackgroundRecoveryIndexes", Migration0099],
[100, "TurnStartReadIndexes", Migration0100],
] as const;

export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const);
Expand Down
26 changes: 26 additions & 0 deletions apps/server/src/persistence/Migrations/100_TurnStartReadIndexes.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
import * as Effect from "effect/Effect";
import * as SqlClient from "effect/sql/SqlClient";

import { DEFERRED_COMMAND_MESSAGE_CONDITION } from "../../orchestration-v2/deferredCommandMessage.ts";

export default Effect.gen(function* () {
const sql = yield* SqlClient.SqlClient;

// A turn start finds these few rows of a long thread without reading the rest of its history.
// The reads in ProjectionStore.getTurnStartContext must spell the same predicates.
yield* sql.unsafe(`
CREATE INDEX IF NOT EXISTS orchestration_v2_projection_messages_deferred_command_idx
ON orchestration_v2_projection_messages(thread_id, created_at, message_id)
WHERE ${DEFERRED_COMMAND_MESSAGE_CONDITION}
`);
yield* sql`
CREATE INDEX IF NOT EXISTS orchestration_v2_projection_subagents_routable_idx
ON orchestration_v2_projection_subagents(thread_id, subagent_id)
WHERE origin = 'provider_native' AND status = 'completed'
`;
yield* sql`
CREATE INDEX IF NOT EXISTS orchestration_v2_projection_context_transfers_fork_idx
ON orchestration_v2_projection_context_transfers(target_thread_id)
WHERE type = 'fork'
`;
});
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ describe("V2 preview upgrade", () => {
[97, "IssueTrackerBindings"],
[98, "ActiveShellIndexIndexes"],
[99, "BackgroundRecoveryIndexes"],
[100, "TurnStartReadIndexes"],
]);
assert.deepStrictEqual(yield* runMigrations(), []);
assert.deepStrictEqual(yield* sql`SELECT * FROM orchestration_v2_legacy_imports`, imports);
Expand Down Expand Up @@ -201,6 +202,7 @@ describe("V2 preview upgrade", () => {
[97, "IssueTrackerBindings"],
[98, "ActiveShellIndexIndexes"],
[99, "BackgroundRecoveryIndexes"],
[100, "TurnStartReadIndexes"],
]);
}).pipe(Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" }))),
);
Expand Down
Loading