Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 28 additions & 4 deletions apps/server/src/cloud/bootService.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1457,6 +1457,7 @@ process.disconnect();
activeVersion: "1.2.3",
});
expect(commands.filter((command) => command.startsWith("schtasks "))).toEqual([
"schtasks /Query /TN t3code-PC-theo /XML",
`schtasks /Create /XML ${plan.unitPath} /TN t3code-PC-theo /F`,
"schtasks /Run /TN t3code-PC-theo",
]);
Expand Down Expand Up @@ -1501,7 +1502,7 @@ process.disconnect();
"<LogonType>InteractiveToken</LogonType>",
"<LogonType>S4U</LogonType>",
);
yield* fs.writeFileString(plan.unitPath, headlessTask);
// Task Scheduler edits its registered copy without changing the saved XML.
control.task = { xml: headlessTask, enabled: true };
expect((yield* service.status).current).toBe(true);

Expand Down Expand Up @@ -1538,7 +1539,8 @@ process.disconnect();

it.effect("hands an in-service install and restart to Task Scheduler on Windows", () =>
Effect.gen(function* () {
const { service, fs, statePath, commands, makeService } = yield* makeHarness("win32");
const { service, fs, statePath, commands, control, makeService } =
yield* makeHarness("win32");
const plan = yield* service.install();
const inside = yield* makeService(undefined, "1.2.4", undefined, true);
const handoff = [
Expand All @@ -1549,15 +1551,37 @@ process.disconnect();
commands.length = 0;

expect(yield* inside.install()).toMatchObject({ handedOff: true, logPath: plan.logPath });
expect(commands.filter((command) => command.startsWith("schtasks "))).toEqual(handoff);
expect(commands.filter((command) => command.startsWith("schtasks "))).toEqual([
"schtasks /Query /TN t3code-PC-theo /XML",
...handoff,
]);
expect(parseServiceState(yield* fs.readFileString(statePath))?.activeVersion).toBe("1.2.4");
expect(
yield* fs.readFileString(`${plan.baseDir}/runtime/${SERVICE_RESTART_PENDING_FILE}`),
).toBe("1.2.4\n");

control.task = {
xml: (yield* fs.readFileString(plan.unitPath)).replace(
"<LogonType>InteractiveToken</LogonType>",
"<LogonType>S4U</LogonType>",
),
enabled: true,
};
commands.length = 0;
expect(yield* inside.restart).toBe("queued");
expect(commands).toEqual(handoff);
expect(commands).toEqual(["schtasks /Query /TN t3code-PC-theo /XML", ...handoff]);
expect(control.task.xml).toContain("<LogonType>S4U</LogonType>");
expect(yield* fs.readFileString(plan.unitPath)).toContain("<LogonType>S4U</LogonType>");

control.task.xml = control.task.xml.replace(
"<LogonType>S4U</LogonType>",
"<LogonType>InteractiveToken</LogonType>",
);
expect(yield* service.restart).toBe("restarted");
expect(control.task.xml).toContain("<LogonType>InteractiveToken</LogonType>");
expect(yield* fs.readFileString(plan.unitPath)).toContain(
"<LogonType>InteractiveToken</LogonType>",
);
}),
);

Expand Down
26 changes: 24 additions & 2 deletions apps/server/src/cloud/bootService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ import {
SERVICE_UNIT_ENV,
compareExactServiceVersions,
disabledServiceTask,
preserveServiceTaskLogonType,
serviceTaskIsDisabled,
serviceUnitBaseDirOf as bootServiceBaseDirOf,
serviceUnitEnvironmentValueOf,
Expand Down Expand Up @@ -1108,6 +1109,14 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: {

const probe = (command: string, args: ReadonlyArray<string>) =>
runner.run({ command, args, timeout: Duration.seconds(5) }).pipe(Effect.option);
const preserveWindowsTaskLogonType = Effect.fn("cloud.boot_service.preserve_task_logon_type")(
function* (unit: string) {
const registered = yield* probe("schtasks", ["/Query", ...windowsTask, "/XML"]);
return Option.isSome(registered) && registered.value.code === 0
? preserveServiceTaskLogonType(unit, registered.value.stdout)
: unit;
},
);
const succeeded = (result: Option.Option<ProcessRunner.ProcessRunOutput>) =>
Option.isSome(result) && result.value.code === 0;
const lingerArgs = [
Expand Down Expand Up @@ -1406,7 +1415,10 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: {
return yield* new BootServiceUpdatePendingError();
}
}
const unit = manager.render(plan, previousUnit);
let unit = manager.render(plan, previousUnit);
if (manager.kind === "schtasks") {
unit = yield* preserveWindowsTaskLogonType(unit);
}
yield* writeDurably(unitPath, unit);
if (!start && manager.kind === "schtasks") yield* refreshWindowsTask(unit);
return plan;
Expand Down Expand Up @@ -1455,7 +1467,17 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: {
) {
return "skipped" as const;
}
// Writes no state, and the launcher lets an update transition in flight
if (manager.kind === "schtasks") {
yield* withSelectionLock(
unitPath,
Effect.gen(function* () {
const savedUnit = yield* fs.readFileString(unitPath);
const preserved = yield* preserveWindowsTaskLogonType(savedUnit);
if (preserved !== savedUnit) yield* writeDurably(unitPath, preserved);
}),
);
}
// Writes no service state, and the launcher lets an update transition in flight
// finish before the queued stop tears it down.
if (runningInsideService && manager.handoff !== undefined) {
yield* runSteps(manager.handoff);
Expand Down
6 changes: 6 additions & 0 deletions apps/server/src/cloud/serviceProtocol.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,12 @@ const unquoteWindowsArgument = (raw: string) =>
export const serviceTaskIsDisabled = (taskXml: string): boolean =>
/<Settings>[\s\S]*?<\/Settings>/.exec(taskXml)?.[0].includes("<Enabled>false</Enabled>") === true;

export function preserveServiceTaskLogonType(taskXml: string, registeredTaskXml: string): string {
const logonType = /<LogonType>(?:InteractiveToken|S4U)<\/LogonType>/;
const registered = logonType.exec(registeredTaskXml)?.[0];
return registered === undefined ? taskXml : taskXml.replace(logonType, registered);
}

/** A task as Task Scheduler should hold it while its service must stay stopped. */
export const disabledServiceTask = (taskXml: string): string =>
serviceTaskIsDisabled(taskXml)
Expand Down
140 changes: 99 additions & 41 deletions apps/server/src/orchestration-v2/ProviderSessionManager.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,10 @@ import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
import * as ProjectService from "../project/ProjectService.ts";
import * as McpProviderSession from "@t3tools/provider-core/server/mcpSession";
import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions";
import * as ProviderLatestVersions from "@t3tools/provider-core/server/ProviderLatestVersions";
import { PiDriver } from "@t3tools/provider-pi/server";
import * as TestProviderHost from "@t3tools/provider-testing/TestProviderHost";
import * as HttpClient from "effect/http/HttpClient";
import * as McpSessionRegistry from "../mcp/McpSessionRegistry.ts";
import * as SqlitePersistence from "../persistence/Sqlite.ts";
import * as ServerSettings from "../serverSettings.ts";
Expand Down Expand Up @@ -616,7 +620,7 @@ const makeProviderAdapter = Effect.fnUntraced(function* (
function layerTest(input: {
readonly state: Ref.Ref<TestProviderRuntimeState>;
readonly continuation?: { readonly key: Ref.Ref<string>; readonly required: boolean };
readonly inferenceCatalog?: Ref.Ref<ReadonlyArray<string>>;
readonly inferenceCatalog?: Ref.Ref<ReadonlyArray<string> | undefined>;
readonly idleTimeoutMs: number;
readonly maxIdlePinMs?: number;
readonly failEventStream?: boolean;
Expand Down Expand Up @@ -708,6 +712,7 @@ function layerTest(input: {
get: () =>
Effect.gen(function* () {
const { continuation, inferenceCatalog } = input;
const catalog = inferenceCatalog ? yield* Ref.get(inferenceCatalog) : undefined;
return {
...adapter,
...(continuation
Expand All @@ -716,9 +721,7 @@ function layerTest(input: {
requiresBoundContinuationIdentity: continuation.required,
}
: {}),
...(inferenceCatalog
? { inferenceCatalog: yield* Ref.get(inferenceCatalog) }
: {}),
...(catalog === undefined ? {} : { inferenceCatalog: catalog }),
};
}),
list: () => Effect.succeed([adapter.instanceId]),
Expand Down Expand Up @@ -1290,47 +1293,102 @@ it.effect("ProviderSessionManagerV2 opens provider sessions inside the installat
}),
);

it.effect("reuses an idle session until a new hosted model needs an updated catalog", () =>
Effect.gen(function* () {
const state = yield* Ref.make(emptyState);
const inferenceCatalog = yield* Ref.make<ReadonlyArray<string>>(["original"]);
const pendingWork = yield* Ref.make(true);
yield* Effect.gen(function* () {
const eventSink = yield* EventSink.EventSinkV2;
const idAllocator = yield* IdAllocator.IdAllocatorV2;
const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
const threadId = ThreadId.make("thread-hosted-model-reconnect");
const providerSessionId = yield* idAllocator.allocate.providerSession({
providerInstanceId: modelSelection.instanceId,
threadId,
});
yield* eventSink.write({
events: [
yield* makeThreadCreatedEvent({ idAllocator, threadId, now: yield* DateTime.now }),
],
it.effect.each(["opencode", "pi", "native-pi"] as const)(
"reuses an idle session until a new hosted model needs an updated %s catalog",
(driver) =>
Effect.gen(function* () {
const state = yield* Ref.make(emptyState);
const readCatalog = Effect.fnUntraced(function* (
models: ReadonlyArray<string>,
routed = true,
) {
if (driver === "opencode") return models;
const instance = yield* PiDriver.create({
instanceId: modelSelection.instanceId,
displayName: undefined,
enabled: true,
environment: routed
? [
{ name: "T3_INFERENCE_KEY", value: "fixture-key", sensitive: true },
{
name: "T3_PI_INFERENCE_CONFIG",
value: JSON.stringify({
baseUrl: "http://127.0.0.1:4000/api/inference/linked/pi/host/v1",
models,
}),
sensitive: false,
},
]
: [],
config: {
...PiDriver.defaultConfig(),
inferenceEnabled: routed,
inferenceHostId: EnvironmentId.make("host"),
inferenceHostLabel: "Host",
inferenceBaseUrl: "http://127.0.0.1:4000/api/inference/linked/pi/host",
inferenceHomePath: "/fixture",
inferenceKey: "fixture-key",
inferenceModels: models,
},
});
return instance.inferenceCatalog;
});
const input = { threadId, providerSessionId, runtimePolicy };
const original = { ...modelSelection, model: "t3proxy/original" };
const initial = yield* manager.open({ ...input, modelSelection: original });
yield* Ref.set(inferenceCatalog, ["original", "added"]);
assert.strictEqual(yield* manager.open({ ...input, modelSelection: original }), initial);
const added = { ...modelSelection, model: "t3proxy/added" };
assert.strictEqual(yield* manager.open({ ...input, modelSelection: added }), initial);
yield* Ref.set(pendingWork, false);
const replaced = yield* manager.open({ ...input, modelSelection: added });
assert.notStrictEqual(replaced, initial);
assert.strictEqual(yield* manager.open({ ...input, modelSelection: added }), replaced);
const inferenceCatalog = yield* Ref.make(
yield* readCatalog(["original"], driver !== "native-pi"),
);
const pendingWork = yield* Ref.make(true);
yield* Effect.gen(function* () {
const eventSink = yield* EventSink.EventSinkV2;
const idAllocator = yield* IdAllocator.IdAllocatorV2;
const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
const threadId = ThreadId.make("thread-hosted-model-reconnect");
const providerSessionId = yield* idAllocator.allocate.providerSession({
providerInstanceId: modelSelection.instanceId,
threadId,
});
yield* eventSink.write({
events: [
yield* makeThreadCreatedEvent({ idAllocator, threadId, now: yield* DateTime.now }),
],
});
const input = { threadId, providerSessionId, runtimePolicy };
const original = {
...modelSelection,
model: driver === "native-pi" ? "native/original" : "t3proxy/original",
};
const initial = yield* manager.open({ ...input, modelSelection: original });
yield* Ref.set(inferenceCatalog, yield* readCatalog(["original", "added"]));
assert.strictEqual(yield* manager.open({ ...input, modelSelection: original }), initial);
const added = { ...modelSelection, model: "t3proxy/added" };
assert.strictEqual(yield* manager.open({ ...input, modelSelection: added }), initial);
yield* Ref.set(pendingWork, false);
const replaced = yield* manager.open({ ...input, modelSelection: added });
assert.notStrictEqual(replaced, initial);
assert.strictEqual(yield* manager.open({ ...input, modelSelection: added }), replaced);
}).pipe(
Effect.provide(
layerTest({
state,
inferenceCatalog,
hasPendingBackgroundWork: Ref.get(pendingWork),
idleTimeoutMs: 60_000,
}),
),
);
}).pipe(
Effect.provide(
layerTest({
state,
inferenceCatalog,
hasPendingBackgroundWork: Ref.get(pendingWork),
idleTimeoutMs: 60_000,
}),
Layer.mergeAll(
TestProviderHost.layer({ runBackgroundWork: false }),
IdAllocator.layer,
McpProviderSessions.layer,
ProviderLatestVersions.layer,
Layer.succeed(
HttpClient.HttpClient,
HttpClient.make(() => Effect.die("Unexpected HTTP")),
),
).pipe(Layer.provideMerge(NodeServices.layer)),
),
);
}),
),
);

it.effect("ProviderSessionManagerV2 opens a duplicate session only once", () =>
Expand Down
3 changes: 1 addition & 2 deletions apps/server/src/orchestration-v2/ProviderSessionManager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3092,8 +3092,7 @@ export const layerWithOptions = (
if (
adapter.continuationKey === live.continuationKey &&
model.startsWith("t3proxy/") &&
live.inferenceCatalog !== undefined &&
!live.inferenceCatalog.includes(model.slice("t3proxy/".length)) &&
!live.inferenceCatalog?.includes(model.slice("t3proxy/".length)) &&
adapter.inferenceCatalog?.includes(model.slice("t3proxy/".length))
) {
return "The hosted model catalog changed; reopening the provider session.";
Expand Down
12 changes: 9 additions & 3 deletions apps/server/src/serviceLauncher.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -466,20 +466,26 @@ setInterval(() => {}, 1_000);
conhostPath: "conhost.exe",
});
yield* fs.writeFileString(unitPath, unit);
// Task Scheduler's copy: `t3 doctor` disabled it to keep the service stopped.
let task = disabledServiceTask(unit.replace("1.1.0", "1.0.0"));
const headlessUnit = unit.replace(
"<LogonType>InteractiveToken</LogonType>",
"<LogonType>S4U</LogonType>",
);
// Task Scheduler keeps logon settings separately from the saved XML.
let task = disabledServiceTask(headlessUnit.replace("1.1.0", "1.0.0"));
const schtasks = async (...args: ReadonlyArray<string>) => {
if (args[0] === "/Create") task = await NodeFSP.readFile(args[2] ?? "", "utf8");
return { stdout: task };
};

yield* Effect.promise(() => registerServiceTask(unitPath, "t3code-PC-theo", schtasks));
assert.equal(task, disabledServiceTask(unit));
assert.equal(task, disabledServiceTask(headlessUnit));
assert.equal(yield* fs.readFileString(unitPath), headlessUnit);
assert.deepEqual(yield* fs.readDirectory(root), ["t3code-task.xml"]);

task = unit.replace("1.1.0", "1.0.0");
yield* Effect.promise(() => registerServiceTask(unitPath, "t3code-PC-theo", schtasks));
assert.equal(task, unit);
assert.equal(yield* fs.readFileString(unitPath), unit);
}),
);

Expand Down
5 changes: 4 additions & 1 deletion apps/server/src/serviceLauncher.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import {
isExactServiceVersion,
parseLegacyServiceState,
disabledServiceTask,
preserveServiceTaskLogonType,
parseServiceState,
serviceTaskIsDisabled,
serviceUnitBaseDirOf,
Expand Down Expand Up @@ -115,13 +116,15 @@ export async function registerServiceTask(
): Promise<void> {
const register = (xmlPath: string) => schtasks("/Create", "/XML", xmlPath, "/TN", taskName, "/F");
const registered = await schtasks("/Query", "/TN", taskName, "/XML");
const savedUnit = await NodeFSP.readFile(unitPath, "utf8");
const unit = preserveServiceTaskLogonType(savedUnit, registered.stdout);
if (unit !== savedUnit) await replaceFileDurably(unitPath, unit);
if (!serviceTaskIsDisabled(registered.stdout)) {
await register(unitPath);
return;
}
const disabledPath = `${unitPath}.${process.pid}.disabled`;
try {
const unit = await NodeFSP.readFile(unitPath, "utf8");
await NodeFSP.writeFile(disabledPath, disabledServiceTask(unit));
await register(disabledPath);
} finally {
Expand Down
Loading
Loading