Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
118 changes: 105 additions & 13 deletions apps/server/src/inference/InferenceProxy.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import * as Effect from "effect/Effect";
import * as Layer from "effect/Layer";
import * as FileSystem from "effect/FileSystem";
import * as PlatformError from "effect/PlatformError";
import * as Predicate from "effect/Predicate";
import * as Option from "effect/Option";
import * as TestClock from "effect/testing/TestClock";
import * as Sink from "effect/Sink";
Expand Down Expand Up @@ -74,6 +75,14 @@ it.effect(
keychainState.onRead = undefined;
const config = yield* ServerConfig.ServerConfig;
const state = yield* ServerSecretStore.ServerSecretStore;
const savedPort = state.get("inference-proxy-state").pipe(
Effect.map((value) => {
const saved = JSON.parse(new TextDecoder().decode(Option.getOrThrow(value))) as {
port: number;
};
return saved.port;
}),
);
const home = yield* fs.makeTempDirectoryScoped();
const hostId = EnvironmentId.make("host");
const peer = { environmentId: EnvironmentId.make("worker"), label: "Workstation" };
Expand All @@ -84,6 +93,8 @@ it.effect(
const downloading = yield* Deferred.make<void>();
const finishDownload = yield* Deferred.make<void>();
let registryOffline = true;
let latestProxyVersion = proxyVersion;
let managementUnavailable = false;
let tailnetCatalogDown = false;
let linked = false;
let linkedUrl = "http://127.0.0.1:23456";
Expand Down Expand Up @@ -120,9 +131,11 @@ it.effect(
const releaseClaudeUsage = yield* Deferred.make<void>();
let lan = false;
let rejectNextLaunch = false;
let currentExit: Deferred.Deferred<void> | undefined;
let currentExit: Deferred.Deferred<void, PlatformError.PlatformError> | undefined;
let expectRecovery = false;
const recovered = yield* Deferred.make<void>();
const restoredCatalog = yield* Deferred.make<void>();
const recoveredCatalog = yield* Deferred.make<void>();
let configPath = "";
let starts = 0;
let stops = 0;
Expand All @@ -138,9 +151,13 @@ it.effect(
method: "spawn",
});
}
const exited = yield* Deferred.make<void>();
const exited = yield* Deferred.make<void, PlatformError.PlatformError>();
if (!tailscale) {
if (ChildProcess.isStandardCommand(command)) configPath = String(command.args[1]);
managementUnavailable = false;
if (ChildProcess.isStandardCommand(command)) {
expect.soft(command.options.detached).toBe(false);
configPath = String(command.args[1]);
}
currentExit = exited;
if (expectRecovery) yield* Deferred.succeed(recovered, undefined);
yield* Effect.addFinalizer(() => Deferred.succeed(exited, undefined));
Expand Down Expand Up @@ -208,6 +225,10 @@ it.effect(
),
);
}
if (managementUnavailable && request.url.endsWith("/v0/management/auth-files"))
return Effect.succeed(
HttpClientResponse.fromWeb(request, new Response(null, { status: 503 })),
);
if (request.url.endsWith("/v0/management/api-call")) {
const body = request.body;
if (body._tag !== "Uint8Array" || !body.text) throw new Error("Missing usage request");
Expand Down Expand Up @@ -278,13 +299,13 @@ it.effect(
HttpClientResponse.fromWeb(
request,
Response.json({
tag_name: "v8.0.61",
tag_name: "v" + latestProxyVersion,
draft: false,
prerelease: false,
assets: [
{
id: 123,
name: "CLIProxyAPI_8.0.61_darwin_aarch64.tar.gz",
name: `CLIProxyAPI_${latestProxyVersion}_darwin_aarch64.tar.gz`,
size: 1,
digest: "sha256:" + "0".repeat(64),
},
Expand Down Expand Up @@ -435,11 +456,20 @@ it.effect(
Layer.provide(
Layer.succeed(ProviderInstanceRegistryMutator.ProviderInstanceRegistryMutator, {
reconcile: (instances) =>
Effect.sync(() => {
Effect.gen(function* () {
registryReconciles++;
const activeCodex = instances[ProviderInstanceId.make("codex")];
activeCodexConfig = activeCodex?.config;
activeCodexEnabled = activeCodex?.enabled;
if (
Predicate.hasProperty(activeCodexConfig, "customModels") &&
Array.isArray(activeCodexConfig.customModels)
) {
if (activeCodexConfig.customModels.includes("gpt-restored"))
yield* Deferred.succeed(restoredCatalog, undefined);
if (activeCodexConfig.customModels.includes("gpt-recovered"))
yield* Deferred.succeed(recoveredCatalog, undefined);
}
}),
reloadSecrets: () => Effect.void,
rebuild: () => Effect.void,
Expand Down Expand Up @@ -619,6 +649,8 @@ it.effect(
sharing: true,
});
yield* startSharing;
const configuredHost = yield* proxy.grant({ environmentId: hostId, label: "Host" });
expect(yield* savedPort).toBe(Number(new URL(configuredHost.baseUrl).port));
expect((yield* proxy.status).enabled).toBe(true);
expect(importedCodexAuth).toMatchObject([
{
Expand Down Expand Up @@ -1782,6 +1814,26 @@ it.effect(
identity = hostId;
yield* proxy.manage({ operation: "checkUpdate" }).pipe(Effect.flip);
registryOffline = false;
managementUnavailable = true;
expect((yield* proxy.manage({ operation: "checkUpdate" })).update).toEqual({
current: proxyVersion,
available: null,
});
expect(stops).toBe(0);
expect((yield* proxy.manage({ operation: "update" })).update).toEqual({
current: proxyVersion,
available: null,
});
expect(stops).toBe(1);
yield* proxy.manage({ operation: "update" });
expect(stops).toBe(1);
managementUnavailable = true;
rejectNextLaunch = true;
yield* proxy.manage({ operation: "update" }).pipe(Effect.flip);
expect(stops).toBe(2);
yield* proxy.manage({ operation: "update" });
latestProxyVersion = "8.0.61";
yield* TestClock.adjust("15 minutes");
expect((yield* proxy.manage({ operation: "checkUpdate" })).update).toEqual({
current: proxyVersion,
available: "8.0.61",
Expand All @@ -1794,31 +1846,53 @@ it.effect(
yield* Deferred.succeed(finishDownload, undefined);
expect((yield* Fiber.join(update)).message).toContain("checksum");
expect((yield* proxy.status).version).toBe(proxyVersion);
expect(stops).toBe(0);
expect(stops).toBe(2);
const staleRuntime = runtime.replace(proxyVersion, "8.0.58");
const otherPlatform = staleRuntime.replace("darwin-arm64", "linux-x64");
yield* fs.makeDirectory(staleRuntime);
yield* fs.makeDirectory(otherPlatform);
const installed = runtime.replace(proxyVersion, "8.0.61");
yield* fs.makeDirectory(installed, { recursive: true });
yield* fs.writeFileString(installed + "/cli-proxy-api", "");
yield* proxy.connect(connect);
const localCandidate = () =>
proxy.routeCandidates(local.key, hostId, "codex", "codex-auto-review");
expect(yield* localCandidate()).toMatchObject([{ baseUrl: local.baseUrl }]);
yield* NodeSocketServer.make({
host: "127.0.0.1",
port: Number(new URL(local.baseUrl).port),
});
rejectNextLaunch = true;
yield* proxy.manage({ operation: "update" }).pipe(Effect.flip);
expect(yield* proxy.status).toMatchObject({ running: true, version: proxyVersion });
expect((yield* proxy.authorize(issued.key))?.key).toBeDefined();
const restoredUrl = (yield* proxy.authorize(issued.key))!.baseUrl;
expect(restoredUrl).not.toBe(local.baseUrl);
expect(yield* localCandidate()).toMatchObject([{ baseUrl: restoredUrl }]);
const restoredPort = Number(new URL(restoredUrl).port);
expect(yield* savedPort).toBe(restoredPort);
yield* NodeSocketServer.make({ host: "127.0.0.1", port: restoredPort });
catalog = [...catalog, { id: "gpt-upgraded", owned_by: "openai" }];
yield* proxy.manage({ operation: "update" });
expect((yield* proxy.status).version).toBe("8.0.61");
const upgradedUrl = (yield* proxy.authorize(issued.key))!.baseUrl;
expect(upgradedUrl).not.toBe(restoredUrl);
expect(
yield* proxy.routeCandidates(local.key, hostId, "codex", "gpt-upgraded"),
).toMatchObject([{ baseUrl: upgradedUrl }]);
const upgradedPort = Number(new URL(upgradedUrl).port);
expect(yield* savedPort).toBe(upgradedPort);
expect(yield* fs.exists(staleRuntime)).toBe(false);
expect(yield* fs.exists(otherPlatform)).toBe(true);
expect(yield* fs.exists(runtime)).toBe(true);
return { grant: issued, oldPort: Number(new URL(local.baseUrl).port) };
return { grant: issued, oldPort: upgradedPort };
}).pipe(Effect.provide(service)),
);
expect(stops).toBe(3);
expect(stops).toBe(5);
const legacyConfig = config.stateDir + "/inference-proxy/config.yaml";
expect(yield* fs.exists(configPath)).toBe(false);
yield* NodeSocketServer.make({ host: "127.0.0.1", port: oldPort });
yield* fs.writeFileString(legacyConfig, "occupied listener configuration");
catalog = [...catalog, { id: "gpt-restored", owned_by: "openai" }];
yield* Effect.scoped(
Effect.gen(function* () {
const proxy = yield* InferenceProxy.InferenceProxy;
Expand All @@ -1829,6 +1903,11 @@ it.effect(
const restoredUrl = (yield* proxy.authorize(grant.key))!.baseUrl;
expect(restoredUrl).toMatch(/^http:\/\/127\.0\.0\.1:/);
expect(new URL(restoredUrl).port).not.toBe(String(oldPort));
const local = yield* proxy.grant({ environmentId: hostId, label: "Local" });
yield* Deferred.await(restoredCatalog);
expect(
yield* proxy.routeCandidates(local.key, hostId, "codex", "gpt-restored"),
).toMatchObject([{ baseUrl: restoredUrl }]);
expect(yield* fs.readFileString(legacyConfig)).toBe("occupied listener configuration");
yield* proxy.manage({ operation: "revoke", environmentId: peer.environmentId });
expect(yield* proxy.authorize(grant.key)).toBeNull();
Expand All @@ -1841,19 +1920,32 @@ it.effect(
expect(restored.key).not.toBe(grant.key);
expect((yield* proxy.grant({ ...peer, automatic: true })).key).toBe(restored.key);
expectRecovery = true;
yield* Deferred.succeed(currentExit!, undefined);
catalog = [...catalog, { id: "gpt-recovered", owned_by: "openai" }];
yield* Deferred.fail(
currentExit!,
PlatformError.systemError({
_tag: "Unknown",
module: "ChildProcess",
method: "exitCode",
description: "Process interrupted due to receipt of signal: 'SIGTERM'",
}),
);
yield* TestClock.adjust("1 second");
yield* Deferred.await(recovered);
expect((yield* proxy.grant({ environmentId: hostId, label: "Local" })).baseUrl).toBe(
restoredUrl,
);
yield* Deferred.await(recoveredCatalog);
expect(
yield* proxy.routeCandidates(local.key, hostId, "codex", "gpt-recovered"),
).toMatchObject([{ baseUrl: restoredUrl }]);
yield* proxy.manage({ operation: "configure", enabled: false, sharing: false });
expect((yield* proxy.status).consumers).toEqual([]);
expect(Option.isSome(yield* state.get("inference-proxy-state"))).toBe(true);
}).pipe(Effect.provide(service)),
);
expect(starts).toBe(5);
expect(stops).toBe(5);
expect(starts).toBe(7);
expect(stops).toBe(7);
}).pipe(
Effect.provide(
ServerSettings.layer.pipe(
Expand Down
38 changes: 31 additions & 7 deletions apps/server/src/inference/InferenceProxy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -536,11 +536,11 @@ const make = Effect.gen(function* () {
yield* store.set("inference-proxy-state", new TextEncoder().encode(yield* encodeSaved(state)));
cached = state;
}, Effect.uninterruptible);
const stop = Effect.gen(function* () {
const stop: Effect.Effect<void> = Effect.gen(function* () {
if (running) yield* Scope.close(running.scope, Exit.void);
running = undefined;
yield* Cache.invalidateAll(routeProbes);
});
yield* Effect.addFinalizer(() => stop);
const writeConfig = Effect.fn("InferenceProxy.writeConfig")(function* (
routing: NonNullable<Saved["routing"]>,
runtime: Pick<NonNullable<typeof running>, "file" | "port" | "key" | "management">,
Expand Down Expand Up @@ -605,6 +605,8 @@ const make = Effect.gen(function* () {
.spawn(
ChildProcess.make(executable, ["-config", file, "-local-model"], {
cwd: directory,
// Desktop updates terminate the backend process group if graceful shutdown stalls.
detached: false,
env: {
HOME: directory,
USERPROFILE: directory,
Expand Down Expand Up @@ -1019,9 +1021,10 @@ const make = Effect.gen(function* () {

const watchChild = Effect.fn("InferenceProxy.watchChild")(function* () {
const child = yield* Queue.take(children);
yield* child.exitCode;
yield* refreshLocalProviders().pipe(Effect.ignoreCause({ log: true }));
// A signal exit is reported as a failed exitCode by the Node spawner.
yield* Effect.ignore(child.exitCode);
if (running?.child !== child) return;
// Local providers call the sidecar directly, so their next turn cannot wake a stopped host.
yield* Effect.sleep("1 second");
yield* mutex
.withPermit(
Expand All @@ -1032,7 +1035,6 @@ const make = Effect.gen(function* () {
)
.pipe(Effect.retry({ times: 2, schedule: Schedule.exponential("1 second") }));
});
yield* watchChild().pipe(Effect.ignore, Effect.forever, Effect.forkScoped);

const status = mutex
.withPermit(
Expand Down Expand Up @@ -1186,7 +1188,27 @@ const make = Effect.gen(function* () {
);
const available =
release.version.localeCompare(current, "en", { numeric: true }) > 0 ? release.version : null;
if (operation === "checkUpdate" || !available) return { update: { current, available } };
if (operation === "checkUpdate") return { update: { current, available } };
if (!available) {
yield* mutex.withPermit(
Effect.gen(function* () {
const before = yield* read;
const previous = running;
const runtime = yield* ensureWatched(before);
if (runtime !== previous) return;
yield* jsonRequest(
runtime.baseUrl + "/v0/management/auth-files",
runtime.management,
Files,
).pipe(
Effect.catch(() =>
stop.pipe(Effect.andThen(ensureWatched(before)), Effect.uninterruptible),
),
);
}),
);
return { update: { current, available } };
}
yield* install(directory, release).pipe(
Effect.provide(installContext),
Effect.mapError(safeFailure("Could not install the verified CLIProxyAPI update.")),
Expand Down Expand Up @@ -2235,7 +2257,7 @@ const make = Effect.gen(function* () {
targetInstanceId?: ProviderInstanceId,
) {
const hostId = yield* environment.getEnvironmentId;
const { enabled, routePools = [], traexHosts } = yield* read;
const { enabled, routePools = [], traexHosts } = yield* mutex.withPermit(read);
const currentSettings = yield* settings.getSettings;
const catalogs = yield* Cache.make({
capacity: 64,
Expand Down Expand Up @@ -2379,6 +2401,8 @@ const make = Effect.gen(function* () {
return refreshRouteCatalogs(false, instanceId).pipe(Effect.ignoreCause);
return fresh ? invalidateCatalog.pipe(Effect.andThen(refreshCatalog)) : refreshCatalog;
};
yield* Effect.addFinalizer(() => stop);
yield* watchChild().pipe(Effect.ignore, Effect.forever, Effect.forkScoped);
yield* watchLocalCodexAuth.pipe(Effect.ignoreCause({ log: true }), Effect.forkScoped);
yield* watchLocalClaudeAuth.pipe(Effect.ignoreCause({ log: true }), Effect.forkScoped);
yield* read.pipe(
Expand Down
Loading