Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 66 additions & 26 deletions apps/mobile/src/features/settings/InferenceAccounts.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,16 @@ import {
inferenceHarness,
isCustomInferenceBackend,
isInferenceProviderConfig,
isInferenceRoutingEnabled,
isManagedSettingPath,
ProviderDriverKind,
type InferenceProxyStatus,
type ServerProvider,
type InferenceDriver,
type InferenceProxyAction,
type InferenceUpstream,
} from "@t3tools/contracts";
import { useRef, useState } from "react";
import { useEffect, useRef, useState } from "react";
import { Linking, TextInput, View } from "react-native";

import { AppText } from "../../components/ui/AppText";
Expand Down Expand Up @@ -105,11 +107,11 @@ function ProviderRoutingSwitch({
const { instanceId } = provider;
const config = environment.serverConfig.settings.providerInstances[instanceId]?.config;
const route = isInferenceProviderConfig(config) ? config : null;
const enabled = route?.inferenceEnabled === true;
const enabled = isInferenceRoutingEnabled(config);
const error = failure && failure.target !== enabled ? failure.message : "";
const locked = isManagedSettingPath(
environment.serverConfig.managedPolicy,
`providerInstances.${instanceId}`,
`providerInstances.${instanceId}.config.inferenceEnabled`,
);
async function change(value: boolean) {
if (inFlight.current) return;
Expand All @@ -121,7 +123,9 @@ function ProviderRoutingSwitch({
consumer: { environmentId, label: environment.label },
environments,
driver,
...(route ? { preferredHostId: route.inferenceHostId } : {}),
...(route
? { preferredHostId: route.inferenceHostId, preserveProviderEnabled: true }
: {}),
})
: await manage({ environmentId, input: { operation: "removeRoute", instanceId } });
inFlight.current = false;
Expand All @@ -143,7 +147,9 @@ function ProviderRoutingSwitch({
subtitle={
error ||
(enabled
? `Using accounts on ${route.inferenceHostLabel || environment.label}.`
? route
? `Using accounts on ${route.inferenceHostLabel || environment.label}.`
: "This inference route is incomplete. Turn it off to use the native provider."
: "Use accounts on this environment or a paired inference host.")
}
value={enabled}
Expand Down Expand Up @@ -225,10 +231,12 @@ function InferenceControls({
environment,
setupDriver,
onConnected,
onOpenHostingSettings,
}: {
environment: SettingsTarget;
setupDriver: InferenceDriver | null;
onConnected: () => void;
onOpenHostingSettings: () => void;
}) {
const { environmentId } = environment;
const { environments } = useEnvironments();
Expand All @@ -242,10 +250,21 @@ function InferenceControls({
const [login, setLogin] = useState<{ url: string; state: string } | null>(null);
const [callback, setCallback] = useState("");
const [notice, setNotice] = useState("");
const accountSetup = useRef({ active: false });
// oxlint-disable-next-line t3code/no-unreasoned-state-sync -- prevents an account operation from routing after its setup panel closes
useEffect(() => {
const setup = { active: true };
accountSetup.current = setup;
return () => {
setup.active = false;
};
}, []);
async function run(input: InferenceProxyAction) {
const setup = accountSetup.current;
setPending(true);
setNotice("");
const result = await command({ environmentId, input });
if (!setup.active) return;
if (result._tag === "Failure") {
setPending(false);
setNotice(String(squashAtomCommandFailure(result)));
Expand Down Expand Up @@ -273,12 +292,19 @@ function InferenceControls({
(result.value.status === "ok" ||
(input.operation === "configure" && input.enabled && input.expectedEnabled === false))
) {
const instanceId = defaultInstanceIdForDriver(ProviderDriverKind.make(setupDriver));
const connected = await enableRouting({
consumer: { environmentId, label: environment.label },
environments,
driver: setupDriver,
preferredHostId: environmentId,
onlyHostId: environmentId,
...(isInferenceProviderConfig(
environment.serverConfig.settings.providerInstances[instanceId]?.config,
)
? { preserveProviderEnabled: true }
: {}),
});
if (!setup.active) return;
if (connected._tag === "Success") onConnected();
else {
const failure = squashAtomCommandFailure(connected);
Expand All @@ -296,29 +322,42 @@ function InferenceControls({
);
const available = updateQuery.data?.update?.available;
const disabled = pending || !permitted;
const requiresHostingSettings =
setupDriver !== null &&
host?.enabled === false &&
environment.serverConfig.environment.capabilities.inferenceConfigureGuard !== true;
return (
<View className="gap-1">
{host ? (
<>
<SettingsSwitchRow
label="Host accounts here"
subtitle={
host.problem ??
(host.running
? `CLIProxyAPI ${host.version} is running.`
: `Enable to install and start CLIProxyAPI ${host.version}.`)
}
disabled={disabled}
value={host.enabled}
onValueChange={(enabled) =>
void run({
operation: "configure",
enabled,
sharing: false,
expectedEnabled: host.enabled,
})
}
/>
{requiresHostingSettings ? (
<ListActionRow
label="Manage inference hosting"
description="Start hosting in Inference accounts, then turn on routing."
disabled={disabled}
onPress={onOpenHostingSettings}
/>
) : (
<SettingsSwitchRow
label="Host accounts here"
subtitle={
host.problem ??
(host.running
? `CLIProxyAPI ${host.version} is running.`
: `Enable to install and start CLIProxyAPI ${host.version}.`)
}
disabled={disabled}
value={host.enabled}
onValueChange={(enabled) =>
void run({
operation: "configure",
enabled,
sharing: false,
expectedEnabled: host.enabled,
})
}
/>
)}
{host.enabled && (
<>
<ListActionRow
Expand Down Expand Up @@ -498,7 +537,8 @@ export function InferenceAccounts({
key={panel}
environment={environment}
setupDriver={panel === "all" ? null : panel}
onConnected={() => setPanel(null)}
onConnected={() => setPanel((current) => (current === panel ? null : current))}
onOpenHostingSettings={() => setPanel("all")}
/>
)}
</>
Expand Down
2 changes: 2 additions & 0 deletions apps/server/src/environment/ServerEnvironment.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -288,6 +288,8 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("ServerEnvironmen
expect(second.capabilities.requiredWorktreeBootstrap).toBe(true);
expect(second.capabilities.usagePriceOverrides).toBe(true);
expect(second.capabilities.inferenceProxy).toBe(true);
expect(second.capabilities.inferenceConfigureGuard).toBe(true);
expect(second.capabilities.inferenceRoutePreservesEnablement).toBe(true);
expect(second.capabilities.threadActiveReorder).toBe(true);
expect(second.capabilities.threadTitleRegeneration).toBe(true);
expect(second.capabilities.threadPullRequests).toBe(true);
Expand Down
2 changes: 2 additions & 0 deletions apps/server/src/environment/ServerEnvironment.ts
Original file line number Diff line number Diff line change
Expand Up @@ -274,6 +274,8 @@ export const make = Effect.gen(function* () {
usagePriceOverrides: true,
usageModelAliases: true,
inferenceProxy: true,
inferenceConfigureGuard: true,
inferenceRoutePreservesEnablement: true,
threadPinning: true,
threadPinReorder: true,
threadActiveReorder: true,
Expand Down
28 changes: 27 additions & 1 deletion apps/server/src/inference/InferenceProxy.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -574,6 +574,21 @@ it.effect(
operation: "removeRoute",
instanceId: ProviderInstanceId.make(route.driver),
});
const incomplete = ProviderInstanceId.make("claudeAgent");
yield* settings.updateProviderInstance({
operation: "upsert",
instanceId: incomplete,
instance: {
driver: ProviderDriverKind.make("claudeAgent"),
enabled: false,
config: { inferenceEnabled: true },
},
});
yield* proxy.manage({ operation: "removeRoute", instanceId: incomplete });
expect((yield* settings.getSettings).providerInstances[incomplete]).toMatchObject({
enabled: false,
config: { inferenceEnabled: false },
});
const startSharing = proxy.manage({
operation: "configure",
enabled: true,
Expand Down Expand Up @@ -1447,7 +1462,7 @@ it.effect(
return Effect.succeed({
instance: {
...instance,
...(enabled ? { enabled: true } : {}),
enabled,
config: { ...instance.config, enabled },
},
result: undefined,
Expand All @@ -1457,6 +1472,9 @@ it.effect(
expect(yield* proxy.hasFallback(issued.key, hostId, "codex")).toBe(false);
expect(yield* proxy.routeCandidates(issued.key, hostId, "codex")).toBeNull();
expect(yield* proxy.authorize(issued.key, hostId, "codex")).toBeNull();
expect((yield* proxy.status).routes.map((route) => route.instanceId)).not.toContain(
ProviderInstanceId.make("codex"),
);
const inferenceProject = ProjectId.make("inference-project");
const codexId = ProviderInstanceId.make("codex");
yield* settings.updateSettings({
Expand All @@ -1467,6 +1485,9 @@ it.effect(
expect(yield* proxy.hasFallback(issued.key, hostId, "codex")).toBe(true);
expect(yield* proxy.routeCandidates(issued.key, hostId, "codex")).toHaveLength(1);
expect(yield* proxy.authorize(issued.key, hostId, "codex")).not.toBeNull();
expect((yield* proxy.status).routes.map((route) => route.instanceId)).toContain(
codexId,
);
const oldCatalog = catalog;
catalog = [...catalog, { id: "gpt-project-catalog-refresh", owned_by: "openai" }];
yield* proxy.refreshModelCatalog(true);
Expand All @@ -1487,6 +1508,11 @@ it.effect(
});
expect(yield* proxy.routeCandidates(issued.key, hostId, "codex")).toBeNull();
expect(yield* proxy.authorize(issued.key, hostId, "codex")).toBeNull();
yield* proxy.connect({ ...connect, grant: issued, preserveProviderEnabled: true });
const resumed = (yield* settings.getSettings).providerInstances[codexId]!;
expect(resumed.enabled).toBe(false);
expect(resumed.config).toMatchObject({ inferenceEnabled: true });
expect(yield* proxy.hasFallback(issued.key, hostId, "codex")).toBe(true);
yield* settings.updateSettings({
projectSettingsOverrides: { [inferenceProject]: null },
});
Expand Down
23 changes: 12 additions & 11 deletions apps/server/src/inference/InferenceProxy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import {
type InferenceProviderConfig,
InferenceHostConnection,
isInferenceProviderConfig,
isInferenceRoutingEnabled,
InferenceRouting,
defaultInferenceRouting,
InferenceProxyError,
Expand Down Expand Up @@ -987,14 +988,15 @@ const make = Effect.gen(function* () {
.withPermit(
Effect.gen(function* () {
const state = yield* read;
const { providerInstances } = yield* settings.getSettings;
const current = yield* settings.getSettings;
let routeProblem: string | null = null;
const summary = {
routes: Object.entries(providerInstances).flatMap(([id, instance]) => {
routes: Object.entries(current.providerInstances).flatMap(([id, instance]) => {
const instanceId = ProviderInstanceId.make(id);
if (
!isInferenceProviderConfig(instance.config) ||
!instance.config.inferenceEnabled ||
instance.enabled === false
!isProviderInstanceEnabledInAnyScope(current, instanceId)
)
return [];
const problem =
Expand All @@ -1010,7 +1012,7 @@ const make = Effect.gen(function* () {
}
return [
{
instanceId: ProviderInstanceId.make(id),
instanceId,
label: `${inferenceHarness(instance.driver)?.label ?? instance.driver}${instance.config.inferenceHostLabel ? ` via ${instance.config.inferenceHostLabel}` : ""}`,
},
];
Expand Down Expand Up @@ -1228,11 +1230,7 @@ const make = Effect.gen(function* () {
}
if (input.operation === "removeRoute") {
const revised = yield* settings.reviseProviderInstance(input.instanceId, (current) => {
if (
!current ||
!isInferenceProviderConfig(current.config) ||
!current.config.inferenceEnabled
)
if (!current || !isInferenceRoutingEnabled(current.config))
return failure("This provider is not using inference routing.");
return Effect.succeed({
instance: { ...current, config: { ...current.config, inferenceEnabled: false } },
Expand Down Expand Up @@ -1842,7 +1840,10 @@ const make = Effect.gen(function* () {
);

const connect = Effect.fn("InferenceProxy.connect")(
function* ({ grant, driver, hostLabel, connection }: InferenceRouteInput, fallback = false) {
function* (
{ grant, driver, hostLabel, connection, preserveProviderEnabled }: InferenceRouteInput,
fallback = false,
) {
const id = yield* environment.getEnvironmentId;
if (id !== grant.consumerId)
return yield* failure("Invalid inference route for this environment.");
Expand Down Expand Up @@ -2025,7 +2026,7 @@ const make = Effect.gen(function* () {
return {
instance: {
...native,
enabled: true,
...(preserveProviderEnabled && current ? {} : { enabled: true }),
config: {
...nativeConfig,
inferenceEnabled: true,
Expand Down
37 changes: 37 additions & 0 deletions apps/server/src/inference/routes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,14 @@ import {
EnvironmentId,
InferenceDriver,
INFERENCE_HARNESSES,
isInferenceProviderConfig,
type CustomModelSetting,
type ServerProviderModel,
type SettingsSecretValue,
type ProviderInstanceConfig,
} from "@t3tools/contracts";
import { fromLenientJson } from "@t3tools/shared/schemaJson";
import * as Equal from "effect/Equal";
import * as Option from "effect/Option";
import * as Schema from "effect/Schema";

Expand Down Expand Up @@ -312,6 +314,41 @@ export function routeProvider(driver: InferenceDriver, label: string | null, rou
};
}

export function isOpenCodeInferenceCatalogChange(
previous: ProviderInstanceConfig,
next: ProviderInstanceConfig,
): boolean {
const current = previous.config;
const updated = next.config;
if (
previous.driver !== "opencode" ||
!isInferenceProviderConfig(current) ||
!current.inferenceEnabled ||
!isInferenceProviderConfig(updated) ||
Equal.equals(current.inferenceModels, updated.inferenceModels)
)
return false;
const environment = previous.environment;
const content = environment?.find(({ name }) => name === "OPENCODE_CONFIG_CONTENT");
const nativeContent = content?.value;
if (environment === undefined || typeof nativeContent !== "string") return false;
const generated = routeProvider("opencode", null, {
baseUrl: current.inferenceBaseUrl,
key: current.inferenceKey,
models: updated.inferenceModels,
homePath: current.inferenceHomePath,
current: () => nativeContent,
}).environment.find(({ name }) => name === "OPENCODE_CONFIG_CONTENT");
return Equal.equals(
{
...previous,
config: { ...current, inferenceModels: updated.inferenceModels },
environment: environment.map((entry) => (entry === content ? generated : entry)),
},
next,
);
}

export function inferenceModelSlugs(config: {
setupMode?: string;
customModels: readonly CustomModelSetting[];
Expand Down
Loading
Loading