Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 12 additions & 3 deletions apps/server/src/process/agentScope.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import * as ProviderProcesses from "./providerProcesses.ts";
import { classifyScope, make, ownedAgentSlice, type ScopeState } from "./agentScope.ts";

const GiB = 1024 ** 3;
const memory = vi.hoisted(() => ({ total: 16 * 1024 ** 3 }));
const memory = vi.hoisted(() => ({ total: 16 * 1024 ** 3, swapKb: 2 * 1024 ** 2 }));
vi.mock("node:os", async (importOriginal) => ({
...(await importOriginal<typeof import("node:os")>()),
totalmem: () => memory.total,
Expand Down Expand Up @@ -133,7 +133,9 @@ const withFakeSystemd = <A, E>(
Effect.provideService(FileSystem.FileSystem, {
...fileSystem,
readFileString: (path, encoding) =>
fileSystem.readFileString(path.replace("/sys/fs/cgroup", cgroupRoot), encoding),
path === "/proc/meminfo"
? Effect.sync(() => `MemTotal: 16384 kB\nSwapTotal: ${memory.swapKb} kB\n`)
: fileSystem.readFileString(path.replace("/sys/fs/cgroup", cgroupRoot), encoding),
writeFileString: (path, data, options) =>
fileSystem.writeFileString(path.replace("/sys/fs/cgroup", cgroupRoot), data, options),
}),
Expand Down Expand Up @@ -331,8 +333,9 @@ describe("AgentScope service", () => {
),
);

it.effect("follows a live memory change at the next launch", () => {
it.effect("follows a live memory or swap change at the next launch", () => {
memory.total = 16 * GiB;
memory.swapKb = 2 * 1024 ** 2;
return withFakeSystemd(({ scope, calls, failingVerbs }) =>
Effect.gen(function* () {
const launch = scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" });
Expand Down Expand Up @@ -361,6 +364,12 @@ describe("AgentScope service", () => {
grown,
grown,
]);

// Without swap, MemoryHigh only throttles; MemoryMax must OOM-kill the runaway.
memory.swapKb = 0;
yield* launch;
expect(memoryLimits().at(-1)).toEqual(["MemoryHigh=infinity", `MemoryMax=${18 * GiB}`]);
memory.swapKb = 2 * 1024 ** 2;
}),
);
});
Expand Down
25 changes: 16 additions & 9 deletions apps/server/src/process/agentScope.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,14 +36,13 @@ export function ownedAgentSlice(stateDir: string): string {
const GiB = 1024 ** 3;

/**
* Agents together leave about 6 GB free before they are throttled and 4 GB
* before the kernel kills one, so the server, sshd, and tailscaled keep
* working. Small machines still give agents at least half of RAM before
* throttling and three quarters before a kill.
* Agents leave about 6 GB free before throttling and 4 GB before an OOM kill (at
* least half and three quarters of RAM go to agents) so the server and sshd keep
* working. Without swap, MemoryHigh stalls every agent and never reaches the kill.
*/
function agentSliceMemoryLimits(totalBytes: number) {
function agentSliceMemoryLimits(totalBytes: number, hasSwap: boolean) {
return {
high: Math.max(totalBytes - 6 * GiB, Math.floor(totalBytes / 2)),
high: hasSwap ? Math.max(totalBytes - 6 * GiB, Math.floor(totalBytes / 2)) : "infinity",
max: Math.max(totalBytes - 4 * GiB, Math.floor(totalBytes * 0.75)),
};
}
Expand Down Expand Up @@ -204,13 +203,20 @@ export const make = Effect.gen(function* () {
});
// VMs and containers can resize RAM live, so each launch re-checks the total
// the limits came from. A failed update is retried by the next launch.
// An unreadable meminfo keeps the throttle, as before swap was checked.
const readHasSwap = fileSystem.readFileString("/proc/meminfo").pipe(
Effect.map((text) => /^SwapTotal:\s*0 kB$/m.exec(text) === null),
Effect.orElseSucceed(() => true),
);
let appliedTotal: number | undefined;
let appliedHasSwap: boolean | undefined;
const memoryLimitsLock = yield* Semaphore.make(1);
const applyMemoryLimits = memoryLimitsLock.withPermits(1)(
Effect.gen(function* () {
const total = NodeOS.totalmem();
if (total === appliedTotal) return;
const limits = agentSliceMemoryLimits(total);
const hasSwap = yield* readHasSwap;
if (total === appliedTotal && hasSwap === appliedHasSwap) return;
const limits = agentSliceMemoryLimits(total, hasSwap);
const applied = yield* setProperties(AGENT_SLICE, [
`MemoryHigh=${limits.high}`,
`MemoryMax=${limits.max}`,
Expand All @@ -225,13 +231,14 @@ export const make = Effect.gen(function* () {
});
}
appliedTotal = total;
appliedHasSwap = hasSwap;
}),
);
yield* applyMemoryLimits;
yield* setProperties(AGENTS_PARENT_SLICE, AGENTS_PARENT_PROPERTIES);
yield* Effect.logInfo("Agent scopes are on", {
slice: AGENT_SLICE,
...agentSliceMemoryLimits(NodeOS.totalmem()),
...agentSliceMemoryLimits(NodeOS.totalmem(), yield* readHasSwap),
oomPolicy: supportsOomPolicy,
});
return {
Expand Down
Loading