Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
163 changes: 163 additions & 0 deletions apps/server/src/slots/SlotObserver.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import { AgentScope, type AgentScopeShape } from "@t3tools/shared/AgentScope";
import type { CapacityFileRead, CapacityHost } from "@t3tools/shared/capacityReadings";
import * as HostProcess from "@t3tools/shared/HostProcess";
import type { DeepPartial } from "@t3tools/shared/Struct";
import { DISPATCH_SAFETY_MS } from "@t3tools/shared/agentSlots";
import * as Duration from "effect/Duration";
import * as Effect from "effect/Effect";
import * as Layer from "effect/Layer";
Expand All @@ -26,10 +27,12 @@ import * as HostResources from "../resourceTelemetry/HostResources.ts";
import * as ServerSettings from "../serverSettings.ts";
import * as SlotGate from "./SlotGate.ts";
import {
makeSlotObserver,
OBSERVE_MAX_HANGING,
OBSERVE_MAX_WAITING,
OBSERVE_READ_TIMEOUT,
SlotCapacityHost,
type SlotObserveInput,
slicePath,
} from "./SlotObserver.ts";

Expand Down Expand Up @@ -167,6 +170,9 @@ const release = (answer: SlotGate.SlotAcquire) =>
Effect.flatMap((gate) => gate.release(answer.type === "allow" ? answer.token : null)),
);

/** A dispatch that ends with no child; its answer. */
const acquireAndRelease = Effect.tap(acquire, release).pipe(Effect.map((answer) => answer.type));

/** Lets forked observations and the fake host's promises finish (scheduler turns, no real wait). */
const settle = Effect.forEach(Array.from({ length: 20 }), () => Effect.yieldNow, { discard: true });

Expand Down Expand Up @@ -682,6 +688,89 @@ describe("SlotGate observe mode", () => {
}),
);

it.effect("a held slot its holder's start used up counts once, slots off as on (#381)", () =>
Effect.gen(function* () {
const HOLD = { key: "w_held", threadId: "waiter", untilMs: 100_000 };
const waiter = ThreadId.make("waiter");
const cases = [
{ name: "slots on", agentSlots: { enabled: true, maxPerMachine: 5 }, slowFirst: false },
// Slots off: the gate never uses the hold up, so it stays in the store.
{ name: "slots off", agentSlots: {}, slowFirst: false },
// The holder's own observation reads the store too late: the next one still sees the start.
{ name: "slots off, first read slow", agentSlots: {}, slowFirst: true },
];
for (const entry of cases) {
const world = newWorld();
let reads = 0;
const counts = yield* run(
world,
{
agentSlots: { mode: "observe", startGraceSeconds: 30, ...entry.agentSlots },
waitFor: { enabled: true },
},
Effect.gen(function* () {
const gate = yield* SlotGate.SlotGate;
yield* gate.attachReservations({
read: Effect.suspend(() => {
reads += 1;
return entry.slowFirst && reads === 1 ? Effect.never : Effect.succeed([HOLD]);
}),
used: () => Effect.void,
});
const answers: Array<string> = [];
// The holder delegates; another thread delegates 5 s later.
const own = yield* gate.acquire({ callerThreadId: waiter, targetInstanceId: MAIN });
answers.push(own.type);
if (entry.slowFirst) {
yield* settle;
yield* TestClock.adjust(OBSERVE_READ_TIMEOUT);
yield* statusAfter(1);
}
yield* TestClock.adjust(Duration.seconds(5));
answers.push(yield* acquireAndRelease);
const started = (yield* statusAfter(2)).last?.pieces;
// Its child shows its run: counted as that run, the hold still used up.
const token = own.type === "allow" ? own.token : null;
yield* gate.started(token, ThreadId.make("child"));
world.threads = [{ id: "child" }];
yield* TestClock.adjust(Duration.seconds(5));
answers.push(yield* acquireAndRelease);
const running = (yield* statusAfter(3)).last?.pieces;
// The child's run ends: nothing of it counts, nor the hold it used.
yield* gate.release(token);
world.threads = [];
yield* TestClock.adjust(Duration.seconds(5));
answers.push(yield* acquireAndRelease);
const ended = (yield* statusAfter(4)).last?.pieces;
// Observe data only: every answer stays today's allow.
assert.deepStrictEqual(answers, ["allow", "allow", "allow", "allow"], entry.name);
return [started, running, ended];
}),
);
assert.deepStrictEqual(counts, [1, 1, 0], entry.name);
}
// The holder's dispatch ends with no child before anyone else asks: nothing counts.
for (const agentSlots of [{ enabled: true, maxPerMachine: 5 }, {}]) {
const status = yield* run(
newWorld(),
{ agentSlots: { mode: "observe", ...agentSlots }, waitFor: { enabled: true } },
Effect.gen(function* () {
const gate = yield* SlotGate.SlotGate;
yield* gate.attachReservations({
read: Effect.succeed([HOLD]),
used: () => Effect.void,
});
yield* release(yield* gate.acquire({ callerThreadId: waiter, targetInstanceId: MAIN }));
yield* TestClock.adjust(Duration.seconds(5));
yield* acquireAndRelease;
return yield* statusAfter(2);
}),
);
assert.strictEqual(status.last?.pieces, 0, JSON.stringify(agentSlots));
}
}),
);

it.effect("a wait store read that hangs never delays the decision; the rule reads slow", () =>
Effect.gen(function* () {
const world = newWorld();
Expand Down Expand Up @@ -755,6 +844,80 @@ describe("SlotGate observe mode", () => {
}),
);

it.effect("the rule's memory keeps only what still counts (#381)", () =>
Effect.gen(function* () {
const world = newWorld();
const HOLD = { key: "w_held", threadId: "waiter", untilMs: 60_000 };
const observer = yield* makeSlotObserver(Effect.succeed({ visible: new Set<string>() })).pipe(
Effect.provide(
Layer.mergeAll(
Layer.succeed(SlotCapacityHost, hostOf(world)),
Layer.succeed(AgentScope, scopesOf(world)),
Layer.succeed(HostProcess.UserId, UID),
Layer.succeed(HostProcess.Platform, "linux"),
),
),
);
let decided = 0;
/** One decision, observed to the end. */
const decide = (input: Partial<SlotObserveInput> & { atMs: number }) =>
Effect.gen(function* () {
yield* observer.observe({
today: ["allow"],
starting: [],
held: Effect.succeed([HOLD]),
startGraceMs: 30_000,
gateRecords: false,
...input,
});
decided += 1;
for (let turn = 0; (yield* observer.status).observed < decided; turn += 1) {
if (turn === 200) return yield* Effect.die(new Error("not observed"));
yield* settle;
}
});
// Slots off: the holder starts (a dispatch that never ends: a lost release).
yield* decide({
atMs: 0,
callerThreadId: "waiter",
ownStart: { token: 1, instanceId: MAIN, atMs: 0 },
});
assert.deepStrictEqual(yield* observer.kept, { starts: 1, usedHolds: 1 }, "kept");
// Past its hold's end and the dispatch's safety limit: the next decision lets both go.
yield* decide({ atMs: DISPATCH_SAFETY_MS, held: Effect.succeed([]) });
assert.deepStrictEqual(yield* observer.kept, { starts: 0, usedHolds: 0 }, "ended");
// Slots switched on (the gate keeps the records): the rule's own go at once.
yield* decide({
atMs: DISPATCH_SAFETY_MS + 1,
callerThreadId: "waiter",
ownStart: { token: 2, instanceId: MAIN, atMs: DISPATCH_SAFETY_MS + 1 },
held: Effect.succeed([{ ...HOLD, untilMs: 2 * DISPATCH_SAFETY_MS }]),
});
yield* decide({ atMs: DISPATCH_SAFETY_MS + 2, gateRecords: true });
assert.deepStrictEqual(yield* observer.kept, { starts: 0, usedHolds: 0 }, "slots on");
// Switched on while a slots-off observation still waits: the slots-on one counts the hold
// as the gate does (the gate never used it up).
const ON = DISPATCH_SAFETY_MS + 3;
yield* observer.observe({
today: ["allow"],
atMs: ON,
starting: [],
held: Effect.succeed([{ ...HOLD, untilMs: 2 * DISPATCH_SAFETY_MS }]),
callerThreadId: "waiter",
startGraceMs: 30_000,
gateRecords: false,
ownStart: { token: 3, instanceId: MAIN, atMs: ON },
});
decided += 1;
yield* decide({
atMs: ON,
gateRecords: true,
held: Effect.succeed([{ ...HOLD, untilMs: 2 * DISPATCH_SAFETY_MS }]),
});
assert.strictEqual((yield* observer.status).last?.pieces, 1, "switched on");
}).pipe(Effect.scoped),
);

it("nests a systemd slice the way systemd does", () => {
assert.strictEqual(
slicePath("app-t3code-agents-0123.slice"),
Expand Down
43 changes: 37 additions & 6 deletions apps/server/src/slots/SlotObserver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,10 @@ export interface SlotObserveInput {
readonly atMs: number;
readonly starting: ReadonlyArray<StartingEntry>;
/**
* The live held slots as the gate counts them (memory and wait store), as of the decision. Read
* in the observer's fiber, so a slow wait store never delays the decision.
* The live held slots as the gate counts them: memory as of the decision, the wait store as of
* when the observation runs (read in the observer's fiber, so a slow store never delays the
* decision). An observation that waits behind others can miss a hold a later decision used up
* meanwhile, or count one granted after its decision: one agent off, for a few seconds.
*/
readonly held: Effect.Effect<ReadonlyArray<SlotReservation>>;
/** `acquire`'s caller: its own held slot is used up by the start it asks for (counted once). */
Expand All @@ -85,6 +87,9 @@ export function slicePath(slice: string): string {
return parts.map((_, index) => `/${parts.slice(0, index + 1).join("-")}.slice`).join("");
}

/** One of the rule's own starts, and the thread that asked for it (its holds are used up). */
type OwnStart = StartingEntry & { readonly threadId?: string | undefined };

interface OwnPiece {
readonly scope: { readonly unit: string; readonly slice: string } | undefined;
}
Expand Down Expand Up @@ -121,7 +126,14 @@ export const makeSlotObserver = Effect.fn("makeSlotObserver")(function* (
* its child shows a run (then it counts as that run, once). A dispatch that ends with no child
* stops counting at once.
*/
let ownStarts: ReadonlyArray<StartingEntry> = [];
let ownStarts: ReadonlyArray<OwnStart> = [];
/**
* While the gate keeps no records: held slots of the threads those starts were for, by key, until
* they end. A start uses its thread's hold up, as the gate's `useUp` does with slots on (with slots
* off the gate leaves it in the store), so the start and the hold never both count, not even once
* the start stops counting (its child shows its run, or its dispatch ends).
*/
const usedHolds = new Map<string, number>();

const readFacts = async (pieces: ReadonlyArray<OwnPiece>): Promise<CapacityFacts> => {
const machine = await readMachine(host);
Expand Down Expand Up @@ -198,9 +210,23 @@ export const makeSlotObserver = Effect.fn("makeSlotObserver")(function* (
input.atMs,
input.startGraceMs,
);
const live = liveReservations(heldRead.value, input.atMs);
if (!input.gateRecords) {
// Also the starts this decision counts (a skipped or slow observation of a start's own
// decision leaves its hold to the next one).
const startedThreads = new Set(own.map((entry) => entry.threadId));
if (input.ownStart !== undefined) startedThreads.add(input.callerThreadId);
for (const reservation of live) {
if (startedThreads.has(reservation.threadId)) {
usedHolds.set(reservation.key, reservation.untilMs);
}
}
}
const held = onePerThread(
liveReservations(heldRead.value, input.atMs).filter(
(reservation) => reservation.threadId !== input.callerThreadId,
live.filter(
(reservation) =>
reservation.threadId !== input.callerThreadId &&
(input.gateRecords || !usedHolds.has(reservation.key)),
),
);
const scopes = yield* Effect.forEach([...running.visible], (threadId) =>
Expand Down Expand Up @@ -256,9 +282,12 @@ export const makeSlotObserver = Effect.fn("makeSlotObserver")(function* (
ownStarts = input.gateRecords
? []
: liveStarting(ownStarts, new Set(), input.atMs, input.startGraceMs);
for (const [key, untilMs] of usedHolds) {
if (input.gateRecords || untilMs <= input.atMs) usedHolds.delete(key);
}
const ownBefore = ownStarts;
if (!input.gateRecords && input.ownStart !== undefined) {
ownStarts = [...ownStarts, input.ownStart];
ownStarts = [...ownStarts, { ...input.ownStart, threadId: input.callerThreadId }];
}
if (waiting >= OBSERVE_MAX_WAITING) {
return Ref.update(status, (current) => ({ ...current, skipped: current.skipped + 1 }));
Expand Down Expand Up @@ -302,5 +331,7 @@ export const makeSlotObserver = Effect.fn("makeSlotObserver")(function* (
);
}),
status: Ref.get(status),
/** What the rule keeps in memory (for tests: nothing past the point it stops counting). */
kept: Effect.sync(() => ({ starts: ownStarts.length, usedHolds: usedHolds.size })),
};
});
6 changes: 3 additions & 3 deletions packages/shared/src/agentSlots.ts
Original file line number Diff line number Diff line change
Expand Up @@ -119,12 +119,12 @@ export interface StartingEntry {
* never get more slots than are free (a lost end stops after `DISPATCH_SAFETY_MS`). After the
* dispatch, a child that shows no run yet (a queued run) counts for the start grace.
*/
export function liveStarting(
entries: ReadonlyArray<StartingEntry>,
export function liveStarting<Entry extends StartingEntry>(
entries: ReadonlyArray<Entry>,
visibleRunIds: ReadonlySet<string>,
nowMs: number,
startGraceMs: number,
): ReadonlyArray<StartingEntry> {
): ReadonlyArray<Entry> {
return entries.filter((entry) => {
if (entry.childThreadId !== undefined && visibleRunIds.has(entry.childThreadId)) return false;
return entry.endedAtMs === undefined
Expand Down
Loading