Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
166 changes: 153 additions & 13 deletions apps/server/src/slots/SlotObserver.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,24 @@ const TWO_RUNNING = [{ id: "a" }, { id: "b" }];
/** The calm wait after a (re)start: the level is "elevated" until this much calm has passed. */
const CALM_WAIT = Duration.seconds(61);

/** The observer alone, its projection read as `readRunning` (the order rows drive it by hand). */
const observerFor = (
world: World,
readRunning: Effect.Effect<
{ readonly visible: ReadonlySet<string> } | undefined
> = Effect.succeed({ visible: new Set<string>() }),
) =>
makeSlotObserver(readRunning).pipe(
Effect.provide(
Layer.mergeAll(
Layer.succeed(SlotCapacityHost, hostOf(world)),
Layer.succeed(AgentScope, scopesOf(world)),
Layer.succeed(HostProcess.UserId, UID),
Layer.succeed(HostProcess.Platform, "linux"),
),
),
);

describe("SlotGate observe mode", () => {
it.effect("fixed mode, old files and unknown modes read nothing for the rule", () =>
Effect.gen(function* () {
Expand Down Expand Up @@ -771,6 +789,99 @@ describe("SlotGate observe mode", () => {
}),
);

it.effect("slots off: a start uses its thread's hold up in every order", () =>
Effect.gen(function* () {
const HOLD = { key: "w_held", threadId: "waiter", untilMs: 100_000 };
// What another thread's decision 5 s after the holder's start counts. Slots on reads the
// same, as the gate uses the hold up at the holder's decision (the gate test above: 0 once
// the dispatch ended with no child, 1 while the child runs).
const cases = [
// The holder's own store read is slow, or its projection read fails; then its dispatch
// ends with no child, so the start stops counting before any read sees it.
{ own: "slow", after: "no child", pieces: 0 },
{ own: "fails", after: "no child", pieces: 0 },
// The holder's observation is skipped (the queue is full); its child shows its run
// before the next decision, while the queued observations still read.
{ own: "skipped", after: "child runs", pieces: 1 },
// The child's run shows before the next store read: the start no longer counts, but it
// still used its hold up (counted as the child's run, once).
{ own: "slow", after: "child runs", pieces: 1 },
] as const;
const got: Array<string> = [];
for (const entry of cases) {
const name = `own ${entry.own}, ${entry.after}`;
const world = newWorld();
let visible = new Set<string>();
let runningFails = false;
const observer = yield* observerFor(
world,
Effect.sync(() => (runningFails ? undefined : { visible })),
);
const decide = (atMs: number, patch: Partial<SlotObserveInput>) =>
observer.observe({
today: ["allow"],
atMs,
starting: [],
held: Effect.succeed([HOLD]),
startGraceMs: 30_000,
gateRecords: false,
...patch,
});
const done = (count: number) =>
Effect.gen(function* () {
for (let turn = 0; turn < 200; turn += 1) {
const status = yield* observer.status;
if (status.observed + status.skipped >= count) return status;
yield* settle;
}
return yield* Effect.die(new Error(`${name}: fewer than ${count} observations`));
});
let decisions = 0;
if (entry.own === "skipped") {
// Other threads' observations fill the queue; the first one's store read takes 1 s.
for (let index = 0; index < OBSERVE_MAX_WAITING; index += 1) {
yield* decide(0, {
callerThreadId: `other-${index}`,
held: Effect.as(Effect.sleep(Duration.seconds(1)), [HOLD]),
});
decisions += 1;
}
}
runningFails = entry.own === "fails";
yield* decide(0, {
callerThreadId: "waiter",
ownStart: { token: 1, instanceId: MAIN, atMs: 0 },
held: entry.own === "slow" ? Effect.never : Effect.succeed([HOLD]),
});
decisions += 1;
yield* settle;
if (entry.own === "slow") yield* TestClock.adjust(OBSERVE_READ_TIMEOUT);
if (entry.own !== "skipped") yield* done(decisions);
runningFails = false;
if (entry.after === "no child") {
yield* observer.released(1, 1_000);
} else {
yield* observer.started(1, "child");
visible = new Set(["child"]);
}
for (let index = 0; entry.own === "skipped" && index < OBSERVE_MAX_WAITING; index += 1) {
yield* TestClock.adjust(Duration.seconds(1));
yield* settle;
}
yield* done(decisions);
yield* decide(5_000, { callerThreadId: "other" });
const status = yield* done(decisions + 1);
assert.strictEqual(status.skipped, entry.own === "skipped" ? 1 : 0, name);
assert.strictEqual(status.last?.readings, "ok", name);
got.push(`${name}: ${status.last?.pieces}`);
}
assert.deepStrictEqual(
got,
cases.map((entry) => `own ${entry.own}, ${entry.after}: ${entry.pieces}`),
);
}).pipe(Effect.scoped),
);

it.effect("a wait store read that hangs never delays the decision; the rule reads slow", () =>
Effect.gen(function* () {
const world = newWorld();
Expand Down Expand Up @@ -848,16 +959,7 @@ describe("SlotGate observe mode", () => {
Effect.gen(function* () {
const world = newWorld();
const HOLD = { key: "w_held", threadId: "waiter", untilMs: 60_000 };
const observer = yield* makeSlotObserver(Effect.succeed({ visible: new Set<string>() })).pipe(
Effect.provide(
Layer.mergeAll(
Layer.succeed(SlotCapacityHost, hostOf(world)),
Layer.succeed(AgentScope, scopesOf(world)),
Layer.succeed(HostProcess.UserId, UID),
Layer.succeed(HostProcess.Platform, "linux"),
),
),
);
const observer = yield* observerFor(world);
let decided = 0;
/** One decision, observed to the end. */
const decide = (input: Partial<SlotObserveInput> & { atMs: number }) =>
Expand All @@ -882,10 +984,10 @@ describe("SlotGate observe mode", () => {
callerThreadId: "waiter",
ownStart: { token: 1, instanceId: MAIN, atMs: 0 },
});
assert.deepStrictEqual(yield* observer.kept, { starts: 1, usedHolds: 1 }, "kept");
assert.deepStrictEqual(yield* observer.kept, { starts: 1, usedHolds: 1, notes: 0 }, "kept");
// Past its hold's end and the dispatch's safety limit: the next decision lets both go.
yield* decide({ atMs: DISPATCH_SAFETY_MS, held: Effect.succeed([]) });
assert.deepStrictEqual(yield* observer.kept, { starts: 0, usedHolds: 0 }, "ended");
assert.deepStrictEqual(yield* observer.kept, { starts: 0, usedHolds: 0, notes: 0 }, "ended");
// Slots switched on (the gate keeps the records): the rule's own go at once.
yield* decide({
atMs: DISPATCH_SAFETY_MS + 1,
Expand All @@ -894,7 +996,11 @@ describe("SlotGate observe mode", () => {
held: Effect.succeed([{ ...HOLD, untilMs: 2 * DISPATCH_SAFETY_MS }]),
});
yield* decide({ atMs: DISPATCH_SAFETY_MS + 2, gateRecords: true });
assert.deepStrictEqual(yield* observer.kept, { starts: 0, usedHolds: 0 }, "slots on");
assert.deepStrictEqual(
yield* observer.kept,
{ starts: 0, usedHolds: 0, notes: 0 },
"slots on",
);
// Switched on while a slots-off observation still waits: the slots-on one counts the hold
// as the gate does (the gate never used it up).
const ON = DISPATCH_SAFETY_MS + 3;
Expand All @@ -915,6 +1021,40 @@ describe("SlotGate observe mode", () => {
held: Effect.succeed([{ ...HOLD, untilMs: 2 * DISPATCH_SAFETY_MS }]),
});
assert.strictEqual((yield* observer.status).last?.pieces, 1, "switched on");
// A start's note that no store read could take (every read hangs): it goes after the
// dispatch's safety limit, or at once when slots switch on.
const slow = yield* observerFor(world);
let slowDecided = 0;
const decideSlow = (input: Partial<SlotObserveInput> & { atMs: number }) =>
Effect.gen(function* () {
yield* slow.observe({
today: ["allow"],
starting: [],
held: Effect.never,
startGraceMs: 30_000,
gateRecords: false,
...input,
});
slowDecided += 1;
yield* settle;
yield* TestClock.adjust(OBSERVE_READ_TIMEOUT);
for (let turn = 0; (yield* slow.status).observed < slowDecided; turn += 1) {
if (turn === 200) return yield* Effect.die(new Error("not observed"));
yield* settle;
}
});
const startAt = (atMs: number, token: number) => ({
atMs,
callerThreadId: "waiter",
ownStart: { token, instanceId: MAIN, atMs },
});
yield* decideSlow(startAt(0, 1));
assert.deepStrictEqual(yield* slow.kept, { starts: 1, usedHolds: 0, notes: 1 }, "unread");
yield* decideSlow({ atMs: DISPATCH_SAFETY_MS });
assert.deepStrictEqual(yield* slow.kept, { starts: 0, usedHolds: 0, notes: 0 }, "limit");
yield* decideSlow(startAt(DISPATCH_SAFETY_MS + 1, 2));
yield* decideSlow({ atMs: DISPATCH_SAFETY_MS + 2, gateRecords: true });
assert.deepStrictEqual(yield* slow.kept, { starts: 0, usedHolds: 0, notes: 0 }, "slots on");
}).pipe(Effect.scoped),
);

Expand Down
32 changes: 22 additions & 10 deletions apps/server/src/slots/SlotObserver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import type { SlotRuleStatus } from "@t3tools/contracts/waitFor";
import { AgentScope } from "@t3tools/shared/AgentScope";
import {
DISPATCH_SAFETY_MS,
liveReservations,
liveStarting,
onePerThread,
Expand Down Expand Up @@ -87,9 +88,6 @@ export function slicePath(slice: string): string {
return parts.map((_, index) => `/${parts.slice(0, index + 1).join("-")}.slice`).join("");
}

/** One of the rule's own starts, and the thread that asked for it (its holds are used up). */
type OwnStart = StartingEntry & { readonly threadId?: string | undefined };

interface OwnPiece {
readonly scope: { readonly unit: string; readonly slice: string } | undefined;
}
Expand Down Expand Up @@ -126,14 +124,20 @@ export const makeSlotObserver = Effect.fn("makeSlotObserver")(function* (
* its child shows a run (then it counts as that run, once). A dispatch that ends with no child
* stops counting at once.
*/
let ownStarts: ReadonlyArray<OwnStart> = [];
let ownStarts: ReadonlyArray<StartingEntry> = [];
/**
* While the gate keeps no records: held slots of the threads those starts were for, by key, until
* they end. A start uses its thread's hold up, as the gate's `useUp` does with slots on (with slots
* off the gate leaves it in the store), so the start and the hold never both count, not even once
* the start stops counting (its child shows its run, or its dispatch ends).
*/
const usedHolds = new Map<string, number>();
/**
* Slots off: threads allowed a start, noted at the decision (before any skip). The next good store
* read uses their holds up, whatever became of the start meanwhile, and drops the notes. Slots on
* drops them all; a note no read took goes after `DISPATCH_SAFETY_MS`.
*/
const startedThreads = new Map<string, number>();

const readFacts = async (pieces: ReadonlyArray<OwnPiece>): Promise<CapacityFacts> => {
const machine = await readMachine(host);
Expand Down Expand Up @@ -212,15 +216,12 @@ export const makeSlotObserver = Effect.fn("makeSlotObserver")(function* (
);
const live = liveReservations(heldRead.value, input.atMs);
if (!input.gateRecords) {
// Also the starts this decision counts (a skipped or slow observation of a start's own
// decision leaves its hold to the next one).
const startedThreads = new Set(own.map((entry) => entry.threadId));
if (input.ownStart !== undefined) startedThreads.add(input.callerThreadId);
for (const reservation of live) {
if (startedThreads.has(reservation.threadId)) {
usedHolds.set(reservation.key, reservation.untilMs);
}
}
startedThreads.clear();
}
const held = onePerThread(
live.filter(
Expand Down Expand Up @@ -285,9 +286,16 @@ export const makeSlotObserver = Effect.fn("makeSlotObserver")(function* (
for (const [key, untilMs] of usedHolds) {
if (input.gateRecords || untilMs <= input.atMs) usedHolds.delete(key);
}
for (const [threadId, atMs] of startedThreads) {
if (input.gateRecords || input.atMs - atMs >= DISPATCH_SAFETY_MS) {
startedThreads.delete(threadId);
}
}
const ownBefore = ownStarts;
if (!input.gateRecords && input.ownStart !== undefined) {
ownStarts = [...ownStarts, { ...input.ownStart, threadId: input.callerThreadId }];
ownStarts = [...ownStarts, input.ownStart];
if (input.callerThreadId !== undefined)
startedThreads.set(input.callerThreadId, input.atMs);
}
if (waiting >= OBSERVE_MAX_WAITING) {
return Ref.update(status, (current) => ({ ...current, skipped: current.skipped + 1 }));
Expand Down Expand Up @@ -332,6 +340,10 @@ export const makeSlotObserver = Effect.fn("makeSlotObserver")(function* (
}),
status: Ref.get(status),
/** What the rule keeps in memory (for tests: nothing past the point it stops counting). */
kept: Effect.sync(() => ({ starts: ownStarts.length, usedHolds: usedHolds.size })),
kept: Effect.sync(() => ({
starts: ownStarts.length,
usedHolds: usedHolds.size,
notes: startedThreads.size,
})),
};
});
Loading