Repository navigation
Expand file tree
/
Copy pathbudget_test.go
More file actions
584 lines (547 loc) · 26.1 KB
/
Copy pathbudget_test.go
File metadata and controls
584 lines (547 loc) · 26.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
package exec
import (
"context"
"fmt"
"testing"
"time"
"github.com/Agent-Field/agentfield/sdk/go/ai"
)
// turnBilling is one turn of a leaf as the ceiling sees it: what the whole
// prompt weighed, how much of that the provider served from its prefix cache,
// and what came back.
//
// The shape is the loop's own. Every turn re-sends the entire transcript, so a
// turn's prompt is the fixed floor plus a full observation window; everything
// in it except this turn's own additions is byte-identical to the previous
// call, which is precisely the prefix a cache serves warm.
type turnBilling struct {
prompt int
cached int
completion int
}
const (
// freshTurnTokens is what one turn genuinely adds: the assistant message it
// just wrote and the tool results it just collected. Everything else in the
// next prompt is a re-send.
freshTurnTokens = 2_500
// completionTurnTokens is an ordinary reply plus its reasoning.
completionTurnTokens = 800
)
// steadyTurn bills one turn of a leaf carrying a full window. warm says the
// provider reported the re-sent prefix as a cache read, which is true from the
// second call of a run onwards wherever prompt caching is actually in play, and
// never true when the plumbing that reports it is dark.
func steadyTurn(windowBytes int, warm bool) turnBilling {
prompt := observationFixedFloorTokens + windowBytes/observationBytesPerToken
turn := turnBilling{prompt: prompt, completion: completionTurnTokens}
if warm {
turn.cached = prompt - freshTurnTokens
}
return turn
}
// warmTurn bills one turn at a stated prefix-cache hit rate, which is how the
// audit measured real runs (92-98% on the nodes that melted) and the one dial
// that decides how far apart the two bounds sit.
func warmTurn(windowBytes, hitPercent int) turnBilling {
turn := steadyTurn(windowBytes, false)
turn.cached = turn.prompt * hitPercent / 100
return turn
}
// calibratedWindow is the observation window the default grant is actually
// calibrated against: the one a leaf gets when nothing could name its model.
//
// The tests below used to spell it observationWindow(1<<20), and that was the
// same number by accident. Every long-context model was clamped to 64KB, so
// naming a frontier model and naming none picked windows within a factor of two
// of each other and it did not matter which was written down. The clamp is gone
// — see TestTheObservationWindowHasNoAbsoluteCeiling — and the two are now
// orders of magnitude apart, so these tests have to say which they mean. They
// mean this one: they are tests about the default grant, and a window sized for
// a million tokens is a question about a grant sized for a million tokens.
func calibratedWindow() int { return observationWindow(0) }
// turnsAffordable counts the turns a leaf can pay for out of one ceiling, using
// the production accounting rather than a copy of it: the same exhausted() the
// loop tests on every pass, which reads both of the leaf's bounds.
//
// caching says the run's prefix cache is both engaged and reported. The first
// turn is always cold — there is no previous call to have warmed anything.
func turnsAffordable(windowBytes, ceiling int, caching bool) int {
return turnsUntilExhausted(ceiling, func(turn int) turnBilling {
return steadyTurn(windowBytes, caching && turn > 0)
})
}
// turnsUntilExhausted runs the ceiling's own arithmetic forward over a supplied
// billing pattern and reports where the loop would have granted the landing.
func turnsUntilExhausted(ceiling int, bill func(turn int) turnBilling) int {
outcome := &Outcome{}
turns := 0
for turns < 10_000 && !exhausted(outcome, ceiling) {
turn := bill(turns)
outcome.Usage.Calls++
outcome.Usage.PromptTokens += turn.prompt
outcome.Usage.CachedTokens += turn.cached
outcome.Usage.CompletionTokens += turn.completion
turns++
}
return turns
}
// The invariant that broke silently, pinned so it cannot break silently again.
//
// abcc5a5 stopped sizing the observation window from the leaf's spend ceiling,
// which was right — a cumulative budget is not a quantity that fits in one
// request — and left the ceiling itself at the value it had been given back
// when a turn cost about 6k prompt tokens. The window went from 25KB to 256KB
// and the wallet did not move, so the same leaf could afford roughly two turns
// where it used to afford a dozen. Measured on one task that meant seven
// extension rounds against trunk's one, 4.5x the tokens and 3.2x the wall.
//
// Nothing in the type system connects those two numbers, and nothing said out
// loud what the connection was. This is that statement: however the window is
// sized, a leaf must still be able to afford at least as many turns as it could
// before the window grew. Turns affordable is what governs whether a leaf
// finishes inside its budget, which is what governs whether the reconciler has
// to buy the rest of the work in extension nodes.
func TestAffordableTurnsDidNotFallWhenTheWindowGrew(t *testing.T) {
// The harness as it was: the window the spend ceiling used to buy, billed
// the way the ceiling used to bill — every re-sent token at full weight,
// because nothing was reading the cached share.
const legacyWindow = DefaultLeafTokens / 6
before := turnsAffordable(legacyWindow, DefaultLeafTokens, false)
if before < 8 {
t.Fatalf("the baseline arithmetic is wrong: %d turns, expected the low teens", before)
}
// The harness as it is: the window the default grant is calibrated against,
// billed with the cached share weighted down to what it actually costs.
after := turnsAffordable(calibratedWindow(), DefaultLeafTokens, true)
if after < before {
t.Fatalf("growing the observation window cost the leaf turns: %d before, %d after — "+
"a leaf that cannot finish inside one budget is bought out in extension nodes",
before, after)
}
}
// The other half of the same statement, and the one the deleted ceiling used to
// hide: window and grant are one decision made twice.
//
// The 64KB clamp made the window a fixed cost, so a run could move from a 600k
// model to a 2M one and afford exactly the same turns. That looked like safety
// and was a category error wearing safety's clothes — it capped MEMORY to
// protect SPEND, and every model above about 600k tokens was handed the same
// sixty-four kilobytes whatever the catalog said about it. The window is now a
// share of the model's own context, which means the grant beside it has to be
// sized for the same model, and nothing inside exec can do that: the grant is
// the caller's, arriving through NewLinear.
//
// So what is pinned is the coupling, as a number a reader can act on. A leaf on
// a much larger model needs a proportionally larger grant to afford the turns
// the default one buys, and if that multiple ever runs away the sizing has
// stopped being proportional and this fails.
func TestALargerWindowCostsAProportionallyLargerGrant(t *testing.T) {
want := turnsAffordable(calibratedWindow(), DefaultLeafTokens, true)
for _, context := range []int{200_000, 1 << 20} {
window := observationWindow(context)
// The grant that buys the same number of turns at this window. Doubling
// rather than solving: what is being asserted is the order of magnitude,
// and the exact figure is the caller's to pick anyway.
grant := DefaultLeafTokens
for grant < 4096*DefaultLeafTokens && turnsAffordable(window, grant, true) < want {
grant *= 2
}
multiple := grant / DefaultLeafTokens
if turnsAffordable(window, grant, true) < want {
t.Fatalf("a %d-token model's %d-byte window never affords %d turns", context, window, want)
}
// The window grew by this much, so the grant may grow by about this much
// and no more. Slack of 2x covers the doubling itself.
if ratio := window / calibratedWindow(); multiple > 2*ratio+2 {
t.Errorf("a %d-token model's window is %dx the calibrated one but needs %dx the grant — "+
"the sizing has stopped being proportional", context, ratio, multiple)
}
t.Logf("a %d-token model: %d-byte window, %dx the default grant to afford %d turns",
context, window, multiple, want)
}
}
// The wrap-up warning fires at 70% of the ceiling and the landing reserve takes
// the tail, and both were calibrated against a turn that cost about 6k tokens.
// A leaf measured at 152k tokens on its FIRST call is past the warning before
// it has done any work, so it spends its entire existence in landing mode — the
// sharpest form of the same defect. One turn must stay one turn — read through
// budgetUsed, which is what the loop itself now tests, so that adding a second
// bound beside the ceiling cannot push a first turn into landing mode either.
func TestOneTurnDoesNotCrossTheWrapUpThreshold(t *testing.T) {
outcome := &Outcome{}
turn := steadyTurn(calibratedWindow(), false)
outcome.Usage.PromptTokens = turn.prompt
outcome.Usage.CompletionTokens = turn.completion
if used := budgetUsed(outcome, DefaultLeafTokens, 0); used > wrapUpAt {
t.Fatalf("a single cold turn reads %.2f used of a %d ceiling, past the %.2f wrap-up mark",
used, DefaultLeafTokens, wrapUpAt)
}
if exhausted(outcome, DefaultLeafTokens) {
t.Fatal("a single cold turn exhausted the leaf")
}
}
// The discount is a discount and never a credit. A provider that reports
// nothing is billed exactly as the ceiling always billed, and a provider that
// reports something impossible cannot make a leaf immortal.
func TestTheCachedDiscountOnlyEverDiscounts(t *testing.T) {
plain := &Outcome{Usage: Usage{PromptTokens: 10_000, CompletionTokens: 1_000}}
if got := spent(plain); got != 11_000 {
t.Fatalf("an unreported prefix cache changed the bill: %d, want 11000", got)
}
warm := &Outcome{Usage: Usage{PromptTokens: 10_000, CachedTokens: 10_000, CompletionTokens: 1_000}}
if got := spent(warm); got != 10_000*cachedTokenWeightPercent/100+1_000 {
t.Fatalf("a fully cached prompt billed %d", got)
}
if spent(warm) >= spent(plain) {
t.Fatal("a cached prompt cost at least as much as a cold one")
}
absurd := &Outcome{Usage: Usage{PromptTokens: 10_000, CachedTokens: 900_000, CompletionTokens: 1_000}}
if got := spent(absurd); got < 1_000 {
t.Fatalf("an over-reported cache read credited the leaf: %d", got)
}
}
// warmRunaway is the node that melted: a leaf that never converges on its own,
// whose re-sent prefix the provider serves warm from the second call onwards.
// It is the exact shape of the worst nodes in the audit — no single prompt above
// ~20k tokens, and a bill the cache discount shrinks by an order of magnitude.
type warmRunaway struct {
window int
hitPercent int
calls int
}
func (w *warmRunaway) CompleteWithMessages(context.Context, []ai.Message, ...ai.Option) (*ai.Response, error) {
w.calls++
turn := warmTurn(w.window, w.hitPercent)
if w.calls == 1 {
turn.cached = 0
}
return &ai.Response{
Choices: []ai.Choice{{
Message: ai.Message{
Role: "assistant",
Content: []ai.ContentPart{{Type: "text", Text: "still working"}},
ToolCalls: []ai.ToolCall{call(fmt.Sprintf("c%d", w.calls), "write",
fmt.Sprintf(`{"path":"out-%d.txt","text":"x"}`, w.calls))},
},
FinishReason: "tool_calls",
}},
Usage: &ai.Usage{
PromptTokens: turn.prompt,
CompletionTokens: turn.completion,
CacheReadInputTokens: turn.cached,
},
}, nil
}
// billedRunaway gives every turn one stated price and always asks to write, so
// the real loop keeps going until one of its own bounds stops it. It exists for
// the landing contract below: warmRunaway is calibrated to the audited leaf and
// other tests depend on that exact shape.
type billedRunaway struct {
turn turnBilling
calls int
}
func (b *billedRunaway) CompleteWithMessages(context.Context, []ai.Message, ...ai.Option) (*ai.Response, error) {
b.calls++
return &ai.Response{
Choices: []ai.Choice{{
Message: ai.Message{
Role: "assistant",
Content: []ai.ContentPart{{Type: "text", Text: "still working"}},
ToolCalls: []ai.ToolCall{call(fmt.Sprintf("b%d", b.calls), "write",
fmt.Sprintf(`{"path":"priced-%d.txt","text":"x"}`, b.calls))},
},
FinishReason: "tool_calls",
}},
Usage: &ai.Usage{
PromptTokens: b.turn.prompt,
CompletionTokens: b.turn.completion,
CacheReadInputTokens: b.turn.cached,
},
}, nil
}
// C1 — The reserve cannot buy a second grant.
//
// The worker trial's four old landing turns billed 122,392 tokens against a
// 150,000-token grant. This leaf makes the same failure unmistakable: every
// turn costs a quarter of its grant, so the old reserve would buy a whole
// second grant. The token allowance ends it after one landing turn instead.
// That same one turn pins C3: even though it carries the landing past its token
// allowance, the workspace still gets one complete turn in which to land.
func TestTheLandingReserveCannotBuyASecondGrant(t *testing.T) {
const grant = 40_000
turn := turnBilling{prompt: grant / 4}
crossingCalls := turnsUntilExhausted(grant, func(int) turnBilling { return turn })
if crossingCalls != 4 {
t.Fatalf("the fixture crossed after %d calls, want four quarter-grant turns", crossingCalls)
}
client := &billedRunaway{turn: turn}
linear := NewLinear(client, workspace(t), nil, 100, grant, time.Hour)
outcome, err := linear.Run(context.Background(), Task{NodeID: 1, Brief: "work"})
if err != nil {
t.Fatal(err)
}
if got := client.calls - crossingCalls; got != 1 {
t.Fatalf("landing calls = %d, want the one turn that leaves the tree consistent", got)
}
crossingSpend := crossingCalls * spentOfTurn(turn)
landingSpend := spent(outcome) - crossingSpend
ceilingWithTurnInFlight := landingAllowance(grant) + spentOfTurn(turn)
if landingSpend > ceilingWithTurnInFlight {
t.Fatalf("landing billed %d, past its %d allowance plus the %d-token turn in flight",
landingSpend, landingAllowance(grant), spentOfTurn(turn))
}
legacyLandingSpend := landingTurns * spentOfTurn(turn)
if legacyLandingSpend <= 2*landingSpend {
t.Fatalf("the old four-turn reserve would bill %d against today's %d; the fixture does not expose the regression",
legacyLandingSpend, landingSpend)
}
if want := grant + grant/4; spent(outcome) != want {
t.Fatalf("leaf spent %d, want %d (1.25 times its grant)", spent(outcome), want)
}
}
// C2 and C3 — The landing still gets its four turns when it can afford them,
// and the token half of the reserve never reduces that floor to zero.
func TestALeanLandingStillGetsItsFourTurns(t *testing.T) {
const grant = 20_000
turn := turnBilling{prompt: 1_000}
crossingCalls := turnsUntilExhausted(grant, func(int) turnBilling { return turn })
client := &billedRunaway{turn: turn}
linear := NewLinear(client, workspace(t), nil, 100, grant, time.Hour)
outcome, err := linear.Run(context.Background(), Task{NodeID: 1, Brief: "work"})
if err != nil {
t.Fatal(err)
}
if got := client.calls - crossingCalls; got != landingTurns {
t.Fatalf("landing calls = %d, want the full %d-turn reserve", got, landingTurns)
}
if outcome.Stop != StopBudget || outcome.Exhausted != StopBudget {
t.Fatalf("stop = %s, exhausted = %q — want the budget landing both times",
outcome.Stop, outcome.Exhausted)
}
}
// C4 — The allowance is sized from the grant, in one place.
func TestTheLandingAllowanceIsSizedFromTheGrant(t *testing.T) {
const grant = 100_000
allowance := landingAllowance(grant)
if allowance <= 0 || allowance >= grant {
t.Fatalf("allowance = %d for a %d grant, want a positive strict fraction", allowance, grant)
}
if got, want := landingAllowance(grant/10), allowance/10; got != want {
t.Fatalf("a tenth-sized grant got %d tokens, want a tenth of %d (%d)", got, allowance, want)
}
for _, grant := range []int{0, -1} {
if got := landingAllowance(grant); got != 0 {
t.Fatalf("landingAllowance(%d) = %d, want none", grant, got)
}
}
}
// The regression itself, driven through the real loop.
//
// A leaf billing at a 98% hit rate spends about a sixth of what it costs the
// provider to run it, so it can push a great many raw tokens through the model
// before its grant is gone — which is what the traces show, twice, at 1.4M and
// 1.6M on single nodes.
//
// THE MONEY IS WHAT ENDS IT, and this is where that claim is checked rather
// than asserted. The raw bound used to be what fired here and is now a warning
// (see meter.go on reuseCeiling for the measurement that demoted both Σ-of-
// prompt bounds). What must still hold is everything the audit actually cared
// about: the leaf ends well inside the turn backstop rather than at it, it ends
// far short of the 1.4M the traces show, it ends the graceful way with a
// landing that delivers what exists — and it ends having spent the grant, which
// is the only honest reason to stop a leaf that is being billed for every turn
// it takes.
func TestACacheDiscountedRunawayLandsOnItsMoney(t *testing.T) {
client := &warmRunaway{window: calibratedWindow(), hitPercent: 98}
linear := NewLinear(client, workspace(t), nil, maxTurnBackstop, DefaultLeafTokens, time.Hour)
outcome, err := linear.Run(context.Background(), Task{NodeID: 1, Brief: "work"})
if err != nil {
t.Fatal(err)
}
// Landing, not guillotine: the leaf was told to wrap up and delivered.
if outcome.Stop != StopBudget || outcome.Exhausted != StopBudget {
t.Fatalf("stop = %s, exhausted = %q — want the budget landing both times",
outcome.Stop, outcome.Exhausted)
}
if outcome.Text == "" {
t.Fatal("the landing delivered nothing; the raw bound must wrap up, not kill")
}
// The grant, not the turn cap, is what fired. If the cap is what caught it,
// the meter is not doing its job and every runaway costs the full backstop.
if outcome.Turns >= maxTurnBackstop {
t.Fatalf("ran %d turns of a %d backstop — the grant did not fire first",
outcome.Turns, maxTurnBackstop)
}
// And the grant held to its own arithmetic, plus the landing reserve it
// gives after crossing.
turn := warmTurn(calibratedWindow(), 98)
slack := (landingTurns + 1) * spentOfTurn(turn)
if spent(outcome) > DefaultLeafTokens+slack {
t.Fatalf("billed spend %d overran the %d grant by more than the %d its landing reserve costs",
spent(outcome), DefaultLeafTokens, slack)
}
// AND IT ENDED FAR SHORT OF THE AUDIT'S OWN CASES. The traces this test was
// written from show single nodes at 1.4M and 1.6M raw prompt tokens; a warm
// runaway that is billed for every turn runs out of money well before it
// reaches either, which is what makes the Σ-of-prompt ceiling that used to
// fire here redundant as well as blind.
if raw := rawSpent(outcome); raw >= 1_400_000 {
t.Fatalf("the runaway pushed %d raw tokens, reaching the audited 1.4M the grant is supposed to forestall", raw)
}
// The whole point, stated as the comparison the audit made: cost alone would
// have bought this leaf several times the work it just did.
costOnly := 0
usage := &Outcome{}
for costOnly < 10_000 && spent(usage) < DefaultLeafTokens {
turn := warmTurn(calibratedWindow(), 98)
usage.Usage.PromptTokens += turn.prompt
usage.Usage.CachedTokens += turn.cached
usage.Usage.CompletionTokens += turn.completion
costOnly++
}
if costOnly <= outcome.Turns {
t.Fatalf("the discounted ceiling bound at %d turns and the raw bound at %d — "+
"this test is not measuring what it claims to", costOnly, outcome.Turns)
}
}
// The bound must hold across the whole range of hit rates a provider can
// report, not just at the one the loop was measured at. However cheap the turns
// become, the leaf converges inside its raw allowance and inside the backstop —
// and at zero caching nothing changes at all, because cost still binds first.
func TestTheRawBoundHoldsAtEveryHitRate(t *testing.T) {
window := calibratedWindow()
cold := turnsUntilExhausted(DefaultLeafTokens, func(int) turnBilling {
return warmTurn(window, 0)
})
for _, hit := range []int{0, 50, 90, 95, 98, 100} {
turns := turnsUntilExhausted(DefaultLeafTokens, func(turn int) turnBilling {
if turn == 0 {
return warmTurn(window, 0)
}
return warmTurn(window, hit)
})
if turns >= maxTurnBackstop {
t.Fatalf("at a %d%% hit rate a leaf runs %d turns before either bound trips, "+
"past the %d backstop", hit, turns, maxTurnBackstop)
}
if turns < cold {
t.Fatalf("at a %d%% hit rate a leaf affords %d turns against %d cold — "+
"the discount must never cost a leaf turns", hit, turns, cold)
}
}
}
// The raw bound is a bound on convergence and never on cost: it may not shorten
// a leaf that is spending honestly. A cold leaf sees exactly the ceiling it was
// granted, because three times a number it cannot reach is not a limit.
//
// And the raw bound no longer LANDS anything: it reaches the leaf as the
// wrap-up warning, which is what a bound that cannot tell hard work from
// circling is worth. The measurement that demoted it is in meter.go against
// reuseCeiling, and the two are the same quantity.
func TestTheRawBoundWarnsAndOnlyTheCostCeilingLands(t *testing.T) {
cold := &Outcome{Usage: Usage{PromptTokens: DefaultLeafTokens - 1, CompletionTokens: 0}}
if exhausted(cold, DefaultLeafTokens) {
t.Fatal("a leaf one token short of its ceiling was called exhausted")
}
cold.Usage.CompletionTokens = 1
if !exhausted(cold, DefaultLeafTokens) {
t.Fatal("a leaf at its cost ceiling was not called exhausted")
}
// The same tokens, now reported as cache reads: cheap enough that cost says
// keep going, plentiful enough that the raw bound says land.
warm := &Outcome{Usage: Usage{
PromptTokens: rawCeiling(DefaultLeafTokens),
CachedTokens: rawCeiling(DefaultLeafTokens),
CompletionTokens: 0,
}}
if spent(warm) >= DefaultLeafTokens {
t.Fatalf("the discount stopped discounting: %d of %d", spent(warm), DefaultLeafTokens)
}
// AND IT IS NOT LANDED FOR IT. The raw ceiling was a stop until ink s9 of
// 2026-08-29 showed what a Σ-of-prompt bound actually measures — turns ×
// mean-context, which climbs identically for a leaf doing hard work and one
// circling. It is a warning now. See PERF.md, "A leaf's bounds".
if exhausted(warm, DefaultLeafTokens) {
t.Fatalf("a leaf inside its %d-token grant was landed at %d raw tokens; the money is the only meter that lands work",
DefaultLeafTokens, rawSpent(warm))
}
// And the wrap-up warning reaches it, on the bound it is actually near.
if budgetUsed(warm, DefaultLeafTokens, 0) <= wrapUpAt {
t.Fatalf("a leaf at its raw bound reads %.2f used, under the %.2f wrap-up mark",
budgetUsed(warm, DefaultLeafTokens, 0), wrapUpAt)
}
}
// The backstop's value is a policy, and this test is where the policy has to
// say so. It was 40, calibrated to the measured spread of well-sized leaves
// (8-16 turns, longest honest leaf 25) — and it also stopped honest complex
// work, which is the worse failure: the token budget is the meter that binds,
// and a turn count exists only for the runaway tokens cannot see (turns gone
// cheap on cache). The 2026-08-12 product decision: complex tasks are never
// stopped by an iteration counter. Four hundred sits an order of magnitude
// past the longest honest leaf while still ending a loop stuck on rails.
func TestTheTurnBackstopIsCalibratedToTheMeasuredSpread(t *testing.T) {
const longestHonestLeaf = 25
if maxTurnBackstop <= longestHonestLeaf*4 {
t.Fatalf("a %d-turn backstop crowds the %d-turn leaves the audit traced finishing honestly",
maxTurnBackstop, longestHonestLeaf)
}
if maxTurnBackstop < 300 || maxTurnBackstop > 500 {
t.Fatalf("the backstop is %d, outside the 300-500 the policy set; changing it is a policy change and says so here", maxTurnBackstop)
}
}
// The backstop is a backstop: it is the loop's own, and no caller may raise it.
func TestTheTurnBackstopCannotBeRaisedByACaller(t *testing.T) {
if got := NewLinear(nil, nil, nil, 1000, DefaultLeafTokens, time.Minute).maxTurns; got != maxTurnBackstop {
t.Fatalf("a caller asking for 1000 turns got %d, want the %d backstop", got, maxTurnBackstop)
}
if got := NewLinear(nil, nil, nil, 0, DefaultLeafTokens, time.Minute).maxTurns; got != maxTurnBackstop {
t.Fatalf("an unset turn count got %d, want the %d backstop", got, maxTurnBackstop)
}
if got := NewLinear(nil, nil, nil, 200, DefaultLeafTokens, time.Minute).maxTurns; got != 200 {
t.Fatalf("a caller asking for 200 turns under the backstop got %d", got)
}
if got := NewLinear(nil, nil, nil, 4, DefaultLeafTokens, time.Minute).maxTurns; got != 4 {
t.Fatalf("a caller asking for a tighter 4 turns got %d", got)
}
}
// spentOfTurn is one warm turn priced the way the grant prices it, so a test
// about the grant's slack is written in the grant's own unit rather than in raw
// tokens that the discount makes incomparable.
func spentOfTurn(turn turnBilling) int {
cached := turn.cached
if cached > turn.prompt {
cached = turn.prompt
}
return turn.prompt - cached + cached*cachedTokenWeightPercent/100 + turn.completion
}
// THE METER IS READ AT LAND, NOT AT THE GRANT.
//
// Both live bounds are read when the landing reserve is handed out, and then the
// landing turns run — more model calls, more seconds. Every ⏳ line and every
// stored leaf_exhausted.reached therefore reported the leaf's spend one turn
// before it stopped: recomputed from one run's own usage rows, 172,791 tokens
// were written down as 152,090 and 199,131 as 178,086, an under-report of 12-14%
// on the one figure an autopsy of a runaway leaf is made of.
func TestTheExhaustionMeterIsReadWhenTheLeafActuallyStops(t *testing.T) {
client := &warmRunaway{window: calibratedWindow(), hitPercent: 98}
linear := NewLinear(client, workspace(t), nil, maxTurnBackstop, DefaultLeafTokens, time.Hour)
outcome, err := linear.Run(context.Background(), Task{NodeID: 1, Brief: "work"})
if err != nil {
t.Fatal(err)
}
if outcome.Meter.Name != MeterCost {
t.Fatalf("the leaf was landed by %q, so this test is not measuring what it claims to",
outcome.Meter.Name)
}
if outcome.Meter.Reached != spent(outcome) {
t.Fatalf("the record says the leaf reached %d and it spent %d — the reading was taken "+
"when the landing was granted, not when the leaf stopped",
outcome.Meter.Reached, spent(outcome))
}
// And the landing really did cost something, or the two figures would agree
// whether or not anything was recomputed.
if landed := spent(outcome) - outcome.Meter.Allowed; landed <= 0 {
t.Fatalf("the landing reserve billed nothing (%d spent against a %d grant), so the "+
"two readings cannot differ", spent(outcome), outcome.Meter.Allowed)
}
}