Repository navigation
Expand file tree
/
Copy pathdispatch.go
More file actions
1559 lines (1516 loc) · 72 KB
/
Copy pathdispatch.go
File metadata and controls
1559 lines (1516 loc) · 72 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
package provider
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"math/rand"
"net/http"
"net/http/httptrace"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/Agent-Field/agentfield/sdk/go/ai"
"github.com/Agent-Field/codeaf/internal/calllog"
lanes "github.com/Agent-Field/codeaf/internal/lane"
"github.com/Agent-Field/codeaf/internal/lane/control"
"github.com/Agent-Field/codeaf/internal/trace"
"github.com/Agent-Field/codeaf/internal/providerhttp"
)
// waitContext is in media.go: one timer, stopped on the way out, so a retry
// loop that returns early does not leave a timer behind for the length of the
// backoff it abandoned.
// ── THE DISPATCHER: ONE LOOP, ONE BUDGET, ONE OWNER ─────────────────────────
//
// This is the one place in this build that decides what a failed call does
// next, and the one place a request for a completion reaches the wire
// (`TestOneDispatcherSendsForACompletion`).
//
// WHAT IT REPLACED. Eleven
// controllers on one request path each owned a budget none of the others could
// see — three transport faults here, six or sixty paced sends here, two minutes
// or ten of pacing here, eight free moves here, four arms in hedge.go, seven
// rungs in endpoints.go, four attempts and two models in internal/session. Their
// product is nobody's number, and the call census of 2026-09-10 measured what it
// produced: chains of sixteen and seventeen identical sends to one machine,
// running eleven minutes, and ending refused anyway.
//
// THERE IS NOW ONE BOUND AND IT IS A DEADLINE IN THE PERSON'S OWN TIME
// ([control.Plan.Deadline], from `lane.Role.GiveUp` — ninety seconds for a
// conversation's turn, four and a half minutes for a task node). What comes next
// is [control.Next], a pure function whose order is the rule's order: another
// machine, the same machine once after the comeback it named itself when it is
// alone, a relaxed shape one rung at a time, none. Nothing under the plan owns a
// retry count, and none of the six constants this block used to hold exists.
const (
baseBackoff = 700 * time.Millisecond
maxErrorPeek = 8 << 10
// maxBackoffShift caps the exponent, not the patience. A patient call may
// take its hundredth attempt, and `1 << 99` is not a long wait — it is an
// overflow, and an overflowed duration is a negative one. Seven doublings
// already put the computed delay past maxProviderWait, which is where every
// wait lands anyway, so clamping here changes nothing about a bounded call
// and makes an unbounded one arithmetic rather than undefined.
maxBackoffShift = 8
// maxProviderWait caps what a Retry-After may ask for. The header may be an
// HTTP date, and a provider that names tomorrow morning is asking a call to
// sleep for hours inside a run the user is watching. A minute is already
// far past the point where waiting beats failing and saying so.
maxProviderWait = time.Minute
)
// ── THE BODY IS WRITTEN PER ATTEMPT, NOT PER CALL ───────────────────────────
//
// THE MEASURED FAILURE (the census, the implementation §6).
// This loop used to be handed one encoded body above it and send those same
// bytes again on every attempt. The bytes carry `provider.order`, `provider.only`
// and `provider.ignore`, so "again" meant "to the same machine": 53 % of the
// multi-attempt chains in ten days never left the lane they started on, three of
// them spent sixteen and seventeen consecutive sends on one machine over eleven
// minutes, and every one of those ended 429 anyway. The chains that DID escape
// escaped on the one attempt that finally moved.
//
// THE LAW: A MACHINE THAT REFUSED THIS CALL IS NOT ASKED AGAIN WHILE ANOTHER ONE
// IS ADMISSIBLE. Each attempt re-encodes ([Client.encodeRequest]), and the
// encoder reads this set through [Client.dropRefusedHere], so the exclusion
// travels the same plumbing every other routing preference does rather than as a
// second opinion beside it. The same machine is re-asked only when the serving
// set under this request is one machine wide — a person's strict pin, a rescue's
// demand, an account-wide pacing that names nobody — and then only after the
// wait it asked for itself.
// refusedHere is every machine that has refused THIS CALL, and it is the whole
// of what makes the next attempt a different request rather than the same one.
//
// IT IS THE CALL'S OWN MEMORY AND NOT THE LEDGER'S. The ledger
// ([velocityLedger.pace]) already holds what the process believes about a
// machine, on its own cooldown, shared by every session in the process; this is
// narrower and shorter-lived — one call's list of who has already said no to it
// — and it is what keeps a machine out of the very next body even when the
// ledger declined to write anything at all (an upstream that named no wait, a
// stream that died before naming its server, a 4xx over a list).
//
// It is carried on [callKnobs] by pointer because the knobs travel by value
// through the repair, relax and ladder chains, and what must not be copied is
// the answer to "who has already refused this call".
type refusedHere struct {
mu sync.Mutex
names []string
}
// add records one machine and reports whether it was new. An empty name is a
// refusal that implicated nobody and records nothing — the attribution law the
// ledger keeps, kept here for the same reason.
func (r *refusedHere) add(name string) bool {
name = strings.TrimSpace(name)
if r == nil || name == "" {
return false
}
r.mu.Lock()
defer r.mu.Unlock()
for _, held := range r.names {
if equalLane(held, name) {
return false
}
}
r.names = append(r.names, name)
return true
}
// count is how many machines have refused this call.
func (r *refusedHere) count() int {
if r == nil {
return 0
}
r.mu.Lock()
defer r.mu.Unlock()
return len(r.names)
}
// list is a copy of what has refused so far, empty when nothing has.
func (r *refusedHere) list() []string {
if r == nil {
return nil
}
r.mu.Lock()
defer r.mu.Unlock()
if len(r.names) == 0 {
return nil
}
return append([]string(nil), r.names...)
}
// ── THE CALL'S ONE PLAN ─────────────────────────────────────────────────────
type callPlanKey struct{}
// dispatchNow is the clock the DEADLINE is read against, and it is deliberately
// the waiting controller's rather than [Client.clock].
//
// A deadline is an account of how long a PERSON has been waiting, which is the
// same argument [waitNow] already makes about the hazard: [Client.clock] is a
// measurement seam that a test fills with a scripted list of instants, and a
// deadline read off one would be this build deciding the shape of a wait nobody
// was having. It is a var rather than a func so a scenario can state ninety
// seconds without spending ninety of them; nothing in production replaces it.
var dispatchNow = waitNow
// withCallPlan puts one question's plan on the context so every send under it
// reads the SAME budget — the same deadline, and the same list of what has been
// tried. A race stamps it once and every arm inherits it, which is what makes
// two arms unable to demand one machine.
func withCallPlan(ctx context.Context, plan control.Plan) context.Context {
if ctx == nil {
ctx = context.Background()
}
return context.WithValue(ctx, callPlanKey{}, plan)
}
// callPlanFrom is the plan this call is being made under, and whether anybody
// built one.
func callPlanFrom(ctx context.Context) (control.Plan, bool) {
if ctx == nil {
return control.Plan{}, false
}
plan, ok := ctx.Value(callPlanKey{}).(control.Plan)
return plan, ok
}
// dispatchPlan is the budget THIS send runs under.
//
// It is the question's plan where there is one — a race built it, every arm
// shares its deadline and its moves — narrowed to what this particular request
// may do: the machines it may go to ([requestSet], which honours a demand), the
// relaxation rungs it actually has, and the model on it.
//
// AND A CALL NOBODY BUILT A PLAN FOR STILL GETS ONE. A non-streamed completion,
// a document parse, a build with no controller at all: each is a question of its
// own, and a question with no deadline is the state this whole wave exists to
// delete. It is built through `lane.PlanFor`, which is the ONE place a plan is
// built, so the deadline is the role's there as it is everywhere.
func (c *Client) dispatchPlan(ctx context.Context, request *ai.Request, knobs callKnobs) control.Plan {
model := c.modelFor(request)
plan, held := callPlanFrom(ctx)
if !held {
plan = lanes.PlanFor(lanes.Choice{}, lanes.Pace{}, RoleFrom(ctx), dispatchNow())
}
plan.Model = model
// THE SET IS THE REQUEST'S OWN AND NOT THE CHOICE'S. A rescue's demand and a
// person's strict pin are both one machine wide however many the frontier
// named, and that is the whole of what makes the single legal repeat legal.
// An empty answer is an OPEN set — the router picks — which is a different
// state from a set of one and is read as one by [control.Next].
if set := requestSet(knobs); len(set) > 0 {
plan.Lane, plan.Alts = strings.TrimSpace(set[0]), nil
for _, lane := range set[1:] {
plan.Alts = append(plan.Alts, control.Alternative{Lane: strings.TrimSpace(lane)})
}
} else {
plan.Lane, plan.Alts = "", nil
}
// AND THE RUNGS ARE THIS REQUEST'S OWN. A rung for a field the request never
// carried is not a move, it is the same request sent twice — so a text-only
// call with no cap and no knob has no ladder at all, and [control.Next] must
// not offer it one (endpoints.go's [Client.relaxationPlan] is the authority
// and this reads it rather than counting rungs a second time).
plan.Shapes = nil
for _, step := range c.relaxationPlan(request, knobs, model) {
plan.Shapes = append(plan.Shapes, step.label)
}
if plan.Moves == nil {
plan.Moves = control.NewMoveLog()
}
if plan.Deadline.IsZero() {
plan.Deadline = dispatchNow().Add(RoleFrom(ctx).GiveUp())
}
return plan
}
// send performs one request with retries, and returns a response whose body has
// not been read.
//
// The request is rebuilt on each attempt rather than reused: its body is a
// reader, and a retried request carrying a drained reader would silently post an
// empty document. AND ITS BYTES ARE REBUILT WITH IT — see THE BODY IS WRITTEN
// PER ATTEMPT above.
func (c *Client) send(ctx context.Context, request *ai.Request, knobs callKnobs, body []byte, stream bool) (*http.Response, error) {
// ANYTHING A PERSON IS STILL OWED IS SAID HERE, ON THE VERY NEXT BODY THAT
// GOES OUT ON A STREAM THEY ARE READING (lanepin.go's [tellRetiredPins],
// prefcarry.go's [tellUncarriedPins], routefirst.go's [tellTakeover]). Each
// is one nil check on a call nobody is reading and on every call after its
// sentence has been said.
//
// IT IS HERE AND NOT AT THE SHAPED DOOR ABOVE IT, and the move is the fix.
// [Client.sendShaped] is entered once per call, so a fact the call learned
// FROM ITS OWN REFUSAL had already gone past the only place that says it:
// the widened retry a retired pin earns goes out through
// [Client.attemptShaped], and every rung of the relaxation ladder likewise,
// and none of them come back through the shaped door. Measured on
// 2026-09-13: a turn demanded a machine the account excludes, collected the
// router's refusal, retired the person's pin, and answered from another
// machine — and the sentence saying so was queued behind a door the rest of
// that turn never opened again. This is the door every BODY passes through,
// which is the grain the promise was written at.
tellRetiredPins(ctx)
tellUncarriedPins(ctx)
tellTakeover(ctx)
// OBSERVED SENDS ARE COUNTED PER Do, SO THE DOOR MUST BE ONE THIS FOUNDATION
// CAN COUNT (attemptobserver.go). Unobserved, this is one context lookup.
observer := actualObserverFrom(ctx)
if observer != nil {
if err := c.RequireActualAttemptCoverage(); err != nil {
return nil, err
}
// The attempt's identity is the trace's per-send ID; a send without a
// trace (the document parser) has none to name.
if knobs.trace == nil {
return nil, &UnsupportedAttemptSurfaceError{SurfaceDocument, "this send carries no attempt identity"}
}
}
var lastErr error
// The wait is sized for the reply the request PERMITS — the caller's answer
// plus the thinking pass's room — and not for the caller's figure alone.
// Sized from the figure, an always-thinking model was allowed to generate
// for longer than the transport would wait, and the empty-answer failure
// came back as a timeout.
ceiling, _ := c.ceilingFor(request, knobs)
httpClient := c.clientFor(c.modelFor(request), stream, ceiling)
if observer != nil {
// A followed redirect is a second paid send that no single Do can see, so
// an observed send hands the 3xx back instead (it is then an ordinary error).
httpClient.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }
}
// The address is part of the bound billing door. It changes only for the
// one separately authorised plan overflow below; every ordinary retry and
// endpoint move stays on the address the service was connected to.
currentBase := c.config.BaseURL
currentDoor := c.config.BillingDoor
usingOverflow := false
switchedDoor := false
// providerWait is the provider's own comeback instruction from the last
// 429 (Retry-After); it outranks our computed backoff for the one attempt
// it was issued for, and is then spent.
var providerWait time.Duration
// Whether this call waits pacing out or gives up on it, and who is told
// while it waits (patience.go). Both ride the context because the adapter
// underneath is shared by every agent in the process.
patient := patientRateLimits(ctx)
notice := pacingNoticeFrom(ctx)
// parked says this call has already announced that it is waiting on the
// provider. It is announced ONCE per call and taken back on EVERY exit —
// through, or given up — because a surface left holding "still waiting"
// for a call that has landed is worse than one that was never told.
parked := false
// park is THE ONE SITE that says whether this call is waiting a provider's
// pacing out, and it says it to both readers of that fact at once: the older
// bool above, and the phase on the seam that carries everything else about
// this call's life (callprogress.go's [CallPaced]). One site, so the two
// cannot disagree — which is the only reason the older spelling is still
// safe to leave standing while its one caller is moved over.
park := func(on bool) {
if parked == on {
return
}
parked = on
if notice != nil {
notice(on)
}
streamWatchFrom(ctx).callPaced(on, dispatchNow())
}
defer func() { park(false) }()
// pacedSince is when this call FIRST drew a 429, and zero until it does. It
// is what tells [PhasePaced] from [PhaseRetrying] — whose fault the wait is —
// and nothing else: the budget it used to be measured against is gone, and
// the deadline it is measured against now belongs to the plan.
var pacedSince time.Time
attempts := 0
var recoveryCtx context.Context
reconnected := false
// connectionPasses chooses the backoff after a reachability check answered
// but the send still failed. It is not a budget: [control.Plan.Deadline] is
// the one thing that ends this call, exactly as it is for every other fault.
connectionPasses := 0
// moved says the last attempt took a machine OFF the next body, so the next
// send is a different request to a different machine and owes nobody a wait.
// See the branch it governs below: a backoff is what we pay to ask the same
// machine again, and it is the only thing it is for.
moved := false
// accountPaced says the last refusal was the ACCOUNT'S OWN CEILING: this base
// has a pool behind the model, it asked the whole key to slow down, and it
// named none of that pool while doing it. There is nothing to put on the next
// body and every machine is behind the same ceiling, which is the move
// generator's ([control.Plan.AccountRefused]).
accountPaced := false
// sentVeto is EVERY MACHINE THAT HAD ALREADY REFUSED THIS CALL when the body
// now in flight was encoded — the exact list [Client.dropRefusedHere] read to
// compose that body.
//
// IT IS NOT ALWAYS THE `ignore` LIST THE BYTES CARRY, and the difference is
// deliberate rather than an approximation. A demanded name is never also
// vetoed, and [velocityLedger.keepTheSetServable] releases one refused name
// from the veto when the vetoes would otherwise empty the set. In both cases
// a machine on this list is one the body asked for again ON PURPOSE — not
// because it might have been forgiven, but because this process had nowhere
// else to send the request. So a refusal from it means the same thing the
// failed veto means, which is why the law below reads this list rather than
// the object: there is no other endpoint to reach.
//
// IT IS SNAPSHOTTED AT THE ENCODE AND NEVER READ BACK AT THE REFUSAL. The
// list is shared by every arm of one question ([refusedHere]), so a sibling
// arm adding a name while this body was on the wire would make it look as
// though this body had asked for a machine it never named.
//
// IT STARTS AT WHAT THE CALLER'S OWN ENCODE SAW, because a recovered send
// re-enters this loop at attempt 0 with a body the ladder already composed
// against these same names ([Client.sendRecovered]); starting empty cost the
// law one extra send to the same machine on every rung.
var sentVeto []string
if knobs.refused != nil {
sentVeto = knobs.refused.list()
}
// move is what the plan says to do next, and it is empty on the first pass
// because the first send is not a recovery from anything.
var move control.Move
// The body this call is carrying, kept ONLY when somebody asked for it: the
// old bodies pin, which puts it on the line of the model-call log, or the
// debug record, which is where bodies are moving to (calllog.go). On every
// ordinary run this is nil and the person's prompts never leave the process.
if knobs.trace != nil && (calllog.Bodies() || trace.For(ctx) != nil) {
knobs.trace.body = body
}
// ── THE ONE BUDGET ──────────────────────────────────────────────────────
//
// Rate limits are not faults — they are the provider pacing us, and
// abandoning work over pacing is the one outcome the concurrency doctrine
// forbids — but that is now a statement about WHICH MOVE comes next and no
// longer about how many attempts of a private count are left. The plan is
// the whole of how long, [control.Next] the whole of what, and neither of
// them is a number this file holds.
plan := c.dispatchPlan(ctx, request, knobs)
// ── HOW MANY MACHINES THERE ARE, AS HONESTLY AS THIS PROCESS CAN SAY ────
//
// It used to be the attempt ceiling — `2 of 6`, where six was a statement
// about patience and not about anything a person could count — so a walk of
// three machines read as though half of something was left over.
//
// THE SET IS THE ANSWER WHERE THERE IS ONE: a demand, or the candidates the
// chooser drew. Where there is none the pool belongs to the router and its
// width is genuinely unknown to us, so the honest denominator is what this
// CALL has met — the machines that have refused it, plus the one it is about
// to ask. It grows as the walk discovers the pool, which is the truth: "the
// third of the three I know about" is a real thing to say, and `3 of 6` is
// not.
width := func() int {
if named := len(plan.Serving()); named > 0 {
return named
}
return knobs.refused.count() + 1
}
// ── A WAIT WE ASKED FOR IS SPENT WHETHER OR NOT THE CLOCK MOVED ─────────
//
// [Client.wait] is a seam, and what a test usually fills it with is a
// function that returns at once: that is how a scenario states a
// two-minute backoff without spending two minutes. Under an attempt count
// that was harmless, because the count was what ended the call. Under a
// deadline it is not: a seam that makes waiting free makes the deadline
// unreachable, and the loop it bounds becomes the unbounded one this wave
// exists to delete.
//
// So the dispatcher charges itself for what it ASKED for. `owed` is the
// part of a wait the clock did not actually take, and it is added to the
// moment the deadline is read against. In production it is a rounding
// error; in a scenario it is the whole of the fiction, honestly kept.
var owed time.Duration
spentAt := func() time.Time { return dispatchNow().Add(owed) }
for attempt := 0; ; attempt++ {
if attempt > 0 {
// AND THE DEADLINE IS ASKED FIRST, because it is the only thing that
// ends a call. `outOfPatience` used to answer here from two
// currencies nobody above could see — six attempts or two minutes
// for a watched call, sixty or ten for a patient one — and it is
// deleted with them.
if plan.Spent(spentAt()) {
break
}
// TWO PASSES ALREADY OWE THE MOVE THEY ARE MAKING, and neither may ask
// the generator for a second one.
//
// A billing-door switch is one: it must reach the authorised address
// before the endpoint controller is asked for another move, or a plan
// pause could be turned into a lane walk or an early end without ever
// trying the metered door.
//
// A reachability check that answered after a pre-send failure is the
// other: it names no machine and earns no routing move. The next send
// stays on this road; after the first recovered pass it pays the
// ordinary fault backoff below. Everything else asks the one move
// generator as before.
if !switchedDoor && !reconnected {
// What the last refusal asked us to wait, and whether it was the
// account's own ceiling rather than a machine's: the two inputs the
// move generator needs that change between moves. A ceiling over the
// whole key leaves nothing to put on the next body, which is what
// stops an open set pretending it has somewhere to go
// ([control.Plan.AccountRefused]).
plan.Comeback = providerWait
plan.AccountRefused = accountPaced
move = control.Next(plan, plan.Moves.List())
if move.Kind == control.MoveNone {
break
}
// A SHAPE IS NOT THIS LOOP'S MOVE TO MAKE. The ladder that takes a
// field off the request is endpoints.go's and it is entered from
// above ([Client.sendRecovered]), because only the layer holding the
// refusal's own body can say which rung it earned. This loop hands
// the refusal back and that layer climbs.
if move.Kind == control.MoveShape {
break
}
plan.Moves.Add(move)
}
}
attempts++
// What the CALL has spent, for the row the completed answer writes at
// the end of it (calllog.go). The count lives on the knobs rather than
// here because a call that is repaired or relaxed comes back through
// this loop with a new body and the same trace.
if knobs.trace != nil {
knobs.trace.attemptBase = currentBase
knobs.trace.attemptBilling = c.attemptBilling(usingOverflow)
knobs.trace.attemptBilled = true
}
knobs.trace.begin()
if attempt > 0 && switchedDoor {
// A BILLING-DOOR SWITCH IS NEITHER A RECONNECTION NOR A RETRY. The
// person's opt-in already chose the next address, so it goes out at once
// and the phase clock's door is the only status change owed here. It is
// asked BEFORE the recovered-pass branch below for exactly that reason:
// a door switch that fell through to it would be narrated as recovering
// from a dropped connection and would sit out a backoff it does not owe.
providerWait = 0
} else if attempt > 0 && reconnected && connectionPasses > 1 {
// THE FIRST RECOVERED PASS IS IMMEDIATE. Every later one means the
// check answered while the request itself still could not leave, which
// is another fault and pays the same growing pause as every other one.
// The wait is charged to the plan even when a test makes waiting free,
// so a fast-answering portal cannot make the deadline unreachable.
delay := backoffFor(connectionPasses-1, 0)
if left := plan.Left(spentAt()); left > 0 && delay > left {
delay = left
}
now := c.clock()
notePhase(ctx, c.modelFor(request), PhaseRetrying,
"", now, now.Add(delay), "")
waitBegan := dispatchNow()
if err := c.wait(ctx, delay); err != nil {
return nil, err
}
if took := dispatchNow().Sub(waitBegan); took < delay {
owed += delay - took
}
if plan.Spent(spentAt()) {
break
}
} else if attempt > 0 && !reconnected && moved {
// A MOVE IS NOT A WAIT. The last attempt excluded the machine that
// refused it, so the body about to go out is a different request to a
// different machine — and sitting out a backoff first would be this
// call serving a sentence another machine handed down. A backoff is
// for asking the SAME machine again, which is the next branch.
//
// The person is told all the same, in the ordinal, with no moment
// attached: there is nothing to count down to, because nothing is
// being waited for.
//
// AND THE ORDINAL'S DENOMINATOR IS REAL. It used to be the attempt
// ceiling — `2 of 6` where six was a constant about patience — so a
// walk of three machines read as though half of something was left.
// It is how many machines this request may actually go to
// ([control.Plan.Serving]), and a set nobody named draws nothing
// rather than a number it cannot stand behind.
now := c.clock()
notePhase(ctx, c.modelFor(request), PhaseRetrying,
ordinalOf(attempts, width()), now, now, "")
providerWait = 0
} else if attempt > 0 && !reconnected {
// AND THE WAIT THE MACHINE ITSELF NAMED IS THE WHOLE WAIT, not a
// floor under our doubling. A [control.MoveWait] is the one legal
// repeat there is — the same bytes to the same machine, because the
// set is one wide or the ceiling is over the whole key — and the
// comeback on it is a fact the answer carried, not a guess. Passing
// it through [backoffFor] made it the LARGER of what was asked for
// and 2^(attempt-1) rungs of a doubling that exists for the case
// where nobody asked for anything: a comeback of three seconds met
// on the fourth attempt was served as eleven, and a person who had
// been told `back in 3s` watched a longer clock.
//
// The cap is still the cap: [maxProviderWait] bounds one wait
// whatever asked for it, which is what keeps an interrupt prompt.
// A CALL THAT HANDS A LIMIT BACK DOES NOT SIT IT OUT (patience.go's
// [WithoutPatientRateLimits]): its caller's next move is another
// route or another model, which beats any window.
if handsBackRateLimits(ctx, lastErr) {
return nil, lastErr
}
delay := backoffFor(attempt, providerWait)
if move.Kind == control.MoveWait && move.Wait > 0 {
delay = min(move.Wait, maxProviderWait)
}
// AND IT IS NEVER LONGER THAN WHAT IS LEFT OF THE CALL. A wait that
// outlives the deadline is a person watching a countdown to a moment
// this build has already decided not to reach.
if left := plan.Left(spentAt()); left > 0 && delay > left {
delay = left
}
// AND A PERSON IS TOLD HOW LONG, WHICH IS THE ONE FACT THIS LOOP
// HAD AND THREW AWAY. Until the phase clock, a conversation parked
// on a rate limit for two minutes said nothing at all: the only
// seam out of here was [WithPacingNotice], a bare bool, and it is
// nil on every chat session (internal/session's agent.go — only a
// task node sets one). The deadline here is REAL — it is the moment
// this request goes out again — which is what makes it something a
// countdown may be drawn from (phase.go).
// PACED OR MERELY TRYING AGAIN, and the difference is whose fault
// the wait is. A rate limit is the router asking us to slow down
// and it is the one wait with a moment attached to it, so it gets
// its own word; everything else here is a fault we are re-asking
// after. [pacedSince] is set on the first 429 of the call and is
// the honest reading of "this call is being paced", where the
// header's own figure is spent after one attempt.
paced := PhaseRetrying
if !pacedSince.IsZero() {
paced = PhasePaced
}
now := c.clock()
notePhase(ctx, c.modelFor(request), paced,
ordinalOf(attempts, width()), now, now.Add(delay), "")
// Spent. It described one moment to come back at, and coming back
// is what we are doing; carrying it forward made a single 429 set
// the floor for every remaining attempt of the call.
providerWait = 0
waitBegan := dispatchNow()
if err := c.wait(ctx, delay); err != nil {
return nil, err
}
if took := dispatchNow().Sub(waitBegan); took < delay {
owed += delay - took
}
}
reconnected, moved, switchedDoor = false, false, false
if waited, err := c.waitConnection(ctx, c.modelFor(request), currentBase, false); err != nil {
return nil, err
} else if waited && knobs.trace != nil {
knobs.trace.connectionRecovered = true
}
// A stream gets its own cancellable context so the idle watchdog has
// something to pull. Every path that does not hand the body back to the
// caller releases it here; the path that does hands the cancel to the
// watchdog, which fires it on stall or on Close.
attemptCtx, cancelAttempt := attemptContext(ctx, stream)
var sent atomic.Bool
// AND THE CONNECTION IS MEASURED AS WELL AS THE ANSWER. The hooks cost
// nothing and they are the only thing that can separate a cold pool from
// a slow model in `ttft_ms` — a person who read an answer for two minutes
// and then typed again used to pay a whole handshake inside this Do, and
// the log filed every millisecond of it as the endpoint being slow
// (conntrace.go).
attemptCtx = httptrace.WithClientTrace(attemptCtx, knobs.trace.openConn(&sent))
// AND THE BYTES ARE WRITTEN HERE, immediately before the send, which is
// the same timing [Client.providerPreferences] exists for: the machine
// that refused a moment ago is off this body, and the ledger's own
// cooldowns are read as they stand now rather than as they stood when the
// call began. A caller that composed its body itself — the document path,
// which posts a multipart form rather than a completion — carries no
// refusal set and keeps the bytes it was given.
if attempt > 0 && knobs.refused != nil {
sentVeto = knobs.refused.list()
written, err := c.encodeRequest(request, knobs)
if err != nil {
cancelAttempt()
return nil, fmt.Errorf("marshal request: %w", err)
}
body = written
if knobs.trace != nil && knobs.trace.body != nil {
knobs.trace.body = body
}
}
httpRequest, err := c.newHTTPRequestAt(attemptCtx, request, body, stream, currentBase)
if err != nil {
cancelAttempt()
return nil, err
}
if err := sharedLimiter.acquire(ctx); err != nil {
cancelAttempt()
return nil, err
}
// Recheck after every wait, including the limiter, because another call
// may have spent the daily allowance while this attempt was queued.
if gate := c.config.RouteGate; gate != nil {
if err := gate(ctx, c.modelFor(request), callTag(ctx)); err != nil {
sharedLimiter.release(false, 0)
cancelAttempt()
return nil, &AttemptDeniedError{Cause: err}
}
}
// THE ONE PLACE A SEND IS ADMITTED: after the request is built and the
// limiter has let it through, and before anything is written to the log
// or the watchdog, so a refusal leaves no trace of a send that never was.
var handle *attemptHandle
if observer != nil {
var err error
if handle, err = c.admitAttempt(ctx, observer, request, knobs, usingOverflow); err != nil {
sharedLimiter.release(false, 0)
cancelAttempt()
return nil, err
}
}
// AND THIS ATTEMPT GETS ITS OWN CONTROLLER. Every send of every request is
// watched, and it is watched from the moment its own bytes leave — see
// EVERY ATTEMPT IS WATCHED in armwatch.go for the 363-second attempt whose
// row said nothing because its arm's first attempt had already spoken.
streamWatchFrom(ctx).attempt(dispatchNow())
attemptBegan := logNow()
// THE ROW THAT SAYS A CALL IS IN FLIGHT, written before the wait rather
// than after it. Without it a planning call four minutes into a
// 65,536-token ceiling is indistinguishable from an idle process: the
// log's newest line is the call BEFORE it, and everything a person can
// see says "nothing since four minutes ago". Its partner is whichever
// row ends this attempt, paired by the id they share.
c.record(recordFacts{
ctx: ctx, request: request, knobs: knobs, stream: stream,
attempt: attempts, began: attemptBegan, phase: calllog.PhaseStart,
})
response, err := providerhttp.Do(&httpClient, httpRequest)
if handle != nil {
handle.afterDo(ctx, err, response != nil && err == nil)
}
if err == nil && response != nil && response.StatusCode == http.StatusPaymentRequired {
// Every call shape passes this transport door, including reflexes,
// titles, tasks, and the affordable retry itself.
announcePaymentRequired(c.config.BaseURL)
}
// Custom transports may omit Request; retain the exact sent object so
// refusal learning never reconstructs it from mutable routing state.
if response != nil && response.Request == nil {
response.Request = httpRequest
}
if err != nil {
sharedLimiter.release(false, 0)
cancelAttempt()
// EVERY ATTEMPT THAT FAILED LEAVES A ROW, not only the last one. A
// call that was paced four times and then landed reads as one slow
// call in any surface above this; the four rows are the whole of
// why it was slow (calllog.go).
c.record(recordFacts{
ctx: ctx, request: request, knobs: knobs, stream: stream,
attempt: attempts, began: attemptBegan, err: err,
})
// A cancelled or expired parent is a decision, not a fault. Retrying
// it would burn the remaining deadline on calls that cannot land.
if ctx.Err() != nil {
if sent.Load() {
c.settle(ctx, c.modelFor(request), nil, receiptTornReason, 0, knobs.trace)
}
return nil, fmt.Errorf("execute request: %w", err)
}
// A TRANSPORT MAY KNOW THE CREDENTIAL IT OWNS IS FINISHED. That is
// not a reachability fault and cannot improve under this dispatcher's
// machine walk; preserve the typed cause for the session's person-facing
// sentence and return after the one request that established it.
if terminalTransportFailureFrom(err) || providerhttp.IsBoundary(err) {
return nil, fmt.Errorf("execute request: %w", err)
}
lastErr = fmt.Errorf("execute request: %w", err)
// AND A FAULT IS NOT A CEILING, whatever an earlier attempt of this
// call learned: the bytes never reached anybody, so nothing has been
// established about where they may go next.
accountPaced = false
// A TRANSPORT FAULT NAMES NOBODY, so nothing is excluded from the next
// body: the bytes never reached a machine that could be blamed, and
// writing one down would be this loop guessing. What the next attempt
// still gets is a fresh encode, which re-reads every cooldown the
// ledger has written since this call began.
if connectionFailure(err) && !sent.Load() {
if knobs.trace != nil {
knobs.trace.connectionRecovered = true
}
// A DNS or dial failure precedes accepted generation. Wait for
// the origin, not another provider behind that same origin. The
// dispatcher plan remains the one bound when the check answers but
// the send still fails.
if recoveryCtx == nil {
var cancelRecovery context.CancelFunc
left := plan.Left(spentAt())
if left <= 0 {
return nil, &ConnectionUnavailableError{}
}
recoveryCtx, cancelRecovery = context.WithTimeout(ctx, left)
defer cancelRecovery()
}
if _, waitErr := c.waitConnection(recoveryCtx, c.modelFor(request), httpRequest.URL.String(), true); waitErr != nil {
if ctx.Err() != nil {
return nil, ctx.Err()
}
if recoveryCtx.Err() != nil {
return nil, &ConnectionUnavailableError{}
}
return nil, waitErr
}
// A healthy check does not spend a machine or a request-shape move.
// The first recovered pass is immediate; a further failure waits
// at the top of the next pass under this plan's deadline.
connectionPasses++
reconnected = true
continue
}
connectionPasses = 0
if c.handBack(ctx, knobs, "", false) {
return nil, lastErr
}
// AND NOTHING BREAKS OUT HERE ANY MORE. `attempt >= maxAttempts-1`
// was the private three-fault budget; the deadline and
// [control.Next] are asked at the top of the next pass, and they are
// the whole of what ends a call. A transport fault names nobody, so
// the move it earns is an ordinary one and it is paid for with the
// backoff above — which is what stops a dead resolver being asked at
// connection speed for the length of the deadline.
continue
}
connectionPasses = 0
if observer != nil && response.StatusCode >= 300 && response.StatusCode < 400 {
// A refused redirect must never reach the content-type learner: an
// HTML redirect is not evidence that this client cannot stream.
err := &UnsupportedAttemptSurfaceError{SurfaceCompletion, "an observed completion cannot follow an HTTP redirect"}
_ = response.Body.Close()
sharedLimiter.release(false, 0)
cancelAttempt()
handle.unknown()
c.record(recordFacts{ctx: ctx, request: request, knobs: knobs, stream: stream,
attempt: attempts, began: attemptBegan, status: response.StatusCode, err: err})
return nil, err
}
potentialRateLimit := response.StatusCode == http.StatusTooManyRequests
// Error bodies can stall too. They are read below before a retry, so
// they need the same idle bound as successful streaming bodies.
if stream {
response.Body = newIdleWatchdog(response.Body, streamIdleTimeout, cancelAttempt)
}
// The provider's comeback instruction is read before the slot goes back,
// because it is what tells the limiter how wide this 429's window is:
// one window, one halving (limiter.go).
var named time.Duration
if potentialRateLimit {
named = retryAfter(response)
}
// ── A REFUSAL THAT NAMED A MACHINE IS THAT MACHINE'S, AND THE OTHERS
// HAVE SAID NOTHING ─────────────────────────────────────────────────
//
// An account-policy 404 and a 400 the
// router RELAYED from the endpoint it picked are neither a person's mistake nor a fact
// about the model: they are one machine saying no, and the answer to them is the same
// walk a 429 gets. Until this line they ended the call, and the rotation that eventually
// moved was the SESSION's — three whole turns apart, each learning the same fact over
// again through the ledger (refusal_test.go's
// `TestThreeAttemptsAfterAnUpstreamRefusalReach…`).
//
// A REFUSAL THAT NAMED NOBODY IS NOT THIS. That one is the router
// reading our own bytes and saying no, and every machine alive will say
// the same thing about the same request ([taxonomy.Evidence.OurBytes]);
// it is handed back whole, exactly as it always was. So is anything this
// adapter can repair by itself — a knob it guessed wrong about reaches
// [Client.sendRepaired] a layer up, and walking it would spend a machine
// to discover a fact the memo already answers.
var peek []byte
if !retryableStatus(response.StatusCode) {
if !relayedByAMachine(response.StatusCode) || c.repairable(c.modelFor(request), knobs) ||
len(knobs.reasoning) > 0 {
sharedLimiter.release(false, 0)
return response, nil
}
read, _ := io.ReadAll(io.LimitReader(response.Body, maxErrorPeek))
if upstream, ok := routerErrorEnvelope(read); !ok || upstream == "" {
response.Body = rewound(read, response.Body)
sharedLimiter.release(false, 0)
return response, nil
}
peek = read
}
// Drain a bounded prefix before closing so the connection can be reused
// and the eventual error still says what the provider complained about.
// A relayed refusal above has already read it and must not read it twice.
if peek == nil {
peek, _ = io.ReadAll(io.LimitReader(response.Body, maxErrorPeek))
}
// A context refusal needs a smaller request, even when the router names
// the machine that refused it. Return it to the conversation before an
// endpoint walk can resend the same oversized bytes. A model capability
// refusal also belongs to its one repair rather than an endpoint walk.
if failure, ok := RefusalFrom(apiError(response.StatusCode, peek)); ok && (failure.Overflow || failure.ReasoningUnsupported) {
response.Body = rewound(peek, response.Body)
sharedLimiter.release(false, 0)
return response, nil
}
response.Body.Close()
cancelAttempt()
// A refusal that is walked past carries no usage block, and "refused" is
// not "free": the send's price is unknown, never zero.
handle.unknown()
lastErr = apiError(response.StatusCode, peek)
// AND THE COMEBACK IS READ OUT OF THE BODY WHEN NO HEADER CARRIED IT. The
// census found `retry_after` recorded on not one row in ten days, while
// every fourth failure was a 429: the header is what [retryAfter] reads
// and this router routinely names its wait in the refusal envelope
// instead. A wait we cannot see is a wait we replace with our own
// doubling, which is how a pool that asked for three seconds was asked
// again after seven hundred milliseconds.
if potentialRateLimit && named <= 0 {
named = retryAfterIn(peek)
}
// EVERY REFUSAL THIS LOOP DRAWS GOES THROUGH THE ONE DOOR, and what is
// done about it is decided there from what the refusal says rather than
// here from the status this loop happens to be holding (velocity.go's
// [Client.refuseLane], which carries the measurement that law comes
// from). A 5xx naming its upstream takes that lane away; a 429 naming
// its pool is paced for the wait it asked for, which is why that wait
// is handed over rather than left in this loop; a 429 naming nobody is
// this account's own ceiling and writes nothing at all.
refusal := c.refuseUpstream(request, knobs, lastErr, "", named)
// ONLY ORDINARY PACING NARROWS THE SHARED LIMITER. A payment refusal, a
// plan pause and a door this key cannot use all arrive under 429 too, but
// none says the request rate is too high.
rateLimited := refusal.Kind == refusalPaced
if !rateLimited {
named = 0
}
sharedLimiter.release(rateLimited, named)
if rateLimited {
providerWait = named
if pacedSince.IsZero() {
pacedSince = time.Now()
}
// The park begins on the FIRST ordinary pacing refusal this call
// draws — and it is announced through [park], which is THE ONE SITE
// that tells both readers of that fact at once. Saying it by hand
// here would leave the call-watch seam believing this call never
// waited (#868).
park(true)
}
// AND THE MACHINE THAT REFUSED IS OFF THE NEXT BODY. This is the whole of
// "never repeat" (see THE BODY IS WRITTEN PER ATTEMPT above): the door
// above has already decided what the refusal MEANS for the process, and
// this decides what it means for the remaining attempts of this one call,
// which is the narrower and shorter-lived question. A refusal that
// implicated no machine adds nothing.
fresh := knobs.refused.add(refusal.Lane)
// AND A PACE THAT NAMED NONE OF A POOL IS THE KEY'S OWN CEILING. Both
// halves are needed and neither is the status alone: a base this process
// has never seen publish a set has one machine, and "come back later"
// from it is answered by coming back later ([Client.baseServesLanes]).
accountPaced = rateLimited && strings.TrimSpace(refusal.Lane) == "" && c.baseServesLanes()
// AND THE MOVE LOG IS CORRECTED TO THE MACHINE THAT ANSWERED. A move
// NAMES THE MACHINE WE EXPECT AND NEVER THE MACHINE WE COMMAND
// ([control.Move]) — `provider.order` is advisory once `allow_fallbacks`
// is on, and R1 measured this router fanning straight past it — so the
// machine a move claimed and the machine that refused are routinely two
// different pools. Writing the claim down and leaving it there tells
// [control.Next] that the lane we asked for has been tried when nothing
// was ever sent to it, which takes away the one machine we actually
// wanted on the evidence of a machine we did not.
//
// So the claim that never reached the wire is given back and the pool
// that really answered is written down in its place. That is exactly
// what [control.MoveLog.Release] is for, and it keeps "never repeat"
// about where the bytes WENT rather than about where we hoped they
// would go.
if served := strings.TrimSpace(refusal.Lane); served != "" {
asked := strings.TrimSpace(move.Lane)
if attempt == 0 {
asked = strings.TrimSpace(plan.Lane)
}
if asked != "" && !strings.EqualFold(asked, served) {
plan.Moves.Release(control.Move{
Kind: control.MoveMachine, Model: plan.Model,
Lane: asked, Shape: move.Shape,
})
}
plan.Moves.Add(control.Move{
Kind: control.MoveMachine, Model: plan.Model,
Lane: served, Shape: move.Shape,
})
}
// AND THE ROW IS WRITTEN FROM WHAT THE DOOR WORKED OUT, which is why it
// is here and not above the door.
//
// The classification is the only thing that can name the machine on a
// refusal that never opened a stream: `refusal.Lane` is the pool the
// router said was full, and it is the same name the ledger is now keyed
// on (velocity.go). Recording the row first and then classifying would
// be the two reading one refusal twice, which is the shape #123 closed —
// and the row would fall back to the demand on every refusal that named
// a machine we had not demanded.
//
// THE COMEBACK TIME GOES ON IT TOO, because it is the only thing that
// makes a repeated send to a refusing machine legal
//. It was recorded on not one row in
// the ten days to 2026-09-10, so eleven hundred paced refusals could not
// be checked against what the provider itself had asked for.
c.record(recordFacts{
ctx: ctx, request: request, knobs: knobs, stream: stream,
attempt: attempts, began: attemptBegan,
status: response.StatusCode, err: lastErr, responseBody: peek,
served: refusal.Lane, retryAfter: named,
})
// MONEY MOVES ONLY THROUGH THIS EXACT OPT-IN. A paused fixed-price window
// never reaches the endpoint walk or model chain: it either takes the one
// configured metered door, once, or returns its typed pause immediately.
if refusal.Kind == refusalPlanPaused {
detail := planPauseDetail(refusal.Reset, c.config.PlanOverflowDoor)
now := c.clock()
postPhase(PhaseNews{Phase: PhasePlanPaused, Detail: detail, Since: now, At: now,
Model: c.modelFor(request), SourceID: c.config.SourceID, PlanReset: refusal.Reset,
Role: RoleFrom(ctx), Session: SessionFrom(ctx), Subject: NodeFrom(ctx)})
useOverflow := c.config.OverflowOnPlanPause &&
strings.TrimSpace(c.config.PlanOverflow) != "" && !usingOverflow &&
claimPlanOverflow(ctx)
if useOverflow {
currentBase = strings.TrimRight(strings.TrimSpace(c.config.PlanOverflow), "/")
currentDoor = strings.TrimSpace(c.config.PlanOverflowDoor)
if phase := phaseClockFrom(ctx); phase != nil {
phase.useDoor(currentDoor)
}
usingOverflow = true
switchedDoor = true
pacedSince = time.Time{}
continue
}
return nil, &PlanPauseError{
Reset: refusal.Reset, OverflowDoor: c.config.PlanOverflowDoor, Cause: lastErr,
}
}