Repository navigation
Expand file tree
/
Copy pathmedia_test.go
More file actions
639 lines (604 loc) · 26.8 KB
/
Copy pathmedia_test.go
File metadata and controls
639 lines (604 loc) · 26.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
package exec
import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"image"
"image/color"
"image/jpeg"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"testing"
"github.com/Agent-Field/agentfield/sdk/go/ai"
"github.com/Agent-Field/codeaf/internal/provider"
)
type execRoundTripFunc func(*http.Request) (*http.Response, error)
func (fn execRoundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) {
return fn(request)
}
type visionWireCapture struct {
calls int
bodies []map[string]any
status int
response string
}
func (c *visionWireCapture) client(t *testing.T) *provider.Client {
t.Helper()
httpClient := &http.Client{Transport: execRoundTripFunc(func(request *http.Request) (*http.Response, error) {
payload, _ := io.ReadAll(request.Body)
var body map[string]any
if err := json.Unmarshal(payload, &body); err != nil {
t.Errorf("decode vision request: %v", err)
}
c.calls++
c.bodies = append(c.bodies, body)
status := c.status
if status == 0 {
status = http.StatusOK
}
response := c.response
if response == "" {
response = `{"model":"vendor/vision-model","choices":[{"index":0,"finish_reason":"stop","message":{"role":"assistant","content":"AgentField is centered in a blue card."}}],"usage":{"prompt_tokens":19,"completion_tokens":8,"cost":0.125}}`
}
return &http.Response{
StatusCode: status, Status: http.StatusText(status), Header: make(http.Header),
Body: io.NopCloser(strings.NewReader(response)), Request: request,
}, nil
})}
client, err := provider.NewClient(provider.Config{
APIKey: "test-key", BaseURL: "https://provider.example/v1", Model: "base/model", HTTPClient: httpClient,
})
if err != nil {
t.Fatal(err)
}
return client
}
type fakeMediaProvider struct {
imageCalls int
speechCalls int
musicCalls int
videoCalls int
imageRequest provider.ImageRequest
speechRequest provider.SpeechRequest
musicRequest provider.MusicRequest
videoRequest provider.VideoRequest
imageResponse *provider.ImageResponse
speechResponse *provider.SpeechResponse
musicResponse *provider.MusicResponse
videoResponse *provider.VideoResponse
imageErr error
speechErr error
musicErr error
videoErr error
}
func (f *fakeMediaProvider) GenerateImage(_ context.Context, request provider.ImageRequest) (*provider.ImageResponse, error) {
f.imageCalls++
f.imageRequest = request
return f.imageResponse, f.imageErr
}
func (f *fakeMediaProvider) Speak(_ context.Context, request provider.SpeechRequest) (*provider.SpeechResponse, error) {
f.speechCalls++
f.speechRequest = request
return f.speechResponse, f.speechErr
}
func (f *fakeMediaProvider) GenerateMusic(_ context.Context, request provider.MusicRequest) (*provider.MusicResponse, error) {
f.musicCalls++
f.musicRequest = request
return f.musicResponse, f.musicErr
}
func (f *fakeMediaProvider) GenerateVideo(_ context.Context, request provider.VideoRequest) (*provider.VideoResponse, error) {
f.videoCalls++
f.videoRequest = request
return f.videoResponse, f.videoErr
}
type fakeModalities map[string]bool
func (f fakeModalities) Supports(modelID, direction, modality string) bool {
return f[modelID+":"+direction+":"+modality]
}
func mediaToolbox(t *testing.T, provider MediaProvider, modalities ModalityCatalog) (*Toolbox, *Workspace) {
t.Helper()
space := workspace(t)
tools := newToolbox(space, "7", nil, nil, &MediaTools{
Provider: provider, Catalog: modalities, ImageModel: "paint/model",
SpeechModel: "voice/model", MusicModel: "music/model", VideoModel: "motion/model",
VideoPrice: 0.5, WorkingModel: "vision/model",
}, 0)
return tools, space
}
func TestGenerateImageWritesReadableNamesReferencesAndUsage(t *testing.T) {
cost := 0.42
fake := &fakeMediaProvider{imageResponse: &provider.ImageResponse{
Data: []provider.GeneratedImage{{Base64: base64.StdEncoding.EncodeToString([]byte("png")), MediaType: "image/png"}},
Usage: &ai.Usage{PromptTokens: 12, Cost: &cost},
}}
tools, space := mediaToolbox(t, fake, fakeModalities{})
reference := filepath.Join(space.Root(), "reference.png")
if err := os.WriteFile(reference, []byte("reference"), 0o644); err != nil {
t.Fatal(err)
}
result := tools.Execute(context.Background(), "generate_image", `{"prompt":"Sunset over Harbor!","n":1,"size":"16:9","reference_paths":["reference.png"]}`)
if result.IsError {
t.Fatal(result.Content)
}
if result.Usage.Calls != 1 || result.Usage.Cost != cost || result.Usage.PromptTokens != 12 {
t.Fatalf("usage = %+v", result.Usage)
}
want := filepath.Join("media", "sunset-over-harbor-1.png")
if _, ok := space.Locate(want); !ok || !strings.Contains(result.Content, filepath.ToSlash(want)) {
t.Fatalf("generated image missing: %q", result.Content)
}
// The reference rides in the image_url ENVELOPE, not as a bare string: the
// endpoint refuses a plain data URL with "expected object, received string".
if fake.imageRequest.AspectRatio != "16:9" || len(fake.imageRequest.InputReferences) != 1 ||
fake.imageRequest.InputReferences[0].Type != "image_url" ||
!strings.HasPrefix(fake.imageRequest.InputReferences[0].ImageURL.URL, "data:image/png;base64,") {
t.Fatalf("image request = %+v", fake.imageRequest)
}
if got := space.Artifacts("7"); len(got) != 1 || got[0] != want {
t.Fatalf("artifacts = %v", got)
}
}
func TestGenerateImageUsesTheSniffedExtensionWhenTheProviderLies(t *testing.T) {
var picture bytes.Buffer
canvas := image.NewRGBA(image.Rect(0, 0, 8, 8))
canvas.Set(0, 0, color.RGBA{R: 40, G: 120, B: 200, A: 255})
if err := jpeg.Encode(&picture, canvas, nil); err != nil {
t.Fatal(err)
}
fake := &fakeMediaProvider{imageResponse: &provider.ImageResponse{
Data: []provider.GeneratedImage{{
Base64: base64.StdEncoding.EncodeToString(picture.Bytes()),
MediaType: "image/png",
}},
}}
tools, space := mediaToolbox(t, fake, fakeModalities{})
result := tools.Execute(context.Background(), "generate_image", `{"prompt":"a plane"}`)
if result.IsError {
t.Fatal(result.Content)
}
if _, ok := space.Locate(filepath.Join("media", "a-plane-1.jpg")); !ok {
t.Fatalf("sniffed JPEG was not saved as .jpg: %q", result.Content)
}
}
func TestMediaToolsAreRegisteredAndHonorTheSpendGate(t *testing.T) {
fake := &fakeMediaProvider{imageResponse: &provider.ImageResponse{
Data: []provider.GeneratedImage{{Base64: base64.StdEncoding.EncodeToString([]byte("png")), MediaType: "image/png"}},
}}
tools, _ := mediaToolbox(t, fake, fakeModalities{})
// Configured is not the same as carried: a media provider on the brain
// puts one discovery tool in the prompt, and asking for the family is what
// puts the five schemas there.
if got := len(tools.Definitions()); got != 6 {
t.Fatalf("unarmed definitions = %d, want five base plus the discovery tool", got)
}
tools.Arm(FamilyMedia)
definitions := tools.Definitions()
// Eleven: five base, the door — which stays put rather than being pulled
// out of the middle of the block and shifting everything behind it — and the
// five media schemas appended at the tail.
if len(definitions) != 11 {
t.Fatalf("definitions = %d, want five base + the door + five media", len(definitions))
}
want := map[string]bool{"generate_image": false, "generate_music": false, "generate_video": false, "speak": false, "view_image": false}
for _, definition := range definitions {
if _, ok := want[definition.Function.Name]; ok {
want[definition.Function.Name] = true
}
}
for name, found := range want {
if !found {
t.Errorf("%s was not registered", name)
}
}
for _, definition := range definitions {
if definition.Function.Name != "view_image" {
continue
}
encoded, _ := json.Marshal(definition)
if !strings.Contains(definition.Function.Description, "vision model looks") || !strings.Contains(string(encoded), `"question"`) {
t.Fatalf("view_image definition = %s", encoded)
}
}
tools.media.BeforeSpend = func(context.Context, float64) error { return errors.New("rail") }
result := tools.Execute(context.Background(), "generate_image", `{"prompt":"harbor"}`)
if !result.IsError || !strings.Contains(result.Content, "paused at the daily budget") || result.Usage != (Usage{}) {
t.Fatalf("gated result = %+v", result)
}
music := tools.Execute(context.Background(), "generate_music", `{"prompt":"harbor song"}`)
video := tools.Execute(context.Background(), "generate_video", `{"prompt":"harbor motion"}`)
if !music.IsError || !video.IsError || fake.imageCalls != 0 || fake.speechCalls != 0 ||
fake.musicCalls != 0 || fake.videoCalls != 0 {
t.Fatalf("generation escaped gate: image=%d speech=%d music=%d video=%d music_result=%+v video_result=%+v",
fake.imageCalls, fake.speechCalls, fake.musicCalls, fake.videoCalls, music, video)
}
}
// MUSIC IS ITS OWN LANE, and this test is what says so on the wire.
//
// generate_music sent its brief to Speak — /audio/speech — for as long as it
// existed, and there is no music behind that endpoint: every call this tool ever
// made failed. The fix is a request of its own
// ([provider.MediaClient.GenerateMusic], streaming chat completions), and what
// this pins is that the composing verb reaches THAT method with the music slot's
// model and never the speaking one.
func TestGenerateMusicComposesOnItsOwnLaneAndNeverThroughSpeech(t *testing.T) {
cost := 0.08
fake := &fakeMediaProvider{musicResponse: &provider.MusicResponse{
Audio: []byte("music"), Format: "mp3", Usage: &ai.Usage{Cost: &cost},
}}
tools, space := mediaToolbox(t, fake, fakeModalities{})
result := tools.Execute(context.Background(), "generate_music", `{"prompt":"Glass Bells at Dawn"}`)
if result.IsError || result.Usage.Calls != 1 || result.Usage.Cost != cost {
t.Fatalf("music result = %+v", result)
}
if fake.musicCalls != 1 || fake.musicRequest.Model != "music/model" || fake.musicRequest.Prompt != "Glass Bells at Dawn" {
t.Fatalf("music request = %+v (calls %d)", fake.musicRequest, fake.musicCalls)
}
// The speaking lane is untouched: a composition brief that reached
// /audio/speech is the bug this replaced.
if fake.speechCalls != 0 {
t.Fatalf("a composition brief was sent to the speech endpoint (%d calls)", fake.speechCalls)
}
if _, ok := space.Locate(filepath.Join("media", "glass-bells-at-dawn.mp3")); !ok {
t.Fatal("music file was not written")
}
fake.musicResponse = nil
fake.musicErr = errors.New("provider detail")
failed := tools.Execute(context.Background(), "generate_music", `{"prompt":"failure"}`)
if !failed.IsError || failed.Usage != (Usage{}) || !strings.Contains(failed.Content, "music generation failed") || strings.Contains(failed.Content, "provider detail") {
t.Fatalf("failed music = %+v", failed)
}
}
func TestGenerateVideoMapsReferencesWritesMP4AndGatesKnownPrice(t *testing.T) {
cost := 1.25
fake := &fakeMediaProvider{videoResponse: &provider.VideoResponse{Video: []byte("video"), Usage: &ai.Usage{Cost: &cost}}}
tools, space := mediaToolbox(t, fake, fakeModalities{})
for _, name := range []string{"first.png", "last.jpg", "style.webp", "palette.gif"} {
if err := os.WriteFile(filepath.Join(space.Root(), name), []byte(name), 0o644); err != nil {
t.Fatal(err)
}
}
var estimated float64
tools.media.BeforeSpend = func(_ context.Context, amount float64) error { estimated = amount; return nil }
result := tools.Execute(context.Background(), "generate_video", `{
"prompt":"Lanterns Across the Harbor","duration":8,"resolution":"720p","aspect_ratio":"16:9",
"reference_paths":["first.png","last.jpg","style.webp","palette.gif"]}`)
if result.IsError || result.Usage.Calls != 1 || result.Usage.Cost != cost || estimated != 0.5 {
t.Fatalf("video result = %+v estimate=%v", result, estimated)
}
request := fake.videoRequest
if request.Duration != 8 || request.Resolution != "720p" || request.AspectRatio != "16:9" ||
len(request.FrameImages) != 2 || len(request.InputReferences) != 2 ||
request.FrameImages[0].FrameType != "first_frame" || request.FrameImages[1].FrameType != "last_frame" ||
request.InputReferences[0].FrameType != "" {
t.Fatalf("video request = %+v", request)
}
if !strings.HasPrefix(request.FrameImages[0].ImageURL.URL, "data:image/png;base64,") ||
!strings.HasPrefix(request.FrameImages[1].ImageURL.URL, "data:image/jpeg;base64,") {
t.Fatalf("video refs = %+v", request.FrameImages)
}
if _, ok := space.Locate(filepath.Join("media", "lanterns-across-the-harbor.mp4")); !ok {
t.Fatal("video file was not written")
}
}
func TestGenerateVideoFailureAndTimeoutAreCalmAndCostNothing(t *testing.T) {
fake := &fakeMediaProvider{videoErr: errors.New("video job failed: policy refused this prompt")}
tools, _ := mediaToolbox(t, fake, fakeModalities{})
failed := tools.Execute(context.Background(), "generate_video", `{"prompt":"failure"}`)
if !failed.IsError || failed.Usage != (Usage{}) || !strings.Contains(failed.Content, "policy refused this prompt") || strings.Contains(failed.Content, "\n") {
t.Fatalf("failed video = %+v", failed)
}
fake.videoErr = provider.ErrVideoTimeout
timedOut := tools.Execute(context.Background(), "generate_video", `{"prompt":"slow"}`)
if !timedOut.IsError || timedOut.Usage != (Usage{}) || !strings.Contains(timedOut.Content, "timed out") || strings.Contains(timedOut.Content, "\n") {
t.Fatalf("timed-out video = %+v", timedOut)
}
}
func TestGenerateImageFailureIsCalmAndRecordsNoUsage(t *testing.T) {
fake := &fakeMediaProvider{imageErr: errors.New("provider exploded with secrets")}
tools, _ := mediaToolbox(t, fake, fakeModalities{})
result := tools.Execute(context.Background(), "generate_image", `{"prompt":"harbor"}`)
if !result.IsError || !strings.Contains(result.Content, "image generation failed") || strings.Contains(result.Content, "secrets") {
t.Fatalf("result = %+v", result)
}
if result.Usage != (Usage{}) {
t.Fatalf("failed generation recorded usage: %+v", result.Usage)
}
}
func TestSpeakWritesMP3AndOnlySuccessfulCallRecordsUsage(t *testing.T) {
cost := 0.07
fake := &fakeMediaProvider{speechResponse: &provider.SpeechResponse{Audio: []byte("audio"), Usage: &ai.Usage{Cost: &cost}}}
tools, space := mediaToolbox(t, fake, fakeModalities{})
result := tools.Execute(context.Background(), "speak", `{"text":"A calm morning","voice":"nova"}`)
if result.IsError || result.Usage.Calls != 1 || result.Usage.Cost != cost {
t.Fatalf("result = %+v", result)
}
if fake.speechRequest.Voice != "nova" || fake.speechRequest.ResponseFormat != "mp3" {
t.Fatalf("speech request = %+v", fake.speechRequest)
}
if _, ok := space.Locate(filepath.Join("media", "a-calm-morning.mp3")); !ok {
t.Fatal("speech file was not written")
}
fake.speechErr = errors.New("nope")
fake.speechResponse = nil
failed := tools.Execute(context.Background(), "speak", `{"text":"failure"}`)
if !failed.IsError || failed.Usage != (Usage{}) || !strings.Contains(failed.Content, "speech synthesis failed") {
t.Fatalf("failed speech = %+v", failed)
}
}
func TestViewImageNativePathIsUnchangedAndCarriesTargetedQuestion(t *testing.T) {
fake := &fakeMediaProvider{}
modalities := fakeModalities{"vision/model:input:image": true}
tools, space := mediaToolbox(t, fake, modalities)
if err := os.WriteFile(filepath.Join(space.Root(), "look.png"), []byte("pixels"), 0o644); err != nil {
t.Fatal(err)
}
loaded := tools.Execute(context.Background(), "view_image", `{"path":"look.png"}`)
if loaded.IsError || len(loaded.Followup) != 2 || loaded.Followup[1].ImageURL == nil ||
!strings.HasPrefix(loaded.Followup[1].ImageURL.URL, "data:image/png;base64,") ||
loaded.Content != "⌾ look.png\nImage loaded for the next turn." || loaded.Followup[0].Text != "Image from look.png:" ||
loaded.Usage != (Usage{}) {
t.Fatalf("loaded = %+v", loaded)
}
targeted := tools.Execute(context.Background(), "view_image", `{"path":"look.png","question":"Does the text read exactly AgentField?"}`)
if targeted.IsError || !strings.Contains(targeted.Followup[0].Text, "Question: Does the text read exactly AgentField?") {
t.Fatalf("targeted native view = %+v", targeted)
}
tools.media.WorkingModel = "text/model"
refused := tools.Execute(context.Background(), "view_image", `{"path":"look.png"}`)
if !refused.IsError || refused.Content != "text/model can't see images — continue with file metadata or use a vision model; no vision model available" || len(refused.Followup) != 0 {
t.Fatalf("refused = %+v", refused)
}
tools.media.WorkingModel = "vision/model"
unsupported := tools.Execute(context.Background(), "view_image", `{"path":"look.svg"}`)
if !unsupported.IsError || !strings.Contains(unsupported.Content, "only png, jpeg, webp, and gif") {
t.Fatalf("unsupported image = %+v", unsupported)
}
}
func TestViewImageProxySendsBase64QuestionAndReturnsAttributedUsage(t *testing.T) {
wire := &visionWireCapture{}
tools, space := mediaToolbox(t, &fakeMediaProvider{}, fakeModalities{})
tools.media.WorkingModel = "text/model"
tools.media.VisionModel = "vendor/vision-model"
tools.media.VisionClient = wire.client(t)
if err := os.WriteFile(filepath.Join(space.Root(), "look.png"), []byte("pixels"), 0o644); err != nil {
t.Fatal(err)
}
gateBeforeCall := false
tools.media.BeforeSpend = func(_ context.Context, amount float64) error {
gateBeforeCall = wire.calls == 0 && amount == 0
return nil
}
result := tools.Execute(context.Background(), "view_image", `{"path":"look.png","question":"Does the text read exactly AgentField?"}`)
if result.IsError || result.Content != "seen by vision-model: AgentField is centered in a blue card." ||
result.Usage.Calls != 1 || result.Usage.PromptTokens != 19 || result.Usage.CompletionTokens != 8 || result.Usage.Cost != 0.125 {
t.Fatalf("proxy result = %+v", result)
}
if wire.calls != 1 || !gateBeforeCall || len(wire.bodies) != 1 {
t.Fatalf("proxy calls=%d gate_before=%t", wire.calls, gateBeforeCall)
}
body := wire.bodies[0]
if body["model"] != "vendor/vision-model" {
t.Fatalf("proxy model = %#v", body["model"])
}
messages, _ := body["messages"].([]any)
if len(messages) != 1 {
t.Fatalf("proxy messages = %#v", body["messages"])
}
message, _ := messages[0].(map[string]any)
parts, _ := message["content"].([]any)
if len(parts) != 2 {
t.Fatalf("proxy content = %#v", message["content"])
}
textPart, _ := parts[0].(map[string]any)
imagePart, _ := parts[1].(map[string]any)
imageURL, _ := imagePart["image_url"].(map[string]any)
if textPart["type"] != "text" || textPart["text"] != "Does the text read exactly AgentField?" ||
imagePart["type"] != "image_url" || !strings.HasPrefix(fmt.Sprint(imageURL["url"]), "data:image/png;base64,") {
t.Fatalf("proxy content parts = %#v", parts)
}
}
func TestViewImageProxyUsesDefaultQuestion(t *testing.T) {
wire := &visionWireCapture{}
tools, space := mediaToolbox(t, &fakeMediaProvider{}, fakeModalities{})
tools.media.WorkingModel = "text/model"
tools.media.VisionModel = "vendor/vision-model"
tools.media.VisionClient = wire.client(t)
if err := os.WriteFile(filepath.Join(space.Root(), "look.jpg"), []byte("pixels"), 0o644); err != nil {
t.Fatal(err)
}
result := tools.Execute(context.Background(), "view_image", `{"path":"look.jpg"}`)
if result.IsError || wire.calls != 1 {
t.Fatalf("default proxy result = %+v calls=%d", result, wire.calls)
}
messages, _ := wire.bodies[0]["messages"].([]any)
message, _ := messages[0].(map[string]any)
parts, _ := message["content"].([]any)
textPart, _ := parts[0].(map[string]any)
if textPart["text"] != defaultImageQuestion {
t.Fatalf("default question = %#v", textPart["text"])
}
}
func TestViewImageProxySpendGateAndFailureRecordNoUsage(t *testing.T) {
for _, test := range []struct {
name string
gate bool
}{
{name: "gate", gate: true},
{name: "provider failure"},
} {
t.Run(test.name, func(t *testing.T) {
wire := &visionWireCapture{}
if !test.gate {
wire.status = http.StatusBadRequest
wire.response = `{"error":{"message":"upstream failed"}}`
}
tools, space := mediaToolbox(t, &fakeMediaProvider{}, fakeModalities{})
tools.media.WorkingModel = "text/model"
tools.media.VisionModel = "vendor/vision-model"
tools.media.VisionClient = wire.client(t)
if err := os.WriteFile(filepath.Join(space.Root(), "look.webp"), []byte("pixels"), 0o644); err != nil {
t.Fatal(err)
}
if test.gate {
tools.media.BeforeSpend = func(context.Context, float64) error { return errors.New("rail") }
}
result := tools.Execute(context.Background(), "view_image", `{"path":"look.webp"}`)
if !result.IsError || result.Usage != (Usage{}) {
t.Fatalf("failed proxy = %+v", result)
}
wantCalls := 1
if test.gate {
wantCalls = 0
if !strings.Contains(result.Content, "paused at the daily budget") {
t.Fatalf("gated proxy = %+v", result)
}
}
if wire.calls != wantCalls {
t.Fatalf("provider calls = %d, want %d", wire.calls, wantCalls)
}
})
}
}
func TestViewImageProxyRailsRefuseBeforeSpendOrCompletion(t *testing.T) {
wire := &visionWireCapture{}
tools, space := mediaToolbox(t, &fakeMediaProvider{}, fakeModalities{})
tools.media.WorkingModel = "text/model"
tools.media.VisionModel = "vendor/vision-model"
tools.media.VisionClient = wire.client(t)
spendCalls := 0
tools.media.BeforeSpend = func(context.Context, float64) error { spendCalls++; return nil }
if err := os.WriteFile(filepath.Join(space.Root(), "look.svg"), []byte("<svg/>"), 0o644); err != nil {
t.Fatal(err)
}
oversized := filepath.Join(space.Root(), "huge.png")
file, err := os.Create(oversized)
if err != nil {
t.Fatal(err)
}
if err := file.Truncate(maxImageInputBytes + 1); err != nil {
file.Close()
t.Fatal(err)
}
if err := file.Close(); err != nil {
t.Fatal(err)
}
wrong := tools.Execute(context.Background(), "view_image", `{"path":"look.svg"}`)
huge := tools.Execute(context.Background(), "view_image", `{"path":"huge.png"}`)
if !wrong.IsError || !strings.Contains(wrong.Content, "only png, jpeg, webp, and gif") ||
!huge.IsError || !strings.Contains(huge.Content, "over the 10 MB image limit") || wire.calls != 0 || spendCalls != 0 {
t.Fatalf("wrong=%+v huge=%+v provider_calls=%d spend_calls=%d", wrong, huge, wire.calls, spendCalls)
}
}
func TestMediaSlugDerivesReadableBoundedNames(t *testing.T) {
if got := MediaSlug(" Sunset, over the Harbor!! "); got != "sunset-over-the-harbor" {
t.Fatalf("slug = %q", got)
}
if got := MediaSlug(strings.Repeat("Long prompt ", 20)); len(got) > 56 || strings.HasSuffix(got, "-") {
t.Fatalf("long slug = %q (%d)", got, len(got))
}
}
// The model argument has exactly three spellings — absent, "best", and a name
// — and every one of them still crosses the same rail and records the same
// usage. The resolved model is named in the artifact receipt, because "which
// model made this" is the first question about a file you keep.
func TestMediaModelArgumentReadsAllThreeSpellingsAndKeepsTheRail(t *testing.T) {
audio := func() *fakeMediaProvider {
cost := 0.09
return &fakeMediaProvider{
imageResponse: &provider.ImageResponse{Data: []provider.GeneratedImage{{Base64: base64.StdEncoding.EncodeToString([]byte("png")), MediaType: "image/png"}}},
speechResponse: &provider.SpeechResponse{Audio: []byte("sound"), Usage: &ai.Usage{Cost: &cost}},
musicResponse: &provider.MusicResponse{Audio: []byte("song"), Format: "mp3", Usage: &ai.Usage{Cost: &cost}},
videoResponse: &provider.VideoResponse{Video: []byte("video"), Usage: &ai.Usage{Cost: &cost}},
}
}
for _, test := range []struct {
name string
tool string
arguments string
modality string
word string
want string
}{
{name: "image slot default", tool: "generate_image", arguments: `{"prompt":"harbor"}`, want: "paint/model"},
{name: "image best", tool: "generate_image", arguments: `{"prompt":"harbor","model":"best"}`,
modality: "image", word: "best", want: "studio/grand"},
{name: "image name", tool: "generate_image", arguments: `{"prompt":"harbor","model":"krea"}`,
modality: "image", word: "krea", want: "studio/grand"},
{name: "speech best", tool: "speak", arguments: `{"text":"a calm morning","model":"best"}`,
modality: "speech", word: "best", want: "studio/grand"},
{name: "music name", tool: "generate_music", arguments: `{"prompt":"glass bells","model":"lyria"}`,
modality: "music", word: "lyria", want: "studio/grand"},
{name: "video best", tool: "generate_video", arguments: `{"prompt":"lanterns","model":"best"}`,
modality: "video", word: "best", want: "studio/grand"},
} {
t.Run(test.name, func(t *testing.T) {
fake := audio()
tools, _ := mediaToolbox(t, fake, fakeModalities{})
var sawModality, sawWord string
tools.media.ResolveModel = func(modality, word string) (string, error) {
sawModality, sawWord = modality, word
return "studio/grand", nil
}
gated := 0
tools.media.BeforeSpend = func(context.Context, float64) error { gated++; return nil }
result := tools.Execute(context.Background(), test.tool, test.arguments)
if result.IsError {
t.Fatal(result.Content)
}
if sawModality != test.modality || sawWord != test.word {
t.Fatalf("resolver saw (%q, %q), want (%q, %q)", sawModality, sawWord, test.modality, test.word)
}
served := fake.imageRequest.Model
switch test.tool {
case "speak":
served = fake.speechRequest.Model
case "generate_music":
// Its own request type, because it is its own endpoint.
served = fake.musicRequest.Model
case "generate_video":
served = fake.videoRequest.Model
}
if served != test.want {
t.Fatalf("served model = %q, want %q", served, test.want)
}
if !strings.Contains(result.Content, test.want) {
t.Fatalf("artifact receipt = %q, want the resolved model named", result.Content)
}
if gated != 1 || result.Usage.Calls != 1 {
t.Fatalf("rail crossings = %d usage = %+v", gated, result.Usage)
}
})
}
}
func TestMediaWrongModalityNameIsRefusedCalmlyAndCostsNothing(t *testing.T) {
fake := &fakeMediaProvider{}
tools, _ := mediaToolbox(t, fake, fakeModalities{})
tools.media.ResolveModel = func(modality, word string) (string, error) {
return "", fmt.Errorf("hexgrad/kokoro-82m makes speech, not %s", modality)
}
spent := 0
tools.media.BeforeSpend = func(context.Context, float64) error { spent++; return nil }
result := tools.Execute(context.Background(), "generate_image", `{"prompt":"harbor","model":"kokoro"}`)
if !result.IsError || !strings.Contains(result.Content, "makes speech, not image") {
t.Fatalf("refusal = %+v", result)
}
if fake.imageCalls != 0 || spent != 0 || result.Usage != (Usage{}) {
t.Fatalf("refusal cost something: calls=%d gate=%d usage=%+v", fake.imageCalls, spent, result.Usage)
}
}
func TestMediaModelArgumentIsInertWithoutACatalog(t *testing.T) {
fake := &fakeMediaProvider{imageResponse: &provider.ImageResponse{
Data: []provider.GeneratedImage{{Base64: base64.StdEncoding.EncodeToString([]byte("png")), MediaType: "image/png"}},
}}
tools, _ := mediaToolbox(t, fake, fakeModalities{})
result := tools.Execute(context.Background(), "generate_image", `{"prompt":"harbor","model":"best"}`)
if result.IsError || fake.imageRequest.Model != "paint/model" {
t.Fatalf("embedder without a catalog = %+v request=%+v", result, fake.imageRequest)
}
}