Repository navigation
Expand file tree
/
Copy pathdocument.go
More file actions
268 lines (247 loc) · 8.51 KB
/
Copy pathdocument.go
File metadata and controls
268 lines (247 loc) · 8.51 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
package provider
import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"path/filepath"
"strings"
"github.com/Agent-Field/agentfield/sdk/go/ai"
)
// THIS ROUTE SENDS NO max_tokens EITHER. It used to send two figures of its
// own — 8 for a parser-backed acknowledgement, 16k for a native extraction —
// and both were the same kind of thing the chat route stopped sending: a
// ceiling this adapter chose for somebody else's model. The 8 in particular was
// a bet that the model would say "received" in eight tokens, which on a model
// that thinks first is eight tokens of thinking and no acknowledgement at all.
// The prompt already asks for what the route wants; what bounds the reply now is
// the model's own default, and what bounds this PROCESS is the byte cap below.
const (
// maxDocumentResponseBytes bounds what one parse reply may be believed to
// be. It is a safety cap on what is read off a socket, never a generation
// parameter: nothing about it travels in the request body.
maxDocumentResponseBytes = 128 << 20
)
// DocumentParseEngine is one verified OpenRouter file-parser engine. Callers
// must always choose one: omitting the plugin would silently select the
// provider's native-then-OCR fallback and make cost an accident.
type DocumentParseEngine string
const (
DocumentParseCloudflare DocumentParseEngine = "cloudflare-ai"
DocumentParseMistralOCR DocumentParseEngine = "mistral-ocr"
DocumentParseNative DocumentParseEngine = "native"
)
type DocumentRequest struct {
Model string
Filename string
MediaType string
Data []byte
Engine DocumentParseEngine
Question string
}
type DocumentResponse struct {
Text string
Hash string
Usage *ai.Usage
}
type documentWireFile struct {
Filename string `json:"filename"`
FileData string `json:"file_data"`
}
type documentWirePart struct {
Type string `json:"type"`
Text string `json:"text,omitempty"`
File *documentWireFile `json:"file,omitempty"`
}
type documentWireMessage struct {
Role string `json:"role"`
Content []documentWirePart `json:"content"`
}
type documentPlugin struct {
ID string `json:"id"`
PDF struct {
Engine DocumentParseEngine `json:"engine"`
} `json:"pdf"`
}
type documentWireRequest struct {
Model string `json:"model"`
Messages []documentWireMessage `json:"messages"`
Plugins []documentPlugin `json:"plugins"`
Usage ai.RequestUsage `json:"usage"`
}
type documentAnnotation struct {
Type string `json:"type"`
File struct {
Hash string `json:"hash"`
Name string `json:"name,omitempty"`
Content []struct {
Type string `json:"type"`
Text string `json:"text,omitempty"`
} `json:"content"`
} `json:"file"`
}
type documentWireResponse struct {
Choices []struct {
Message struct {
Content json.RawMessage `json:"content"`
Annotations []documentAnnotation `json:"annotations"`
} `json:"message"`
} `json:"choices"`
Usage *ai.Usage `json:"usage,omitempty"`
Error struct {
Metadata struct {
FileAnnotations []documentAnnotation `json:"file_annotations"`
} `json:"metadata"`
} `json:"error"`
}
// ParseDocument performs the completion that activates OpenRouter's file parser
// and harvests text from file annotations. Native is the sole exception:
// OpenRouter produces no annotations for native files, so the model is
// explicitly asked to return extracted text.
//
// The acknowledgement is kept small by the PROMPT — "Acknowledge receipt." —
// and no longer by a ceiling this file picked (see the const block).
func (c *Client) ParseDocument(ctx context.Context, request DocumentRequest) (*DocumentResponse, error) {
if c == nil {
return nil, fmt.Errorf("document client is nil")
}
// Not an observed door (attemptobserver.go): refused before any network use.
if err := refuseObserved(ctx, SurfaceDocument); err != nil {
return nil, err
}
model := strings.TrimSpace(request.Model)
if model == "" {
model = c.config.Model
}
if strings.TrimSpace(request.Filename) == "" {
return nil, fmt.Errorf("document filename is required")
}
if strings.TrimSpace(request.MediaType) == "" {
return nil, fmt.Errorf("document media type is required")
}
if len(request.Data) == 0 {
return nil, fmt.Errorf("document data is empty")
}
switch request.Engine {
case DocumentParseCloudflare, DocumentParseMistralOCR, DocumentParseNative:
default:
return nil, fmt.Errorf("unknown document parser engine %q", request.Engine)
}
prompt := "Acknowledge receipt."
if request.Engine == DocumentParseNative {
prompt = "Extract all readable document text. Preserve headings, lists, and table contents. Return only the extracted text."
if question := strings.TrimSpace(request.Question); question != "" {
prompt += "\nThe downstream worker will answer this question from the extraction: " + question
}
}
plugin := documentPlugin{ID: "file-parser"}
plugin.PDF.Engine = request.Engine
wire := documentWireRequest{
Model: model,
Messages: []documentWireMessage{{Role: "user", Content: []documentWirePart{
{Type: "text", Text: prompt},
{Type: "file", File: &documentWireFile{
Filename: filepath.Base(request.Filename),
FileData: "data:" + request.MediaType + ";base64," + base64.StdEncoding.EncodeToString(request.Data),
}},
}}},
Plugins: []documentPlugin{plugin},
Usage: ai.RequestUsage{Include: true},
}
body, err := json.Marshal(wire)
if err != nil {
return nil, fmt.Errorf("marshal document request: %w", err)
}
// No ceiling on the metadata either. It is what the model-call log and the
// transport read the request's shape from, and a figure here that is not on
// the wire is a receipt describing a request nobody sent.
metadata := &ai.Request{Model: model}
httpResponse, err := c.send(ctx, metadata, callKnobs{}, body, false)
if err != nil {
return nil, err
}
defer httpResponse.Body.Close()
payload, err := io.ReadAll(io.LimitReader(httpResponse.Body, maxDocumentResponseBytes+1))
if err != nil {
return nil, fmt.Errorf("read document response: %w", err)
}
if len(payload) > maxDocumentResponseBytes {
return nil, fmt.Errorf("document response exceeded %d bytes", maxDocumentResponseBytes)
}
var decoded documentWireResponse
if err := json.Unmarshal(payload, &decoded); err != nil {
if httpResponse.StatusCode >= 400 {
return nil, apiError(httpResponse.StatusCode, payload)
}
return nil, fmt.Errorf("decode document response: %w", err)
}
if request.Engine != DocumentParseNative {
annotations := make([]documentAnnotation, 0)
for _, choice := range decoded.Choices {
annotations = append(annotations, choice.Message.Annotations...)
}
annotations = append(annotations, decoded.Error.Metadata.FileAnnotations...)
text, hash := documentAnnotationText(annotations)
if strings.TrimSpace(text) != "" {
return &DocumentResponse{Text: text, Hash: hash, Usage: decoded.Usage}, nil
}
if httpResponse.StatusCode >= 400 {
return nil, apiError(httpResponse.StatusCode, payload)
}
return nil, fmt.Errorf("document parser returned no file annotations")
}
if httpResponse.StatusCode >= 400 {
return nil, apiError(httpResponse.StatusCode, payload)
}
for _, choice := range decoded.Choices {
if text := documentMessageText(choice.Message.Content); strings.TrimSpace(text) != "" {
return &DocumentResponse{Text: text, Usage: decoded.Usage}, nil
}
}
return nil, fmt.Errorf("native document model returned no extracted text")
}
func documentAnnotationText(annotations []documentAnnotation) (string, string) {
seen := make(map[string]bool)
var text []string
firstHash := ""
for _, annotation := range annotations {
hash := strings.TrimSpace(annotation.File.Hash)
if annotation.Type != "file" || hash == "" || seen[hash] {
continue
}
seen[hash] = true
if firstHash == "" {
firstHash = hash
}
for _, part := range annotation.File.Content {
if part.Type == "text" && strings.TrimSpace(part.Text) != "" {
text = append(text, strings.TrimSpace(part.Text))
}
}
}
return strings.Join(text, "\n\n"), firstHash
}
func documentMessageText(raw json.RawMessage) string {
if len(raw) == 0 || string(raw) == "null" {
return ""
}
var text string
if json.Unmarshal(raw, &text) == nil {
return strings.TrimSpace(text)
}
var parts []struct {
Type string `json:"type"`
Text string `json:"text"`
}
if json.Unmarshal(raw, &parts) != nil {
return ""
}
var kept []string
for _, part := range parts {
if part.Type == "text" && strings.TrimSpace(part.Text) != "" {
kept = append(kept, strings.TrimSpace(part.Text))
}
}
return strings.Join(kept, "\n")
}