feat: tag web search sources with the search that found them

agent.go

@@ -1605,6 +1605,7 @@ func (a *agent) processStepStream(ctx context.Context, stream StreamResponse, op
 				ID:               part.ID,
 				URL:              part.URL,
 				Title:            part.Title,
+				ToolCallID:       part.SourceToolCallID,
 				ProviderMetadata: part.ProviderMetadata,
 			}
 			stepContent = append(stepContent, sourceContent)

content.go

@@ -412,12 +412,16 @@ const (
 
 // SourceContent represents a source that has been used as input to generate the response.
 type SourceContent struct {
-	SourceType       SourceType       `json:"source_type"` // "url" or "document"
-	ID               string           `json:"id"`
-	URL              string           `json:"url"` // for URL sources
-	Title            string           `json:"title"`
-	MediaType        string           `json:"media_type"` // for document sources (IANA media type)
-	Filename         string           `json:"filename"`   // for document sources
+	SourceType SourceType `json:"source_type"` // "url" or "document"
+	ID         string     `json:"id"`
+	URL        string     `json:"url"` // for URL sources
+	Title      string     `json:"title"`
+	MediaType  string     `json:"media_type"` // for document sources (IANA media type)
+	Filename   string     `json:"filename"`   // for document sources
+	// ToolCallID is the provider-executed tool call, such as a web search,
+	// that returned this source. It is empty for sources the response text
+	// cites.
+	ToolCallID       string           `json:"tool_call_id,omitempty"`
 	ProviderMetadata ProviderMetadata `json:"provider_metadata"`
 }
 

content_json.go

@@ -187,6 +187,7 @@ func (s SourceContent) MarshalJSON() ([]byte, error) {
 		Title            string           `json:"title,omitempty"`
 		MediaType        string           `json:"media_type,omitempty"`
 		Filename         string           `json:"filename,omitempty"`
+		ToolCallID       string           `json:"tool_call_id,omitempty"`
 		ProviderMetadata ProviderMetadata `json:"provider_metadata,omitempty"`
 	}{
 		SourceType:       s.SourceType,
@@ -195,6 +196,7 @@ func (s SourceContent) MarshalJSON() ([]byte, error) {
 		Title:            s.Title,
 		MediaType:        s.MediaType,
 		Filename:         s.Filename,
+		ToolCallID:       s.ToolCallID,
 		ProviderMetadata: s.ProviderMetadata,
 	})
 	if err != nil {
@@ -221,6 +223,7 @@ func (s *SourceContent) UnmarshalJSON(data []byte) error {
 		Title            string                     `json:"title,omitempty"`
 		MediaType        string                     `json:"media_type,omitempty"`
 		Filename         string                     `json:"filename,omitempty"`
+		ToolCallID       string                     `json:"tool_call_id,omitempty"`
 		ProviderMetadata map[string]json.RawMessage `json:"provider_metadata,omitempty"`
 	}
 
@@ -234,6 +237,7 @@ func (s *SourceContent) UnmarshalJSON(data []byte) error {
 	s.Title = aux.Title
 	s.MediaType = aux.MediaType
 	s.Filename = aux.Filename
+	s.ToolCallID = aux.ToolCallID
 
 	if len(aux.ProviderMetadata) > 0 {
 		metadata, err := UnmarshalProviderMetadata(aux.ProviderMetadata)

json_test.go

@@ -610,6 +610,25 @@ func (m *mockProviderData) UnmarshalJSON(data []byte) error {
 	return nil
 }
 
+func TestSourceContentJSONRoundTrip(t *testing.T) {
+	for _, source := range []SourceContent{
+		{SourceType: SourceTypeURL, ID: "found", URL: "https://example.com", ToolCallID: "ws_1"},
+		{SourceType: SourceTypeURL, ID: "cited", URL: "https://example.com", Title: "Example"},
+	} {
+		data, err := json.Marshal(source)
+		if err != nil {
+			t.Fatalf("failed to marshal source: %v", err)
+		}
+		decoded, err := UnmarshalContent(data)
+		if err != nil {
+			t.Fatalf("failed to unmarshal source: %v", err)
+		}
+		if !reflect.DeepEqual(decoded, source) {
+			t.Errorf("round trip mismatch: got %#v, want %#v", decoded, source)
+		}
+	}
+}
+
 func TestPromptSerialization(t *testing.T) {
 	t.Run("serialize prompt (message slice)", func(t *testing.T) {
 		prompt := Prompt{

model.go

@@ -187,6 +187,8 @@ type StreamPart struct {
 	SourceType SourceType `json:"source_type"`
 	URL        string     `json:"url"`
 	Title      string     `json:"title"`
+	// SourceToolCallID is SourceContent.ToolCallID for source parts.
+	SourceToolCallID string `json:"source_tool_call_id,omitempty"`
 
 	ProviderMetadata ProviderMetadata `json:"provider_metadata"`
 }

providers/anthropic/anthropic.go

@@ -1683,8 +1683,9 @@ func (a languageModel) Generate(ctx context.Context, call fantasy.Call) (*fantas
 			if !ok {
 				continue
 			}
-			// Extract search results as sources/citations, preserving
-			// encrypted_content for multi-turn round-tripping.
+			// Emit found pages as sources tagged with the search's ID,
+			// and keep them in metadata with encrypted_content for
+			// multi-turn round-tripping.
 			toolResult := fantasy.ToolResultContent{
 				ToolCallID:       webSearchResult.ToolUseID,
 				ToolName:         "web_search",
@@ -1698,6 +1699,7 @@ func (a languageModel) Generate(ctx context.Context, call fantasy.Call) (*fantas
 						ID:         item.URL,
 						URL:        item.URL,
 						Title:      item.Title,
+						ToolCallID: webSearchResult.ToolUseID,
 					})
 					metadataResults = append(metadataResults, WebSearchResultItem{
 						URL:              item.URL,
@@ -1887,11 +1889,12 @@ func (a languageModel) Stream(ctx context.Context, call fantasy.Call) (fantasy.S
 					if items := contentBlock.Content.OfWebSearchResultBlockArray; len(items) > 0 {
 						for _, item := range items {
 							if !yield(fantasy.StreamPart{
-								Type:       fantasy.StreamPartTypeSource,
-								ID:         item.URL,
-								SourceType: fantasy.SourceTypeURL,
-								URL:        item.URL,
-								Title:      item.Title,
+								Type:             fantasy.StreamPartTypeSource,
+								ID:               item.URL,
+								SourceType:       fantasy.SourceTypeURL,
+								URL:              item.URL,
+								Title:            item.Title,
+								SourceToolCallID: contentBlock.ToolUseID,
 							}) {
 								return
 							}

providers/anthropic/anthropic_test.go

@@ -2166,13 +2166,16 @@ func TestGenerate_WebSearchResponse(t *testing.T) {
 	require.Equal(t, "web_search", toolCalls[0].ToolName)
 	require.Equal(t, "srvtoolu_01", toolCalls[0].ToolCallID)
 
-	// SourceContent entries for each search result.
+	// SourceContent entries for each search result, tagged with the
+	// search that found them.
 	require.Len(t, sources, 2)
 	require.Equal(t, "https://example.com/ai-news", sources[0].URL)
 	require.Equal(t, "Latest AI News", sources[0].Title)
 	require.Equal(t, fantasy.SourceTypeURL, sources[0].SourceType)
+	require.Equal(t, "srvtoolu_01", sources[0].ToolCallID)
 	require.Equal(t, "https://example.com/ml-update", sources[1].URL)
 	require.Equal(t, "ML Update", sources[1].Title)
+	require.Equal(t, "srvtoolu_01", sources[1].ToolCallID)
 
 	// ToolResultContent with provider metadata preserving encrypted_content.
 	require.Len(t, toolResults, 1)
@@ -2776,6 +2779,7 @@ func TestStream_WebSearchResponse(t *testing.T) {
 	require.Equal(t, "https://example.com/ai-news", sourceParts[0].URL)
 	require.Equal(t, "Latest AI News", sourceParts[0].Title)
 	require.Equal(t, fantasy.SourceTypeURL, sourceParts[0].SourceType)
+	require.Equal(t, "srvtoolu_01", sourceParts[0].SourceToolCallID)
 
 	// Text block emits a text delta.
 	require.NotEmpty(t, textDeltas, "should have text deltas")

providers/openai/responses_language_model.go

@@ -330,14 +330,6 @@ func (o responsesLanguageModel) prepareParams(call fantasy.Call) (*responses.Res
 		params.Truncation = responses.ResponseNewParamsTruncationAuto
 	}
 
-	if len(include) > 0 {
-		includeParams := make([]responses.ResponseIncludable, len(include))
-		for i, inc := range include {
-			includeParams[i] = responses.ResponseIncludable(string(inc))
-		}
-		params.Include = includeParams
-	}
-
 	if modelConfig.isReasoningModel {
 		if call.Temperature != nil {
 			params.Temperature = param.Opt[float64]{}
@@ -404,6 +396,18 @@ func (o responsesLanguageModel) prepareParams(call fantasy.Call) (*responses.Res
 		params.ToolChoice = toolChoice
 	}
 
+	if hasResponsesWebSearchTool(tools) && !slices.Contains(include, IncludeWebSearchCallActionSources) {
+		include = append(include, IncludeWebSearchCallActionSources)
+	}
+
+	if len(include) > 0 {
+		includeParams := make([]responses.ResponseIncludable, len(include))
+		for i, inc := range include {
+			includeParams[i] = responses.ResponseIncludable(string(inc))
+		}
+		params.Include = includeParams
+	}
+
 	return params, warnings, nil
 }
 
@@ -887,6 +891,12 @@ func effectiveResponseStore(requested bool, response *responses.Response) bool {
 	return requested && response.JSON.ExtraFields["store"].Raw() != "false"
 }
 
+func hasResponsesWebSearchTool(tools []responses.ToolUnionParam) bool {
+	return slices.ContainsFunc(tools, func(tool responses.ToolUnionParam) bool {
+		return tool.OfWebSearch != nil || tool.OfWebSearchPreview != nil
+	})
+}
+
 func isResponsesWebSearchToolCall(toolCallPart fantasy.ToolCallPart) bool {
 	return toolCallPart.ToolName == "web_search" ||
 		toolCallPart.ToolName == "web_search_preview"
@@ -1190,21 +1200,27 @@ func (o responsesLanguageModel) Generate(ctx context.Context, call fantasy.Call)
 			// a ToolCallContent and ToolResultContent as a pair,
 			// matching the vercel/ai pattern for provider tools.
 			//
-			// Note: source citations come from url_citation annotations
-			// on the message text (handled in the "message" case above),
-			// not from the web_search_call action.
-			wsMeta := webSearchCallToMetadata(outputItem.ID, outputItem.Action)
+			// Found pages become sources tagged with the search's ID.
+			// Citations come untagged from url_citation annotations on
+			// the message text (handled in the "message" case above).
+			// A non-streamed response only has the final output, where
+			// OpenAI also lists pages the answer cited among the
+			// search's sources, so those appear as found pages here.
 			content = append(content, fantasy.ToolCallContent{
 				ProviderExecuted: true,
 				ToolCallID:       outputItem.ID,
 				ToolName:         "web_search",
+				Input:            webSearchCallInput(outputItem.Action),
 			})
+			for _, source := range webSearchCallSources(outputItem.ID, outputItem.Action) {
+				content = append(content, source)
+			}
 			content = append(content, fantasy.ToolResultContent{
 				ProviderExecuted: true,
 				ToolCallID:       outputItem.ID,
 				ToolName:         "web_search",
 				ProviderMetadata: fantasy.ProviderMetadata{
-					Name: wsMeta,
+					Name: webSearchCallToMetadata(outputItem.ID, outputItem.Status, outputItem.Action),
 				},
 			})
 
@@ -1446,9 +1462,10 @@ func (o responsesLanguageModel) Stream(ctx context.Context, call fantasy.Call) (
 					}
 
 				case "web_search_call":
-					// Provider-executed web search completed.
-					// Source citations come from url_citation annotations
-					// on the streamed message text, not from the action.
+					// Provider-executed web search completed. Its found
+					// pages are sources tagged with the search's ID;
+					// citations come untagged from url_citation
+					// annotations on the message text.
 					if !yield(fantasy.StreamPart{
 						Type: fantasy.StreamPartTypeToolInputEnd,
 						ID:   done.Item.ID,
@@ -1459,10 +1476,22 @@ func (o responsesLanguageModel) Stream(ctx context.Context, call fantasy.Call) (
 						Type:             fantasy.StreamPartTypeToolCall,
 						ID:               done.Item.ID,
 						ToolCallName:     "web_search",
+						ToolCallInput:    webSearchCallInput(done.Item.Action),
 						ProviderExecuted: true,
 					}) {
 						return
 					}
+					for _, source := range webSearchCallSources(done.Item.ID, done.Item.Action) {
+						if !yield(fantasy.StreamPart{
+							Type:             fantasy.StreamPartTypeSource,
+							ID:               source.ID,
+							SourceType:       source.SourceType,
+							URL:              source.URL,
+							SourceToolCallID: source.ToolCallID,
+						}) {
+							return
+						}
+					}
 					// Emit a ToolResult so the agent framework
 					// includes it in round-trip messages.
 					if !yield(fantasy.StreamPart{
@@ -1471,7 +1500,7 @@ func (o responsesLanguageModel) Stream(ctx context.Context, call fantasy.Call) (
 						ToolCallName:     "web_search",
 						ProviderExecuted: true,
 						ProviderMetadata: fantasy.ProviderMetadata{
-							Name: webSearchCallToMetadata(done.Item.ID, done.Item.Action),
+							Name: webSearchCallToMetadata(done.Item.ID, done.Item.Status, done.Item.Action),
 						},
 					}) {
 						return
@@ -1766,12 +1795,15 @@ func toWebSearchToolParam(pt fantasy.ProviderDefinedTool) responses.ToolUnionPar
 
 // webSearchCallToMetadata converts an OpenAI web search call output
 // into our structured metadata for round-tripping.
-func webSearchCallToMetadata(itemID string, action responses.ResponseOutputItemUnionAction) *WebSearchCallMetadata {
-	meta := &WebSearchCallMetadata{ItemID: itemID}
+func webSearchCallToMetadata(itemID, status string, action responses.ResponseOutputItemUnionAction) *WebSearchCallMetadata {
+	meta := &WebSearchCallMetadata{ItemID: itemID, Status: status}
 	if action.Type != "" {
 		a := &WebSearchAction{
-			Type:  action.Type,
-			Query: action.Query,
+			Type:    action.Type,
+			Queries: action.Queries,
+			Query:   action.Query,
+			URL:     action.URL,
+			Pattern: action.Pattern,
 		}
 		for _, src := range action.Sources {
 			a.Sources = append(a.Sources, WebSearchSource{
@@ -1790,6 +1822,45 @@ func responsesTextMetadata(itemID string, phase responses.ResponseOutputMessageP
 	}
 }
 
+// webSearchCallSources returns the pages a web_search_call found as URL
+// sources tagged with the call's ID.
+func webSearchCallSources(itemID string, action responses.ResponseOutputItemUnionAction) []fantasy.SourceContent {
+	sources := make([]fantasy.SourceContent, 0, len(action.Sources))
+	for _, src := range action.Sources {
+		if src.URL == "" {
+			continue
+		}
+		sources = append(sources, fantasy.SourceContent{
+			SourceType: fantasy.SourceTypeURL,
+			ID:         src.URL,
+			URL:        src.URL,
+			ToolCallID: itemID,
+		})
+	}
+	return sources
+}
+
+// webSearchCallInput encodes the web_search_call action as its tool call
+// input: the action type with the queries of a search (falling back to the
+// deprecated query), the URL of open_page, or the URL and pattern of
+// find_in_page.
+func webSearchCallInput(action responses.ResponseOutputItemUnionAction) string {
+	queries := action.Queries
+	if len(queries) == 0 && action.Query != "" {
+		queries = []string{action.Query}
+	}
+	encoded, err := json.Marshal(struct {
+		Type    string   `json:"type,omitempty"`
+		Queries []string `json:"queries,omitempty"`
+		URL     string   `json:"url,omitempty"`
+		Pattern string   `json:"pattern,omitempty"`
+	}{Type: action.Type, Queries: queries, URL: action.URL, Pattern: action.Pattern})
+	if err != nil {
+		return "{}"
+	}
+	return string(encoded)
+}
+
 // GetReasoningMetadata extracts reasoning metadata from provider options for responses models.
 func GetReasoningMetadata(providerOptions fantasy.ProviderOptions) *ResponsesReasoningMetadata {
 	if openaiResponsesOptions, ok := providerOptions[Name]; ok {

providers/openai/responses_options.go

@@ -188,6 +188,10 @@ const (
 	IncludeFileSearchCallResults IncludeType = "file_search_call.results"
 	// IncludeMessageOutputTextLogprobs includes message output text log probabilities.
 	IncludeMessageOutputTextLogprobs IncludeType = "message.output_text.logprobs"
+	// IncludeWebSearchCallActionSources includes the pages a web search
+	// found in web_search_call action.sources. Requests with a web
+	// search tool add it automatically.
+	IncludeWebSearchCallActionSources IncludeType = "web_search_call.action.sources"
 )
 
 // ServiceTier represents the service tier for OpenAI Responses API.
@@ -448,11 +452,19 @@ type WebSearchSource struct {
 
 // WebSearchAction represents the action taken during a web search call.
 type WebSearchAction struct {
-	// Type is the kind of action: "search", "open_page", or "find".
+	// Type is the kind of action: "search", "open_page", or "find_in_page".
 	Type string `json:"type"`
-	// Query is the search query (present when Type is "search").
+	// Queries are the search queries. OpenAI can omit them.
+	Queries []string `json:"queries,omitempty"`
+	// Query is the deprecated single-query field that OpenAI replaced
+	// with Queries.
 	Query string `json:"query,omitempty"`
-	// Sources are the results returned by the search.
+	// URL is the page an open_page or find_in_page action used.
+	URL string `json:"url,omitempty"`
+	// Pattern is the text a find_in_page action looked for.
+	Pattern string `json:"pattern,omitempty"`
+	// Sources are the pages the search found. OpenAI only returns them
+	// when the request includes web_search_call.action.sources.
 	Sources []WebSearchSource `json:"sources,omitempty"`
 }
 
@@ -463,6 +475,8 @@ type WebSearchAction struct {
 type WebSearchCallMetadata struct {
 	// ItemID is the server-side ID of the web_search_call output item.
 	ItemID string `json:"item_id"`
+	// Status is the item's status, such as "completed" or "failed".
+	Status string `json:"status,omitempty"`
 	// Action contains the structured action data from the search.
 	Action *WebSearchAction `json:"action,omitempty"`
 }