FIX: Do not score write-only target deliveries as attack responses

doc/code/memory/5_advanced_memory.ipynb

@@ -227,7 +227,7 @@
       "\n",
       "\u001b[1m  🎯 Outcome\u001b[0m\n",
       "\u001b[33m    • Status: ❓ UNDETERMINED\u001b[0m\n",
-      "\u001b[37m    • Reason: No objective scorer configured\u001b[0m\n",
+      "\u001b[37m    • Reason: Objective target returned no response to score\u001b[0m\n",
       "\n",
       "\u001b[1m\u001b[44m\u001b[37m Conversation History with Objective Target \u001b[0m\n",
       "\u001b[34m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n",
@@ -263,7 +263,7 @@
       "\n",
       "\u001b[1m  🎯 Outcome\u001b[0m\n",
       "\u001b[33m    • Status: ❓ UNDETERMINED\u001b[0m\n",
-      "\u001b[37m    • Reason: No objective scorer configured\u001b[0m\n",
+      "\u001b[37m    • Reason: Objective target returned no response to score\u001b[0m\n",
       "\n",
       "\u001b[1m\u001b[44m\u001b[37m Conversation History with Objective Target \u001b[0m\n",
       "\u001b[34m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n",

pyrit/backend/services/message_send_service.py

@@ -774,14 +774,19 @@ async def _execute_message_async(
             if progress is not None:
                 progress.state = MessageSendState.FINALIZING
             current_pieces = await self._memory.get_message_pieces_async(conversation_id=msg_conversation_id)
-            last_response = (
-                delivered_response.message_pieces[0]
-                if isinstance(delivered_response, Message) and delivered_response.api_role == "assistant"
-                else next(
-                    (piece for piece in current_pieces if piece.id not in prior_ids and piece.role == "assistant"),
-                    None,
+            if isinstance(delivered_response, Message) and PromptNormalizer.is_write_only_delivery(delivered_response):
+                # A write-only target returned the sent request, which is never a response to score,
+                # whatever role the request carries.
+                last_response = None
+            else:
+                last_response = (
+                    delivered_response.message_pieces[0]
+                    if isinstance(delivered_response, Message) and delivered_response.api_role == "assistant"
+                    else next(
+                        (piece for piece in current_pieces if piece.id not in prior_ids and piece.role == "assistant"),
+                        None,
+                    )
                 )
-            )
             if (
                 last_response is not None
                 and validated.scoring_inputs

pyrit/executor/attack/core/attack_scoring.py

@@ -9,7 +9,7 @@
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING
 
-from pyrit.models import ScoringExpectation
+from pyrit.models import ScoringExpectation, SurfaceScorable, TraceScorable
 from pyrit.score import MessageScorer
 
 if TYPE_CHECKING:
@@ -71,26 +71,53 @@ def prepare_attack_scoring(
     return prepared
 
 
+# Evidence of what the target did (written files, invoked tools), acquired from a source rather
+# than read from message text.
+_SIDE_EFFECT_SCORABLE_TYPES = frozenset({SurfaceScorable, TraceScorable})
+
+
+def _judges_side_effects(scorer: Scorer) -> bool:
+    # Every leaf must read side effects; leaves of a composite may read different kinds.
+    return all(not _SIDE_EFFECT_SCORABLE_TYPES.isdisjoint(types) for types in scorer.get_leaf_scorable_types())
+
+
 async def score_attack_response_async(
     *,
     response: Message,
     objective_scorer: Scorer | None,
     auxiliary_scorers: Sequence[Scorer] | None,
     expectation: ScoringExpectation | None,
+    side_effect_evidence_only: bool = False,
 ) -> dict[str, list[Score]]:
     """
     Score an attack response with the objective scorer and the auxiliary scorers that apply.
 
+    Args:
+        response (Message): The message that anchors scoring for the turn.
+        objective_scorer (Scorer | None): The scorer that decides the attack outcome.
+        auxiliary_scorers (Sequence[Scorer] | None): Additional scorers whose results are stored.
+        expectation (ScoringExpectation | None): The scoring question.
+        side_effect_evidence_only (bool): Run only scorers that judge side-effect evidence (written
+            files or invoked tools) rather than message text. Use this when ``response`` is a
+            delivered request with no target response, so the request is never judged as response text.
+
     Returns:
         dict[str, list[Score]]: The ``objective_scores`` and ``auxiliary_scores``.
     """
     prepared = prepare_attack_scoring(
         objective_scorer=objective_scorer, auxiliary_scorers=auxiliary_scorers, expectation=expectation
     )
+    auxiliary = list(zip(prepared.auxiliary_scorers, prepared.auxiliary_expectations, strict=True))
+    if side_effect_evidence_only:
+        if objective_scorer is not None and not _judges_side_effects(objective_scorer):
+            objective_scorer = None
+        auxiliary = [(scorer, selected) for scorer, selected in auxiliary if _judges_side_effects(scorer)]
+        if objective_scorer is None and not auxiliary:
+            return {"objective_scores": [], "auxiliary_scores": []}
     return await MessageScorer.score_response_async(
         response=response,
         objective_scorer=objective_scorer,
-        auxiliary_scorers=prepared.auxiliary_scorers,
+        auxiliary_scorers=[scorer for scorer, _ in auxiliary],
         expectation=prepared.objective_expectation,
-        auxiliary_expectations=prepared.auxiliary_expectations,
+        auxiliary_expectations=[selected for _, selected in auxiliary],
     )

pyrit/executor/attack/multi_turn/chunked_request.py

@@ -292,6 +292,8 @@ async def _perform_async(self, *, context: ChunkedRequestAttackContext) -> Attac
 
         # Send each chunk request and collect responses
         response = None
+        delivered_without_response = False
+        received_response = False
         for idx, chunk_prompt in enumerate(chunk_prompts):
             logger.info(f"Sending chunk request {idx + 1}/{len(chunk_prompts)}")
 
@@ -319,8 +321,18 @@ async def _perform_async(self, *, context: ChunkedRequestAttackContext) -> Attac
                     send_context=context.prepended_history_send_context,
                 )
 
+            if response and PromptNormalizer.is_write_only_delivery(response):
+                # The returned request is not a chunk of the target's output, so it must not be
+                # combined into the value that gets scored.
+                delivered_without_response = True
+                response = None
+                context.chunk_responses.append("")
+                context.executed_turns += 1
+                continue
+
             # Store the response
             if response:
+                received_response = True
                 response_text = response.get_value()
                 context.chunk_responses.append(response_text)
                 logger.info(f"Received chunk {idx + 1}: {response_text[:50]}...")
@@ -334,13 +346,19 @@ async def _perform_async(self, *, context: ChunkedRequestAttackContext) -> Attac
         combined_value = "\n".join(context.chunk_responses)
         logger.info(f"Combined {len(context.chunk_responses)} chunk responses")
 
-        # Score the combined value if scorer is configured
-        score = await self._score_combined_value_async(
-            combined_value=combined_value, objective=context.objective, expectation=context.expectation
-        )
+        if delivered_without_response and not received_response:
+            # No chunk produced output, so there is nothing to score. Output from other chunks is
+            # still scored when some chunks were only delivered.
+            score = None
+            outcome, outcome_reason = AttackOutcome.UNDETERMINED, "Objective target returned no response to score"
+        else:
+            # Score the combined value if scorer is configured
+            score = await self._score_combined_value_async(
+                combined_value=combined_value, objective=context.objective, expectation=context.expectation
+            )
 
-        # Determine the outcome
-        outcome, outcome_reason = self._determine_attack_outcome(score=score)
+            # Determine the outcome
+            outcome, outcome_reason = self._determine_attack_outcome(score=score)
 
         # Create attack result
         return AttackResult(

pyrit/executor/attack/multi_turn/crescendo.py

@@ -671,6 +671,12 @@ async def _send_prompt_to_objective_target_async(
         if not response:
             raise ValueError("No response received from objective target")
 
+        if PromptNormalizer.is_write_only_delivery(response):
+            raise ValueError(
+                "The objective target returned no response. Crescendo needs a target response "
+                "for each turn, so it cannot run against a write-only target."
+            )
+
         return response
 
     async def _check_refusal_async(self, context: CrescendoAttackContext, objective: str) -> Score:

pyrit/executor/attack/multi_turn/multi_prompt_sending.py

@@ -259,6 +259,7 @@ async def _perform_async(self, *, context: MultiTurnAttackContext[Any]) -> Attac
 
         response = None
         score = None
+        delivered_without_response = False
 
         for message_index, current_message in enumerate(context.params.user_messages):
             logger.info(f"Processing message {message_index + 1}/{len(context.params.user_messages)}")
@@ -268,9 +269,18 @@ async def _perform_async(self, *, context: MultiTurnAttackContext[Any]) -> Attac
                 current_message=current_message, context=context
             )
 
+            # Only the final send is judged, so each send replaces what the previous one left.
+            response = response_message
+            delivered_without_response = response_message is not None and PromptNormalizer.is_write_only_delivery(
+                response_message
+            )
+            if delivered_without_response:
+                # Keep delivering the sequence; the returned request is not a response.
+                context.executed_turns += 1
+                continue
+
             # Update context with latest response (may be None if sending failed)
             if response_message:
-                response = response_message
                 context.last_response = response
                 context.executed_turns += 1
 
@@ -284,26 +294,33 @@ async def _perform_async(self, *, context: MultiTurnAttackContext[Any]) -> Attac
                     )
                     break
             else:
-                response = None
                 self._logger.warning(f"Failed to send message {message_index + 1}, terminating")
                 break
 
-        # Score the last response including auxiliary and objective scoring
+        # Score the last response including auxiliary and objective scoring. A delivered request is
+        # judged only by scorers that read side-effect evidence, never as response text.
         if response is not None:
             score = await self._evaluate_response_async(
-                response=response, objective=context.objective, expectation=context.expectation
+                response=response,
+                objective=context.objective,
+                expectation=context.expectation,
+                side_effect_evidence_only=delivered_without_response,
             )
         else:
             score = None
 
         # Determine the outcome
-        outcome, outcome_reason = self._determine_attack_outcome(response=response, score=score, context=context)
+        if delivered_without_response and score is None:
+            outcome, outcome_reason = AttackOutcome.UNDETERMINED, "Objective target returned no response to score"
+        else:
+            outcome, outcome_reason = self._determine_attack_outcome(response=response, score=score, context=context)
 
+        last_response = None if delivered_without_response else response
         return AttackResult(
             conversation_id=context.session.conversation_id,
             objective=context.objective,
             atomic_attack_identifier=AtomicAttackIdentifier.build(attack_identifier=self.get_identifier()),
-            last_response=response.get_piece() if response else None,
+            last_response=last_response.get_piece() if last_response else None,
             automated_score=score,
             related_conversations=context.related_conversations,
             outcome=outcome,
@@ -396,7 +413,12 @@ async def _send_prompt_to_objective_target_async(
             )
 
     async def _evaluate_response_async(
-        self, *, response: Message, objective: str, expectation: ScoringExpectation
+        self,
+        *,
+        response: Message,
+        objective: str,
+        expectation: ScoringExpectation,
+        side_effect_evidence_only: bool = False,
     ) -> Score | None:
         """
         Evaluate the response against the objective using the configured scorers.
@@ -408,6 +430,8 @@ async def _evaluate_response_async(
             response (Message): The response from the model.
             objective (str): The natural-language description of the attack's objective.
             expectation (ScoringExpectation): The effective scoring question.
+            side_effect_evidence_only (bool): Run only scorers that judge side-effect evidence, for a
+                delivered request with no target response. Defaults to False.
 
         Returns:
             Score | None: The score from the objective scorer if configured, or None if
@@ -424,6 +448,7 @@ async def _evaluate_response_async(
                 objective_scorer=self._objective_scorer,
                 auxiliary_scorers=self._auxiliary_scorers,
                 expectation=expectation,
+                side_effect_evidence_only=side_effect_evidence_only,
             )
 
         objective_scores = scoring_results["objective_scores"]

pyrit/executor/attack/multi_turn/red_teaming.py

@@ -602,6 +602,12 @@ async def _send_prompt_to_objective_target_async(
                 "Please check the target configuration and ensure it is reachable."
             )
 
+        if PromptNormalizer.is_write_only_delivery(response):
+            raise ValueError(
+                "The objective target returned no response. Red teaming needs a target response "
+                "for each turn, so it cannot run against a write-only target."
+            )
+
         return response
 
     async def _score_response_async(self, *, context: MultiTurnAttackContext[Any]) -> Score | None: