FIX: persist LLMGenericTextConverter target exchanges through the normalizer

pyrit/converter/llm_generic_text_converter.py

@@ -14,7 +14,11 @@
 
 from pyrit.common.apply_defaults import REQUIRED_VALUE, apply_defaults
 from pyrit.converter.converter import Converter, ConverterResult
-from pyrit.exceptions.exception_classes import _DynamicStopAfterAttempt, get_retry_max_num_attempts
+from pyrit.exceptions.exception_classes import (
+    EmptyResponseException,
+    _DynamicStopAfterAttempt,
+    get_retry_max_num_attempts,
+)
 from pyrit.exceptions.exceptions_helpers import log_exception
 from pyrit.models import (
     ComponentIdentifier,
@@ -23,6 +27,7 @@
     PromptDataType,
     SeedPrompt,
 )
+from pyrit.prompt_normalizer import PromptNormalizer
 from pyrit.prompt_target import CHAT_TARGET_REQUIREMENTS, PromptTarget
 
 if TYPE_CHECKING:
@@ -221,17 +226,11 @@ async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text
         if not self.input_supported(input_type):
             raise ValueError("Input type not supported")
 
-        conversation_id = str(uuid.uuid4())
-
+        system_prompt = None
         if self._system_prompt_template:
             system_prompt = self._system_prompt_template.render_template_value(
                 **self._render_values(template=self._system_prompt_template, objective=prompt)
             )
-            (
-                await self._converter_target.set_system_prompt_async(
-                    system_prompt=system_prompt, conversation_id=conversation_id
-                )
-            )
 
         converted_prompt = prompt
         if self._user_prompt_template_with_objective:
@@ -245,7 +244,6 @@ async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text
                     role="user",
                     original_value=prompt,
                     converted_value=converted_prompt,
-                    conversation_id=conversation_id,
                     sequence=1,
                     original_value_data_type=input_type,
                     converted_value_data_type=input_type,
@@ -254,7 +252,7 @@ async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text
             ]
         )
 
-        response_text = await self._send_with_retries_async(request)
+        response_text = await self._send_with_retries_async(request, system_prompt=system_prompt)
         return ConverterResult(output_text=response_text, output_type="text")
 
     def _render_values(self, *, template: SeedPrompt, objective: str | None) -> dict[str, Any]:
@@ -273,7 +271,7 @@ def _render_values(self, *, template: SeedPrompt, objective: str | None) -> dict
             values["objective"] = objective
         return values
 
-    async def _send_with_retries_async(self, request: Message) -> str:
+    async def _send_with_retries_async(self, request: Message, *, system_prompt: str | None = None) -> str:
         """
         Send the request to the converter target, retrying on configured exception types.
 
@@ -283,8 +281,12 @@ async def _send_with_retries_async(self, request: Message) -> str:
         comes from ``self._retry_wait_max_seconds`` (or no wait when unset). The final
         exception is re-raised unchanged.
 
+        Each attempt is persisted in its own conversation (see ``_send_attempt_async``), so
+        every exchange with the converter target, including failed attempts, is kept in memory.
+
         Args:
             request (Message): The message to send to the converter target.
+            system_prompt (str | None): The system prompt to set on each attempt's conversation.
 
         Returns:
             str: The post-processed response text from ``_process_response``.
@@ -293,9 +295,9 @@ async def _send_with_retries_async(self, request: Message) -> str:
             RuntimeError: Defensive guard for an unreachable code path; tenacity always
                 re-raises the underlying exception when retries are exhausted.
         """
+        normalizer = PromptNormalizer()
         if not self._retry_exceptions:
-            response = await self._converter_target.send_prompt_async(message=request)
-            return self._process_response(response[0].get_value())
+            return await self._send_attempt_async(request=request, system_prompt=system_prompt, normalizer=normalizer)
 
         stop_strategy: stop_base = (
             stop_after_attempt(self._max_retry_attempts)
@@ -316,11 +318,60 @@ async def _send_with_retries_async(self, request: Message) -> str:
             after=log_exception,
         ):
             with attempt:
-                response = await self._converter_target.send_prompt_async(message=request)
-                return self._process_response(response[0].get_value())
+                # Each attempt persists its request, so a retry needs fresh piece IDs.
+                attempt_request = request if attempt.retry_state.attempt_number == 1 else request.duplicate()
+                return await self._send_attempt_async(
+                    request=attempt_request, system_prompt=system_prompt, normalizer=normalizer
+                )
 
         raise RuntimeError("unreachable: tenacity reraises on exhaustion")  # pragma: no cover
 
+    async def _send_attempt_async(
+        self, *, request: Message, system_prompt: str | None, normalizer: PromptNormalizer
+    ) -> str:
+        """
+        Send one attempt through the normalizer, which persists the request and the response.
+
+        Each attempt uses a new conversation, so a retry never sees an earlier attempt's turn
+        in its history, and the earlier attempt stays in memory.
+
+        Args:
+            request (Message): The message to send to the converter target.
+            system_prompt (str | None): The system prompt to set on the attempt's conversation.
+            normalizer (PromptNormalizer): The normalizer that sends and persists the exchange.
+
+        Returns:
+            str: The post-processed response text from ``_process_response``.
+
+        Raises:
+            EmptyResponseException: If the converter target returns an empty response.
+            Exception: Whatever the converter target raised, after the failed exchange is persisted.
+        """
+        conversation_id = str(uuid.uuid4())
+        if system_prompt is not None:
+            await self._converter_target.set_system_prompt_async(
+                system_prompt=system_prompt, conversation_id=conversation_id
+            )
+        try:
+            response = await normalizer.send_prompt_async(
+                message=request, conversation_id=conversation_id, target=self._converter_target
+            )
+        except Exception as error:
+            # The normalizer persists the request and an error response, then wraps the target's
+            # error in a plain Exception. Raise the target's own error, as before persistence.
+            if type(error) is not Exception or not isinstance(error.__cause__, Exception):
+                raise
+            target_error = error.__cause__
+        else:
+            # The normalizer records an empty reply as an empty response instead of raising.
+            if any(piece.response_error == "empty" for piece in response.message_pieces):
+                raise EmptyResponseException(
+                    message=f"The converter target returned an empty response: {conversation_id}"
+                )
+            return self._process_response(response.get_value())
+        # Raised outside the handler so the target's error is not chained to the normalizer's wrapper.
+        raise target_error
+
     def _process_response(self, response_text: str) -> str:
         """
         Post-process the raw LLM response text.

tests/unit/converter/test_generic_llm_converter.py

@@ -22,13 +22,16 @@
     TranslationConverter,
     VariationConverter,
 )
-from pyrit.exceptions import InvalidJsonException
+from pyrit.exceptions import EmptyResponseException, InvalidJsonException
 from pyrit.memory import SQLiteMemory
 from pyrit.models import ConverterIdentifier, Message, MessagePiece, SeedPrompt
 from pyrit.prompt_normalizer import ConverterConfiguration, PromptNormalizer
 from pyrit.prompt_target.common.prompt_target import PromptTarget
 from pyrit.registry import ConverterRegistry
 
+# The converters persist their target exchanges through the normalizer, which needs memory.
+pytestmark = pytest.mark.usefixtures("patch_central_database")
+
 
 @pytest.fixture
 def mock_target() -> MagicMock:
@@ -41,7 +44,7 @@ def mock_target() -> MagicMock:
             )
         ]
     )
-    target.send_prompt_async = AsyncMock(return_value=[response])
+    target.send_prompt_async = AsyncMock(side_effect=lambda **_: [response.duplicate()])
     target.get_identifier.return_value = get_mock_target_identifier("MockLLMTarget")
     return target
 
@@ -595,3 +598,109 @@ async def test_per_call_prompt_does_not_change_converter_identifier(mock_target,
     assert read_after.get_identifier().hash == identifier_before_conversion.hash
     assert {identifier.hash for identifier in sent_identifiers} == {identifier_before_conversion.hash}
     assert identifier_before_conversion.params.get("template_parameters_hash") is None
+
+
+class _RecordingTarget(MockPromptTarget):
+    """A local chat target that records the history each call receives and can fail on demand."""
+
+    def __init__(self, *, error: Exception | None = None) -> None:
+        super().__init__()
+        self.error = error
+        self.histories: list[list[str]] = []
+
+    async def _send_prompt_to_target_async(self, *, normalized_conversation: list[Message]) -> list[Message]:
+        self.histories.append([message.api_role for message in normalized_conversation])
+        if self.error is not None:
+            raise self.error
+        return await super()._send_prompt_to_target_async(normalized_conversation=normalized_conversation)
+
+
+async def _converter_conversations_async(memory: SQLiteMemory) -> dict[str, list[MessagePiece]]:
+    conversations: dict[str, list[MessagePiece]] = defaultdict(list)
+    for piece in sorted(await memory.get_message_pieces_async(), key=lambda piece: piece.sequence):
+        conversations[piece.conversation_id].append(piece)
+    return dict(conversations)
+
+
+async def test_convert_async_persists_the_target_exchange(sqlite_instance: SQLiteMemory) -> None:
+    target = _RecordingTarget()
+    converter = LLMGenericTextConverter(
+        converter_target=target,
+        system_prompt_template=SeedPrompt(value="Rewrite the text.", data_type="text"),
+        user_prompt_template_with_objective=SeedPrompt(
+            value="Text: {{ objective }}", data_type="text", parameters=["objective"]
+        ),
+    )
+
+    result = await converter.convert_async(prompt="hello")
+
+    [pieces] = (await _converter_conversations_async(sqlite_instance)).values()
+    assert [piece.api_role for piece in pieces] == ["system", "user", "assistant"]
+    assert pieces[1].original_value == "hello"
+    assert pieces[1].converted_value == "Text: hello"
+    assert pieces[2].converted_value == result.output_text == "default"
+
+
+async def test_each_retry_attempt_is_persisted_in_its_own_conversation(sqlite_instance: SQLiteMemory) -> None:
+    class _RetryOnceConverter(LLMGenericTextConverter):
+        RETRY_EXCEPTIONS = (InvalidJsonException,)
+        failures_left = 1
+
+        def _process_response(self, response_text: str) -> str:
+            if self.failures_left:
+                self.failures_left -= 1
+                raise InvalidJsonException(message="bad")
+            return response_text
+
+    target = _RecordingTarget()
+    converter = _RetryOnceConverter(
+        converter_target=target, system_prompt_template=SeedPrompt(value="Rewrite the text.", data_type="text")
+    )
+
+    result = await converter.convert_async(prompt="hello")
+
+    assert result.output_text == "default"
+    conversations = await _converter_conversations_async(sqlite_instance)
+    assert len(conversations) == 2
+    for pieces in conversations.values():
+        assert [piece.api_role for piece in pieces] == ["system", "user", "assistant"]
+    # The retry starts from a clean conversation instead of replaying the failed attempt.
+    assert target.histories == [["system", "user"], ["system", "user"]]
+
+
+async def test_failed_target_call_is_persisted_and_raises_the_target_error(sqlite_instance: SQLiteMemory) -> None:
+    converter = LLMGenericTextConverter(converter_target=_RecordingTarget(error=ValueError("target down")))
+
+    with pytest.raises(ValueError, match="target down"):
+        await converter.convert_async(prompt="hello")
+
+    [pieces] = (await _converter_conversations_async(sqlite_instance)).values()
+    assert [piece.api_role for piece in pieces] == ["user", "assistant"]
+    assert pieces[0].original_value == "hello"
+    assert pieces[1].response_error == "processing"
+    assert "target down" in pieces[1].converted_value
+
+
+async def test_retry_on_target_error_matches_the_unwrapped_error(sqlite_instance: SQLiteMemory) -> None:
+    target = _RecordingTarget(error=ConnectionError("flaky"))
+    converter = LLMGenericTextConverter(converter_target=target, retry_exceptions=(ConnectionError,))
+
+    with pytest.raises(ConnectionError, match="flaky"):
+        await converter.convert_async(prompt="hello")
+
+    # RETRY_MAX_NUM_ATTEMPTS is set to 2 in conftest.py; both failed exchanges are kept.
+    assert len(target.histories) == 2
+    assert len(await _converter_conversations_async(sqlite_instance)) == 2
+
+
+async def test_empty_target_response_raises_empty_response_exception(sqlite_instance: SQLiteMemory) -> None:
+    converter = LLMGenericTextConverter(
+        converter_target=_RecordingTarget(error=EmptyResponseException(message="nothing"))
+    )
+
+    with pytest.raises(EmptyResponseException):
+        await converter.convert_async(prompt="hello")
+
+    [pieces] = (await _converter_conversations_async(sqlite_instance)).values()
+    assert [piece.api_role for piece in pieces] == ["user", "assistant"]
+    assert pieces[1].response_error == "empty"

tests/unit/converter/test_image_prompt_style_converter.py

@@ -13,6 +13,9 @@
 from pyrit.prompt_target.common.prompt_target import PromptTarget
 from pyrit.registry import ConverterRegistry
 
+# The converters persist their target exchanges through the normalizer, which needs memory.
+pytestmark = pytest.mark.usefixtures("patch_central_database")
+
 
 @pytest.fixture
 def mock_target() -> MagicMock:
@@ -25,7 +28,7 @@ def mock_target() -> MagicMock:
             )
         ]
     )
-    target.send_prompt_async = AsyncMock(return_value=[response])
+    target.send_prompt_async = AsyncMock(side_effect=lambda **_: [response.duplicate()])
     target.get_identifier.return_value = get_mock_target_identifier("MockLLMTarget")
     return target
 

tests/unit/converter/test_math_prompt_converter.py

@@ -11,6 +11,9 @@
 from pyrit.models import Message, MessagePiece, SeedPrompt
 from pyrit.prompt_target import PromptTarget
 
+# The converters persist their target exchanges through the normalizer, which needs memory.
+pytestmark = pytest.mark.usefixtures("patch_central_database")
+
 
 async def test_math_prompt_converter_convert_async():
     # Mock the converter target - use MagicMock for synchronous methods

tests/unit/converter/test_persuasion_converter.py

@@ -10,6 +10,9 @@
 from pyrit.exceptions.exception_classes import InvalidJsonException
 from pyrit.models import Message, MessagePiece
 
+# The converters persist their target exchanges through the normalizer, which needs memory.
+pytestmark = pytest.mark.usefixtures("patch_central_database")
+
 
 def test_persuasion_converter_raises_when_converter_target_is_none():
     with pytest.raises(ValueError, match="converter_target is required"):
@@ -75,7 +78,7 @@ async def test_persuasion_converter_send_prompt_async_bad_json_exception_retries
                 )
             ]
         )
-        mock_create.return_value = [message]
+        mock_create.side_effect = lambda **_: [message.duplicate()]
 
         with patch("asyncio.sleep", new_callable=AsyncMock):
             with pytest.raises(InvalidJsonException):
@@ -102,7 +105,9 @@ async def test_persuasion_converter_extracts_mutated_text(sqlite_instance):
             )
         ]
     )
-    with patch.object(prompt_target, "send_prompt_async", new=AsyncMock(return_value=[response])):
+    with patch.object(
+        prompt_target, "send_prompt_async", new=AsyncMock(side_effect=lambda **_: [response.duplicate()])
+    ):
         result = await prompt_persuasion.convert_async(prompt="testing")
     assert result.output_text == "rephrased prompt"
 
@@ -123,7 +128,9 @@ async def test_persuasion_converter_missing_mutated_text_raises_invalid_json(sql
             )
         ]
     )
-    with patch.object(prompt_target, "send_prompt_async", new=AsyncMock(return_value=[response])):
+    with patch.object(
+        prompt_target, "send_prompt_async", new=AsyncMock(side_effect=lambda **_: [response.duplicate()])
+    ):
         with patch("asyncio.sleep", new_callable=AsyncMock):
             with pytest.raises(InvalidJsonException, match="missing 'mutated_text' key"):
                 await prompt_persuasion.convert_async(prompt="testing")

tests/unit/converter/test_random_translation_converter.py

@@ -15,6 +15,9 @@
 from pyrit.models import Message, MessagePiece, PromptDataType
 from pyrit.prompt_target.common.prompt_target import PromptTarget
 
+# The converters persist their target exchanges through the normalizer, which needs memory.
+pytestmark = pytest.mark.usefixtures("patch_central_database")
+
 
 def test_random_translation_converter_raises_when_converter_target_is_none():
     with pytest.raises(ValueError, match="converter_target is required"):
@@ -32,7 +35,7 @@ def mock_target() -> MagicMock:
             )
         ]
     )
-    target.send_prompt_async = AsyncMock(return_value=[response])
+    target.send_prompt_async = AsyncMock(side_effect=lambda **_: [response.duplicate()])
     target.get_identifier.return_value = get_mock_target_identifier("MockTranslationTarget")
     return target
 

tests/unit/converter/test_role_play_tool_call_converter.py

@@ -16,6 +16,10 @@
 from pyrit.registry import ConverterRegistry
 from pyrit.registry.resolution import derive_parameters
 
+# The converters persist their target exchanges through the normalizer, which needs memory.
+pytestmark = pytest.mark.usefixtures("patch_central_database")
+
+
 INCOMING_MESSAGE = "Please refund $25 to the original payment method for order-1."
 VALID_CALL = 'refund {"order": "order-1", "amount": 25}'
 ACTION_DESIRED = 'refund {"order": "order-1", "amount": 80}'
@@ -30,7 +34,7 @@ def isolate_defaults() -> Generator[None, None, None]:
 def _mock_target() -> MagicMock:
     target = MagicMock(spec=PromptTarget)
     response = Message(message_pieces=[MessagePiece(role="assistant", original_value="written message")])
-    target.send_prompt_async = AsyncMock(return_value=[response])
+    target.send_prompt_async = AsyncMock(side_effect=lambda **_: [response.duplicate()])
     target.get_identifier.return_value = get_mock_target_identifier("MockRolePlayTarget")
     return target