Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 9 additions & 4 deletions .github/instructions/scenarios.instructions.md
Original file line number Diff line number Diff line change
Expand Up @@ -206,9 +206,14 @@ Options:
The catalog reads options and defaults from registry metadata, never from an estimate.
- Explicit `load_datasets_async()` remains missing-only, regardless of the scan's policy.
It does not refresh or change that policy.
- Resume replays the saved launch request; the stored run plan fixes the selected seed groups,
so caps and fetch policies cannot change a resumed run. Dataset names are checked by the
stored scenario identity. Resume never prepares datasets.
- Resume replays the saved launch request and rebuilds the population stored with the run, so caps,
fetch policies, and later dataset edits, refreshes, or deletes cannot change a resumed run.
Dataset names are checked by the stored scenario identity. Resume never prepares datasets or calls
`_resolve_seed_groups_by_dataset_async`. State that hook sets up for builders (such as a scorer)
must also be set up in `_restore_resume_state_async`: derive it from the stored population or save
it in the run's metadata, and read datasets only when neither works.
- The rebuilt plan must equal the stored plan exactly, or resume refuses and names the atomic groups
that differ.
- Keep custom shaping in `_build_attack_groups()` or `_build_groups_by_dataset_async()`.
Reads use `_collect_seeds_for_dataset_async()` and must not fetch.
- Build synthesized prompts from those filtered seeds, not a second unfiltered memory query.
Expand Down Expand Up @@ -305,7 +310,7 @@ Scenario-specific scorers, selectors, and construction options stay on the scena

On resume, `context.stored_run_plan` carries the original plan. Restore committed
per-input technique choices from it instead of selecting again. The base restores
the planned source population before builders check compatibility. Saved adaptive
the stored planned population before builders check compatibility. Saved adaptive
choices must identify the full technique, including its prompts, not only its inner attack.

### Zero-boilerplate matrix scenarios
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT license.

"""
add scenario plan seed groups.

Revision ID: c7d1e9a3b5f2
Revises: a3f7c9e2b641
Create Date: 2026-11-16 15:30:00.000000
"""

from collections.abc import Sequence

import sqlalchemy as sa
from alembic import op

import pyrit.memory.memory_models

# revision identifiers, used by Alembic.
revision: str = "c7d1e9a3b5f2"
down_revision: str | None = "a3f7c9e2b641"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None


def upgrade() -> None:
"""Store the seed groups each new scenario run planned, so resume doesn't read datasets."""
op.create_table(
"ScenarioPlanSeedGroupEntries",
sa.Column("id", pyrit.memory.memory_models.CustomUUID(), nullable=False),
sa.Column("scenario_result_id", pyrit.memory.memory_models.CustomUUID(), nullable=False),
sa.Column("dataset_name", sa.Unicode(), nullable=False),
sa.Column("position", sa.INTEGER(), nullable=False),
sa.Column("seed_group", sa.JSON(), nullable=False),
sa.Column("has_media", sa.Boolean(), nullable=False),
sa.ForeignKeyConstraint(["scenario_result_id"], ["ScenarioResultEntries.id"], ondelete="CASCADE"),
sa.PrimaryKeyConstraint("id"),
)
op.create_index(
"ix_ScenarioPlanSeedGroupEntries_scenario_result_id_position",
"ScenarioPlanSeedGroupEntries",
["scenario_result_id", "position"],
unique=False,
)
op.create_index(
"ix_ScenarioPlanSeedGroupEntries_has_media", "ScenarioPlanSeedGroupEntries", ["has_media"], unique=False
)


def downgrade() -> None:
"""Drop stored plan seed groups; runs planned after the upgrade can no longer resume."""
op.drop_index("ix_ScenarioPlanSeedGroupEntries_has_media", table_name="ScenarioPlanSeedGroupEntries")
op.drop_index(
"ix_ScenarioPlanSeedGroupEntries_scenario_result_id_position", table_name="ScenarioPlanSeedGroupEntries"
)
op.drop_table("ScenarioPlanSeedGroupEntries")
19 changes: 18 additions & 1 deletion pyrit/memory/dataset_lifecycle.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,13 @@
DatasetLifecycleLock,
DatasetMediaEntry,
PromptMemoryEntry,
ScenarioPlanSeedGroupEntry,
ScenarioResultEntry,
ScorableContentEntry,
SeedEntry,
planned_media_values,
)
from pyrit.models import MEDIA_PATH_DATA_TYPES
from pyrit.models import MEDIA_PATH_DATA_TYPES, ScenarioRunState

if TYPE_CHECKING:
from collections.abc import Awaitable, Callable, Sequence
Expand Down Expand Up @@ -129,6 +132,8 @@ def _entry_media_values(entry: object) -> list[str]:
return []
if isinstance(entry, ScorableContentEntry):
return [entry.value] if entry.data_type in MEDIA_PATH_DATA_TYPES else []
if isinstance(entry, ScenarioPlanSeedGroupEntry):
return planned_media_values(entry.seed_group)
if isinstance(entry, PromptMemoryEntry):
return [
value
Expand Down Expand Up @@ -206,6 +211,9 @@ def stored_media_keys(session: Session) -> set[str]:
"""
Read all live seed, original/converted message, and loose scorer-media references.

Media planned by a scenario run that hasn't completed counts too, so deleting a dataset
doesn't remove files a paused run still needs to resume.

Returns:
set[str]: Canonical identities of the referenced media.
"""
Expand All @@ -226,6 +234,15 @@ def stored_media_keys(session: Session) -> set[str]:
(PromptMemoryEntry.converted_value, PromptMemoryEntry.converted_value_data_type),
):
values.extend(session.scalars(select(value_column).where(type_column.in_(MEDIA_PATH_DATA_TYPES))).all())
for seed_group in session.scalars(
select(ScenarioPlanSeedGroupEntry.seed_group)
.join(ScenarioResultEntry, ScenarioResultEntry.id == ScenarioPlanSeedGroupEntry.scenario_result_id)
.where(
ScenarioPlanSeedGroupEntry.has_media,
ScenarioResultEntry.scenario_run_state != ScenarioRunState.COMPLETED.value,
)
):
values.extend(planned_media_values(seed_group))
return _media_keys(session, [value for value in values if value])


Expand Down
81 changes: 78 additions & 3 deletions pyrit/memory/memory_interface.py
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,7 @@
PromptConverterIdentifierEntry,
PromptMemoryEntry,
ScenarioIdentifierEntry,
ScenarioPlanSeedGroupEntry,
ScenarioResultEntry,
ScorableContentEntry,
ScoreEntry,
Expand All @@ -130,6 +131,7 @@
AttackResult,
AttackResultRole,
AttackResultSelection,
AttackSeedGroup,
AttackTechniqueIdentifier,
ComponentIdentifier,
ContentEntryScorable,
Expand Down Expand Up @@ -7699,17 +7701,33 @@ def _execute_get_unique_attack_attribution(self) -> dict[str, list[str]]:
]
return {"operators": sorted(operators), "operations": sorted(operations)}

def _execute_add_scenario_results_to_memory(self, *, scenario_results: Sequence[ScenarioResult]) -> None:
def _execute_add_scenario_results_to_memory(
self,
*,
scenario_results: Sequence[ScenarioResult],
planned_seed_groups: Mapping[uuid.UUID, Mapping[str, Sequence[AttackSeedGroup]]] | None = None,
) -> None:
"""
Insert a list of scenario results into the memory storage.

Args:
scenario_results: Sequence of ScenarioResult objects to store in the database.
planned_seed_groups: The population each result's run planned, keyed by result ID and
then by dataset name, stored in the same transaction.

Raises:
SQLAlchemyError: If a scenario result or identifier graph cannot be persisted.
"""
entries = [ScenarioResultEntry(entry=scenario_result) for scenario_result in scenario_results]
plan_entries = [
ScenarioPlanSeedGroupEntry(
scenario_result_id=result_id, dataset_name=dataset_name, position=position, seed_group=seed_group
)
for result_id, population in (planned_seed_groups or {}).items()
for position, (dataset_name, seed_group) in enumerate(
(name, group) for name, groups in population.items() for group in groups
)
]
with closing(self._get_session()) as session:
try:
for scenario_result in scenario_results:
Expand All @@ -7720,6 +7738,8 @@ def _execute_add_scenario_results_to_memory(self, *, scenario_results: Sequence[
for entry in entries:
self._validate_activity_operation(session=session, entry=entry)
session.add_all(entries)
session.flush()
session.add_all(plan_entries)
self._recheck_activity_operations(session=session, entries=entries)
session.commit()
except SQLAlchemyError as error:
Expand Down Expand Up @@ -7876,7 +7896,7 @@ def _execute_update_scenario_metadata(
Replace the ``scenario_metadata`` JSON blob on an existing scenario result.

Used by the scenario layer to persist first-run state (e.g.
``objective_hashes``) that resume needs to replay. Performs a
an adaptive run's choices) that resume needs to replay. Performs a
targeted UPDATE so it doesn't clobber other columns.

Args:
Expand Down Expand Up @@ -7946,6 +7966,24 @@ def _execute_increment_scenario_number_tries(
session.commit()
return new_tries

def _execute_get_planned_seed_groups(self, *, scenario_result_id: str) -> dict[str, list[AttackSeedGroup]]:
"""
Read a run's stored planned seed groups.

Returns:
dict[str, list[AttackSeedGroup]]: The population by dataset name, in planned order.
"""
with closing(self._get_session()) as session:
entries = session.scalars(
select(ScenarioPlanSeedGroupEntry)
.where(ScenarioPlanSeedGroupEntry.scenario_result_id == uuid.UUID(scenario_result_id))
.order_by(ScenarioPlanSeedGroupEntry.position)
).all()
population: dict[str, list[AttackSeedGroup]] = {}
for entry in entries:
population.setdefault(entry.dataset_name, []).append(entry.get_seed_group())
return population

def _execute_get_scenario_result_header(self, *, scenario_result_id: str) -> ScenarioResult | None:
"""Return one ScenarioResult header without hydrating linked attack results."""
with closing(self._get_session()) as session:
Expand Down Expand Up @@ -12508,6 +12546,43 @@ async def add_scenario_results_to_memory_async(self, *, scenario_results: Sequen
self._execute_add_scenario_results_to_memory, scenario_results=scenario_results
)

async def add_planned_scenario_result_async(
self, *, scenario_result: ScenarioResult, seed_groups_by_dataset: Mapping[str, Sequence[AttackSeedGroup]]
) -> None:
"""
Insert a new scenario result together with the seed groups its run planned.

Resume rebuilds the run from these groups, so later dataset changes don't affect it.

Args:
scenario_result (ScenarioResult): The new scenario result.
seed_groups_by_dataset (Mapping[str, Sequence[AttackSeedGroup]]): The planned population,
by dataset name, in the order the run was built from.

Raises:
SQLAlchemyError: If the result, its identifier graph, or its seed groups cannot be persisted.
"""
return await self._run_database_operation_async(
self._execute_add_scenario_results_to_memory,
scenario_results=[scenario_result],
planned_seed_groups={scenario_result.id: seed_groups_by_dataset},
)

async def get_planned_seed_groups_async(self, *, scenario_result_id: str) -> dict[str, list[AttackSeedGroup]]:
"""
Return the seed groups a scenario run planned, as stored when the run was created.

Args:
scenario_result_id (str): The scenario result ID.

Returns:
dict[str, list[AttackSeedGroup]]: The planned population by dataset name, in planned
order. Empty when the run planned no groups or was created before they were stored.
"""
return await self._run_database_operation_async(
self._execute_get_planned_seed_groups, scenario_result_id=scenario_result_id
)

def update_scenario_run_state(
self,
*,
Expand Down Expand Up @@ -12706,7 +12781,7 @@ async def update_scenario_metadata_async(self, *, scenario_result_id: str, metad
Replace the ``scenario_metadata`` JSON blob on an existing scenario result.

Used by the scenario layer to persist first-run state (e.g.
``objective_hashes``) that resume needs to replay. Performs a
an adaptive run's choices) that resume needs to replay. Performs a
targeted UPDATE so it doesn't clobber other columns.

Args:
Expand Down
89 changes: 89 additions & 0 deletions pyrit/memory/memory_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,12 +42,14 @@
from pyrit.memory.analytics_identity_v1 import ObjectiveTargetAnalyticsIdentityV1
from pyrit.memory.analytics_sql import ResolvedAttackIdentifierHash
from pyrit.models import (
MEDIA_PATH_DATA_TYPES,
SEED_RESPONSE_JSON_SCHEMA_METADATA_KEY,
AtomicAttackEvaluationIdentifier,
AtomicAttackIdentifier,
AttackIdentifier,
AttackOutcome,
AttackResult,
AttackSeedGroup,
AttackTechniqueIdentifier,
ChatMessageRole,
ComponentIdentifier,
Expand Down Expand Up @@ -2680,3 +2682,90 @@ def __str__(self) -> str:
str: String representation of the scenario result entry
"""
return f"ScenarioResultEntry: {self.scenario_name} (version {self.scenario_version})"


_SIMULATED_CONVERSATION_PROMPT_FIELDS = (
"adversarial_chat_system_prompt",
"simulated_target_system_prompt",
"next_message_system_prompt",
)


def planned_media_values(seed_group: dict[str, Any]) -> list[str]:
"""
Return the media paths a stored planned seed group references.

Args:
seed_group (dict[str, Any]): A ``ScenarioPlanSeedGroupEntry.seed_group`` value.

Returns:
list[str]: The media paths, in seed order.
"""
values: list[str] = []
for seed in seed_group.get("seeds", []):
prompts = [seed]
if seed.get("seed_type") == "simulated_conversation":
prompts = [seed.get(field) for field in _SIMULATED_CONVERSATION_PROMPT_FIELDS]
values.extend(
prompt["value"]
for prompt in prompts
if isinstance(prompt, dict) and prompt.get("data_type") in MEDIA_PATH_DATA_TYPES
)
return values


class ScenarioPlanSeedGroupEntry(Base):
"""
One seed group a scenario run planned, stored when the run is created.

Resume rebuilds the run's population from these rows instead of reading its datasets, so
editing, refreshing, or deleting a dataset doesn't change what a run resumes on. Rows are
deleted with their scenario result.
"""

__tablename__ = "ScenarioPlanSeedGroupEntries"
__table_args__ = (
Index("ix_ScenarioPlanSeedGroupEntries_scenario_result_id_position", "scenario_result_id", "position"),
Index("ix_ScenarioPlanSeedGroupEntries_has_media", "has_media"),
)

id: Mapped[uuid.UUID] = mapped_column(CustomUUID, primary_key=True)
scenario_result_id: Mapped[uuid.UUID] = mapped_column(
CustomUUID, ForeignKey("ScenarioResultEntries.id", ondelete="CASCADE"), nullable=False
)
#: The population key the group was planned under, usually its dataset name.
dataset_name: Mapped[str] = mapped_column(Unicode, nullable=False)
#: The group's order in the planned population, across all datasets.
position: Mapped[int] = mapped_column(INTEGER, nullable=False)
#: The ``AttackSeedGroup`` as JSON.
seed_group: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
#: Whether ``seed_group`` references media files, so media cleanup reads only these rows.
has_media: Mapped[bool] = mapped_column(Boolean, nullable=False)

def __init__(
self, *, scenario_result_id: uuid.UUID, dataset_name: str, position: int, seed_group: AttackSeedGroup
) -> None:
"""
Store one planned seed group.

Args:
scenario_result_id (uuid.UUID): The run that planned the group.
dataset_name (str): The population key the group was planned under.
position (int): The group's order in the planned population.
seed_group (AttackSeedGroup): The planned group.
"""
self.id = uuid.uuid4()
self.scenario_result_id = scenario_result_id
self.dataset_name = dataset_name
self.position = position
self.seed_group = seed_group.model_dump(mode="json")
self.has_media = bool(planned_media_values(self.seed_group))

def get_seed_group(self) -> AttackSeedGroup:
"""
Rebuild the planned seed group.

Returns:
AttackSeedGroup: The group, equal to the one the run planned.
"""
return AttackSeedGroup.model_validate(self.seed_group)
Loading
Loading