Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .github/instructions/scenarios.instructions.md
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,11 @@ Options:
before overrides, validation, and preparation. Do not rewrite user selections in validators.
- Declare `DATASET_OPTIONS` (or parameter-aware `get_dataset_options()`): `ANY` only for
generic named populations, `SUBSET` for approved alternatives, `FIXED` for ingredients.
The base `get_dataset_options()` reports `FIXED` for inline `seeds` / `seed_groups`, since
named sources can't replace them; keep that in an override. Their limits and filters keep
the declared capabilities. Set `min_selected`
when a scenario has no default sources and can't run without one, so clients mark the
selection required instead of checking the scenario name.
Disable limits that would truncate ingredients. Compound source edits are ambiguous;
preserve child shaping/caps and allow only supported global edits.
- Estimates expose `dataset_defaults`, `effective_datasets`, and `dataset_options`.
Expand Down
12 changes: 11 additions & 1 deletion doc/gui/0_gui.md
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,15 @@ Each scenario defines a test workflow with its techniques, data, and scoring.
Open one to select a target and adjust the settings it supports, such as
techniques, dataset selection, and concurrency.

**Browse datasets** in the Datasets section header opens the dataset catalog in a
new tab. **Reset datasets** returns the datasets and the per-dataset limit to the
scenario's defaults, and appears once you change them. To open a scanner with
datasets already selected, add one `dataset` parameter per name, for example
`/scanner/launch/foundry.red_team_agent?dataset=harmbench&dataset=my_dataset`. The
names replace the default or preset datasets; other dataset settings, such as
limits and filters, stay. A scenario that can't take the linked datasets won't
launch with them and offers a link to open it without them.

Use the chevron on the right of a technique to inspect its identity JSON.
This does not change the checkbox selection on the left.

Expand Down Expand Up @@ -258,7 +267,8 @@ for shared behavior.

Browse datasets already loaded into backend memory. Open a dataset to inspect
its examples, search text, or filter by seed type. Supported media can be
previewed on request.
previewed on request. The **Origins** filter lists datasets whose seeds came from
a given origin, such as local files or a remote provider.

Use **Import into Chat** to stage an example directly into Chat without copying
text or reattaching files:
Expand Down
12 changes: 7 additions & 5 deletions frontend/e2e/numeric-controls.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -253,7 +253,8 @@ test('dataset sources and explicit loading are separate from Parameters', async
await section.getByRole('button', { name: 'Edit Maximum objectives' }).click()
await total.fill('0')
await total.press('Escape')
await section.getByRole('button', { name: 'Reset datasets' }).click()
// Reset clears only source and per-dataset edits, so a total-only edit doesn't offer it.
await expect(section.getByRole('button', { name: 'Reset datasets' })).toHaveCount(0)
await expect(section.getByRole('alert')).toContainText('Maximum objectives must be a positive integer')
await expect(page.getByRole('button', { name: 'Launch scan', exact: true })).toBeDisabled()
await section.screenshot({ path: testInfo.outputPath('datasets-invalid.png') })
Expand Down Expand Up @@ -293,8 +294,8 @@ test('dataset rows add immediately, edit in place, share filters, and reset with
await page.goto(`/scanner/launch/${SCENARIO.scenario_name}`)
const section = page.getByRole('region', { name: 'Datasets', exact: true })
await section.getByRole('combobox', { name: 'Add dataset' }).click()
await page.getByRole('option', { name: 'custom_objectives (user)', exact: true }).click()
await expect(section.getByText('custom_objectives (user)', { exact: true })).toBeVisible()
await page.getByRole('option', { name: 'custom_objectives (User)', exact: true }).click()
await expect(section.getByText('custom_objectives (User)', { exact: true })).toBeVisible()
await section.getByRole('button', { name: 'Remove harmbench' }).click()
await expect(section.getByRole('button', { name: 'Load datasets' })).toHaveCount(0)
await section.getByRole('button', { name: 'Edit custom_objectives objective limit' }).click()
Expand All @@ -304,9 +305,10 @@ test('dataset rows add immediately, edit in place, share filters, and reset with
await section.getByRole('button', { name: 'Filter data type Image' }).click()
await expect(section.getByRole('button', { name: 'Filter data type Text' })).toHaveAttribute('aria-pressed', 'true')
await expect(section.getByRole('link', { name: 'Browse datasets' })).toHaveAttribute('href', '/datasets')
await expect(section.getByRole('link', { name: 'Browse datasets' })).toHaveAttribute('target', '_blank')
await expect(section.getByText('Filters:', { exact: true })).toHaveCount(0)
await section.getByRole('button', { name: 'Reset datasets' }).click()
await expect(section.getByText('harmbench (unknown)', { exact: true })).toBeVisible()
await expect(section.getByText('harmbench (Unknown)', { exact: true })).toBeVisible()
await expect(section.getByRole('button', { name: 'Load datasets' })).toBeVisible()
await expect(section.getByRole('button', { name: 'Remove type filter text' })).toHaveCount(0)
})
Expand Down Expand Up @@ -345,7 +347,7 @@ test('dataset refresh requires preview confirmation and warns before replacing u
await page.getByRole('combobox', { name: 'Objective Target', exact: true }).selectOption('mock-target')
const section = page.getByRole('region', { name: 'Datasets', exact: true })
await section.getByRole('combobox', { name: 'Add dataset' }).click()
await page.getByRole('option', { name: 'custom_objectives (user)', exact: true }).click()
await page.getByRole('option', { name: 'custom_objectives (User)', exact: true }).click()
await section.getByRole('button', { name: 'Remove harmbench' }).click()
await section.getByRole('radio', { name: 'Refresh', exact: true }).check()
await expect(section.getByRole('radio', { name: 'Refresh', exact: true })).toBeChecked()
Expand Down
4 changes: 2 additions & 2 deletions frontend/e2e/scenario-datasets.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -215,11 +215,11 @@ test('configures, saves, refreshes, estimates without a target, and launches the
const search = page.getByRole('combobox', { name: 'Add dataset', exact: true })
await search.focus()
await search.pressSequentially('SEARCHABLE')
await expect(page.getByRole('listbox').getByRole('option')).toHaveText(['acceptance_searchable (local)'])
await expect(page.getByRole('listbox').getByRole('option')).toHaveText(['acceptance_searchable (Local files)'])
await capture(page, testInfo, 'dataset-search-keyboard', search)
await search.press('ArrowDown')
await search.press('Enter')
await expect(page.getByRole('row').filter({ hasText: 'acceptance_searchable (local)' })).toBeVisible()
await expect(page.getByRole('row').filter({ hasText: 'acceptance_searchable (Local files)' })).toBeVisible()
await setLimit(page, 'acceptance_primary objective limit', '3')
await setLimit(page, 'acceptance_searchable objective limit', '1')
await setLimit(page, 'Maximum objectives', '4')
Expand Down
20 changes: 20 additions & 0 deletions frontend/src/components/Datasets/DatasetCatalog.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -577,6 +577,26 @@ describe('DatasetCatalog', () => {
expect(screen.getByRole('link', { name: '(unnamed)' })).toBeInTheDocument()
})

it('filters to generated datasets from the origin control', async () => {
const user = userEvent.setup()
mockCatalog([
makeDataset({ name: 'support_policy', selection_key: 'dataset:named:support_policy', origins: ['generated'] }),
makeDataset({ name: 'harmbench', selection_key: 'dataset:named:harmbench', origins: ['remote'] }),
])
render(<Harness initialPath="/datasets" />)
await screen.findByRole('link', { name: 'support_policy' })

await user.click(screen.getByTestId('dataset-origin-filter'))
expect(await screen.findByRole('menuitemcheckbox', { name: 'Remote provider' })).toBeInTheDocument()
await user.click(screen.getByRole('menuitemcheckbox', { name: 'Generated' }))

expect(currentUrl().searchParams.getAll('origin')).toEqual(['generated'])
expect(screen.getByRole('link', { name: 'support_policy' })).toBeInTheDocument()
expect(screen.queryByRole('link', { name: 'harmbench' })).not.toBeInTheDocument()
await user.keyboard('{Escape}')
expect(screen.getByTestId('dataset-origin-filter')).toHaveValue('Origins: Generated')
})

it('reaches search, clear, and the dataset link from the keyboard', async () => {
const user = userEvent.setup()
render(<Harness initialPath="/datasets" />)
Expand Down
22 changes: 21 additions & 1 deletion frontend/src/components/Datasets/DatasetCatalog.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import {
import { SearchRegular } from '@fluentui/react-icons'
import { useLocation, useNavigate, useSearchParams } from 'react-router'

import { datasetOriginLabel } from '@/components/Scenarios/datasetPresentation'
import SearchableMultiCombobox from '@/components/SearchableMultiCombobox'
import { useLoadedDatasets } from '@/hooks/useLoadedDatasets'
import type { DatasetInfo, FilterOption } from '@/types'
Expand Down Expand Up @@ -61,6 +62,14 @@ function harmOptions(datasets: readonly DatasetInfo[], view: DatasetCatalogQuery
return uniqueLabels(values)
}

function originOptions(datasets: readonly DatasetInfo[]): FilterOption[] {
const values = new Set<string>()
for (const dataset of datasets) for (const origin of dataset.origins ?? []) values.add(origin)
return [...values]
.map((value) => ({ value, label: datasetOriginLabel(value) }))
.sort((left, right) => left.label.localeCompare(right.label, 'en', { sensitivity: 'base' }))
}

/** Stored datasets and available providers share URL-backed views and filters. */
export default function DatasetCatalog() {
const styles = useDatasetCatalogStyles()
Expand All @@ -87,6 +96,7 @@ export default function DatasetCatalog() {
const selectedModalities = [...new Set(query.modalities.map(
(modality: string) => normalizeDatasetModality(modality, modalityAliases),
))]
const origins = useMemo(() => originOptions(viewDatasets), [viewDatasets])
const filtersActive = !catalogQueryIsDefault({ ...query, view: 'memory' })

const updateQuery = (next: DatasetCatalogQuery): void => {
Expand Down Expand Up @@ -116,7 +126,7 @@ export default function DatasetCatalog() {
<Text size={200}>
{view === 'memory'
? 'Filters describe stored seeds. Not labeled means stored seeds without harm categories.'
: 'Filters use provider declarations for built-ins and stored values for other datasets. Not labeled applies only to stored values.'}
: 'Filters use provider declarations for built-ins and stored values for other datasets. Origins and Not labeled apply only to stored values.'}
</Text>

<div className={styles.filters}>
Expand Down Expand Up @@ -166,6 +176,16 @@ export default function DatasetCatalog() {
onChange={(_, data) => updateQuery({ ...query, unlabeled: data.checked === true })}
label="Not labeled"
/>
<SearchableMultiCombobox
className={styles.filterControl}
ariaLabel="Origins"
placeholder="All origins"
summaryPrefix="Origins"
options={origins}
selectedOptions={[...query.origins]}
onSelect={(selected) => updateQuery({ ...query, origins: selected })}
testId="dataset-origin-filter"
/>
<Select
className={styles.filterControl}
aria-label="Sort datasets"
Expand Down
20 changes: 20 additions & 0 deletions frontend/src/components/Datasets/datasetQuery.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,7 @@ describe('datasetQuery', () => {
modalities: ['image/png', 'text'],
harmCategories: ['a&b', 'データ'],
unlabeled: true,
origins: ['generated', 'remote'],
sort: 'logical_examples_desc',
}
const params = catalogQueryToSearchParams(query)
Expand All @@ -142,6 +143,7 @@ describe('datasetQuery', () => {
expect(catalogQueryIsDefault(restored)).toBe(false)
expect(params.getAll('modality')).toEqual(['image/png', 'text'])
expect(params.getAll('harm_category')).toEqual(['a&b', 'データ'])
expect(params.getAll('origin')).toEqual(['generated', 'remote'])
})

it('ignores an unknown sort value', () => {
Expand All @@ -160,12 +162,15 @@ describe('datasetQuery', () => {
['harm_category', 'violence'],
['harm_category', 'データ'],
['harm_category', 'a&b'],
['origin', 'generated'],
['origin', 'generated'],
])

expect(catalogQueryFromSearchParams(params)).toEqual({
...DEFAULT_DATASET_CATALOG_QUERY,
modalities: ['text', 'image/png', 'audio_path'],
harmCategories: ['violence', 'a&b', 'データ'],
origins: ['generated'],
})
})

Expand Down Expand Up @@ -215,6 +220,21 @@ describe('datasetQuery', () => {
})).toEqual([])
})

it('filters by any matching seed origin and treats a missing origin list as no match', () => {
const generated = makeDataset({ name: 'Generated', selection_key: 'dataset:named:generated', origins: ['generated'] })
const mixed = makeDataset({ name: 'Mixed', selection_key: 'dataset:named:mixed', origins: ['local', 'remote'] })
const legacy = makeDataset({ name: 'Legacy', selection_key: 'dataset:named:legacy' })
const datasets = [generated, mixed, legacy]

expect(filterDatasets(datasets, DEFAULT_DATASET_CATALOG_QUERY).map((item) => item.name))
.toEqual(['Generated', 'Legacy', 'Mixed'])
expect(filterDatasets(datasets, { ...DEFAULT_DATASET_CATALOG_QUERY, origins: ['generated'] }).map((item) => item.name))
.toEqual(['Generated'])
expect(filterDatasets(datasets, { ...DEFAULT_DATASET_CATALOG_QUERY, origins: ['remote', 'generated'] }).map((item) => item.name))
.toEqual(['Generated', 'Mixed'])
expect(catalogQueryIsDefault({ ...DEFAULT_DATASET_CATALOG_QUERY, origins: ['generated'] })).toBe(false)
})

it('sorts names and known counts with deterministic ties', () => {
const alpha = makeDataset({
name: 'alpha',
Expand Down
13 changes: 13 additions & 0 deletions frontend/src/components/Datasets/datasetQuery.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import type { DatasetInfo } from '@/types'
const SEARCH_PARAM = 'q'
const MODALITY_PARAM = 'modality'
const HARM_CATEGORY_PARAM = 'harm_category'
const ORIGIN_PARAM = 'origin'
const UNLABELED_PARAM = 'unlabeled'
const SORT_PARAM = 'sort'

Expand Down Expand Up @@ -30,6 +31,8 @@ export interface DatasetCatalogQuery {
readonly modalities: readonly string[]
readonly harmCategories: readonly string[]
readonly unlabeled: boolean
/** Seed origins such as `generated` or `remote`; a dataset matches if any of its seeds has one. */
readonly origins: readonly string[]
readonly sort: DatasetSort
}

Expand All @@ -38,6 +41,7 @@ export const DEFAULT_DATASET_CATALOG_QUERY: DatasetCatalogQuery = {
modalities: [],
harmCategories: [],
unlabeled: false,
origins: [],
sort: DEFAULT_DATASET_SORT,
}

Expand All @@ -55,6 +59,7 @@ export function catalogQueryFromSearchParams(params: URLSearchParams): DatasetCa
modalities: [...new Set(params.getAll(MODALITY_PARAM))],
harmCategories: [...new Set(params.getAll(HARM_CATEGORY_PARAM))],
unlabeled: params.get(UNLABELED_PARAM) === 'true',
origins: [...new Set(params.getAll(ORIGIN_PARAM))],
sort: sortValue != null && isDatasetSort(sortValue) ? sortValue : DEFAULT_DATASET_SORT,
}
}
Expand All @@ -67,6 +72,7 @@ export function catalogQueryToSearchParams(query: DatasetCatalogQuery): URLSearc
for (const modality of new Set(query.modalities)) params.append(MODALITY_PARAM, modality)
for (const category of new Set(query.harmCategories)) params.append(HARM_CATEGORY_PARAM, category)
if (query.unlabeled) params.set(UNLABELED_PARAM, 'true')
for (const origin of new Set(query.origins)) params.append(ORIGIN_PARAM, origin)
if (query.sort !== DEFAULT_DATASET_SORT) params.set(SORT_PARAM, query.sort)
return params
}
Expand All @@ -77,6 +83,7 @@ export function catalogQueryIsDefault(query: DatasetCatalogQuery): boolean {
&& query.modalities.length === 0
&& query.harmCategories.length === 0
&& !query.unlabeled
&& query.origins.length === 0
&& query.sort === DEFAULT_DATASET_SORT
}

Expand Down Expand Up @@ -132,6 +139,11 @@ function matchesHarm(dataset: DatasetInfo, query: DatasetCatalogQuery): boolean
return categoryMatch || unlabeledMatch
}

function matchesOrigins(dataset: DatasetInfo, origins: readonly string[]): boolean {
if (origins.length === 0) return true
return origins.some((origin) => dataset.origins?.includes(origin) === true)
}

function countSort(sort: DatasetSort): 'asc' | 'desc' | null {
switch (sort) {
case 'logical_examples':
Expand Down Expand Up @@ -167,6 +179,7 @@ export function filterDatasets(
if (!matchesSearch(dataset.name, query.search)) continue
if (!matchesModalities(dataset, query)) continue
if (!matchesHarm(dataset, query)) continue
if (!matchesOrigins(dataset, query.origins)) continue
matched.push(dataset)
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -866,7 +866,7 @@ describe('ScenarioPresetEditor edit mode', () => {
renderEdit()

await user.click(await screen.findByRole('combobox', { name: 'Add dataset' }))
await user.click(screen.getByRole('option', { name: 'xstest (unknown)', exact: true }))
await user.click(screen.getByRole('option', { name: 'xstest (Unknown)', exact: true }))
await editLimit(user, 'Maximum objectives', '25')
await user.click(screen.getByTestId('save-preset-btn'))

Expand All @@ -876,6 +876,14 @@ describe('ScenarioPresetEditor edit mode', () => {
expect(saved.datasets?.max_total).toBe(25)
})

it('opens the dataset catalog in a new tab so unsaved preset edits stay', async () => {
renderEdit()

const actions = await screen.findByRole('group', { name: 'Dataset actions' })
expect(within(actions).getByRole('link', { name: 'Browse datasets' })).toHaveAttribute('target', '_blank')
expect(screen.getAllByRole('link', { name: 'Browse datasets' })).toHaveLength(1)
})

it('refuses to save while the dataset control reports an invalid limit', async () => {
const user = userEvent.setup()
renderEdit()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import { useNavigate, useParams } from 'react-router'

import ParameterField from '@/components/Parameters/ParameterField'
import type { ParameterFormValue } from '@/components/Parameters/parameterForm'
import DatasetActions from '@/components/Scenarios/DatasetActions'
import ScenarioDatasetTable from '@/components/Scenarios/ScenarioDatasetTable'
import ScenarioTechniqueSelector from '@/components/Scenarios/ScenarioTechniqueSelector'
import { scenarioDatasetDefaults, scenarioDatasetOptions } from '@/components/Scenarios/datasetPresentation'
Expand Down Expand Up @@ -507,7 +508,7 @@ function ScenarioPresetEditorContent({ mode, presetName }: ScenarioPresetEditorC
</MessageBarBody>
</MessageBar>
)}
{datasetBaseline && (
{datasetBaseline && <>
<ScenarioDatasetTable
request={config.datasets}
defaults={datasetBaseline.defaults}
Expand All @@ -521,7 +522,8 @@ function ScenarioPresetEditorContent({ mode, presetName }: ScenarioPresetEditorC
onChange={(datasets) => updateConfig({ datasets })}
onValidationChange={setDatasetError}
/>
)}
<DatasetActions />
</>}
</div>
</section>
</>
Expand Down
13 changes: 13 additions & 0 deletions frontend/src/components/Scenarios/DatasetActions.styles.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
import { makeStyles, tokens } from '@fluentui/react-components'

import { mobileTouchTarget } from '@/styles/touchTargets'

export const useDatasetActionsStyles = makeStyles({
root: {
display: 'flex',
flexWrap: 'wrap',
alignItems: 'center',
gap: tokens.spacingHorizontalS,
'& > a': { ...mobileTouchTarget },
},
})
17 changes: 17 additions & 0 deletions frontend/src/components/Scenarios/DatasetActions.test.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
import { render, screen, within } from '@testing-library/react'
import { FluentProvider, webLightTheme } from '@fluentui/react-components'

import DatasetActions from './DatasetActions'

describe('DatasetActions', () => {
it('should open the dataset catalog in a new tab so the form keeps its edits', () => {
render(<FluentProvider theme={webLightTheme}><DatasetActions /></FluentProvider>)

const actions = screen.getByRole('group', { name: 'Dataset actions' })
const browse = within(actions).getByRole('link', { name: 'Browse datasets' })
expect(browse).toHaveAttribute('href', '/datasets')
expect(browse).toHaveAttribute('target', '_blank')
expect(browse).toHaveAttribute('rel', 'noopener noreferrer')
expect(within(actions).getAllByRole('link')).toHaveLength(1)
})
})
Loading
Loading