Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion .github/workflows/frontend_tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,10 @@ jobs:
config: playwright.config.ts
shard: 1
total: 1
- project: scan-launch
config: playwright.scan-launch.config.ts
shard: 1
total: 1
- project: analytics
config: playwright.analytics.config.ts
shard: 1
Expand Down Expand Up @@ -147,7 +151,7 @@ jobs:
run: npx playwright install --with-deps chromium

- name: Build the isolated fixture frontend
if: matrix.project == 'analytics' || matrix.project == 'paired-outcomes' || matrix.project == 'scenario-hierarchy' || startsWith(matrix.project, 'scenario-datasets-') || matrix.project == 'scenario-techniques'
if: matrix.project == 'analytics' || matrix.project == 'paired-outcomes' || matrix.project == 'scenario-hierarchy' || startsWith(matrix.project, 'scenario-datasets-') || matrix.project == 'scenario-techniques' || matrix.project == 'scan-launch'
run: npm run build

- name: Run E2E shard (no credentials needed)
Expand Down
7 changes: 4 additions & 3 deletions doc/code/setup/0_setup.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,10 @@ select the target to test and any scoring or adversarial targets the chosen tech
needs, or configure them through an initializer. Deterministic scorers do not need a
scoring model, and techniques without generation do not need an adversarial model.

In CoPyRIT, you can create targets and scorers from the launch form without losing
your draft. Missing roles are shown before a scan starts rather than filled with an
implicit OpenAI target. To use the same target for multiple roles, select it for each.
In CoPyRIT, configure targets in the Registry and scorer defaults through initializers.
Select registered targets and an objective scorer in the scan form; suite launches use
configured scorer defaults. Missing launch requirements produce configuration errors
rather than an implicit OpenAI target. Helper roles must be configured explicitly.
See the [scanner options](../../scanner/0_scanner.md#running-scans) for CLI selections.

## Configuration Options
Expand Down
241 changes: 0 additions & 241 deletions frontend/e2e/explicit-targets.seeded.spec.ts

This file was deleted.

1 change: 0 additions & 1 deletion frontend/e2e/numeric-controls.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,6 @@ async function mockNumericControlApis(page: Page): Promise<void> {
'/api/datasets': { items: datasetItems },
[`/api/scenarios/catalog/${SCENARIO.scenario_name}`]: SCENARIO,
[`/api/scenarios/catalog/${SCENARIO.scenario_name}/estimate`]: SCENARIO.default_run_size,
[`/api/scenarios/catalog/${SCENARIO.scenario_name}/readiness`]: { bindings: [], issues: [] },
}
if (path.endsWith('/datasets/load')) {
const edits: ScenarioDatasetRequest = route.request().postDataJSON()?.datasets ?? {}
Expand Down
154 changes: 154 additions & 0 deletions frontend/e2e/scan-launch.seeded.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
import { mkdir } from 'node:fs/promises'
import path from 'node:path'

import { expect, test, type APIRequestContext, type Page } from '@playwright/test'

import type { RunScenarioRequest, ScenarioRunSummary, TargetListResponse } from '../src/types'
import { compatibilityHeaders } from './_compatibility'
import { waitForAppReady } from './_runtime'

test.use({
// eslint-disable-next-line no-empty-pattern
extraHTTPHeaders: async ({}, runTest) => runTest(compatibilityHeaders()),
})

async function targetSnapshot(request: APIRequestContext): Promise<string[]> {
const identities: string[] = []
let cursor: string | null | undefined
do {
const response = await request.get('/api/targets', {
params: { limit: 200, ...(cursor ? { cursor } : {}) },
})
expect(response.ok(), await response.text()).toBe(true)
const targets: TargetListResponse = await response.json()
identities.push(...targets.items.map(target =>
JSON.stringify([target.target_registry_name, target.identifier.hash])))
if (!targets.pagination.has_more) break
expect(targets.pagination.next_cursor).toBeTruthy()
cursor = targets.pagination.next_cursor
} while (cursor)
return identities.sort()
}

async function capture(page: Page, name: string): Promise<void> {
const directory = process.env.PYRIT_EVIDENCE_DIR || test.info().outputPath('scan-launch')
await mkdir(directory, { recursive: true })
await page.screenshot({ path: path.join(directory, `${name}.png`), fullPage: true, animations: 'disabled' })
}

for (const width of [1280, 390]) {
test(`browses and launches with the original scan controls at ${width}px @seeded`, async ({ page, request }) => {
await page.setViewportSize({ width, height: 900 })
const initialTargets = await targetSnapshot(request)
const catalog = await request.get('/api/scenarios/catalog')
expect(catalog.ok(), await catalog.text()).toBe(true)
await page.goto('/scanner/launch/technique.execute')
await waitForAppReady(page)
await expect(page.getByTestId('scenario-error')).toHaveCount(0)
await expect(page.getByText('Environment variable OPENAI_CHAT_MODEL is required')).toHaveCount(0)
await expect(page.getByRole('button', { name: /Create .*target|Create objective scorer/i })).toHaveCount(0)
await expect(page.getByRole('combobox', { name: 'Scoring Target', exact: true })).toHaveCount(0)
await expect(page.getByRole('combobox', { name: 'Objective scorer', exact: true }))
.toHaveValue('')
await expect(page.getByRole('option', { name: 'Use scenario default' })).toHaveCount(1)
const objective = page.getByRole('textbox', { name: 'objective', exact: true })
await objective.fill(`Offline scan acceptance ${width}`)
await page.getByRole('radio', { name: 'flip', exact: true }).check()
expect(await targetSnapshot(request)).toEqual(initialTargets)
await page.getByRole('region', { name: 'Objective', exact: true }).scrollIntoViewIfNeeded()
await capture(page, `scan-default-${width}`)

const target = page.getByTestId('scenario-target-select')
if (await target.inputValue() === '') {
await page.getByTestId('launch-scenario-btn').click()
await expect(page.getByRole('alert').filter({ hasText: 'Select a target.' })).toBeVisible()
}
await target.selectOption('browser_offline')
await page.getByRole('region', { name: 'Objective', exact: true }).scrollIntoViewIfNeeded()
await capture(page, `scan-configured-${width}`)
await page.getByTestId('launch-scenario-btn').click()
const preview = page.getByRole('dialog')
await expect(preview.getByText('Use scenario default')).toBeVisible()
await expect(preview.getByText('Scoring Target', { exact: true })).toHaveCount(0)
await capture(page, `scan-preview-${width}`)
const started = page.waitForResponse(response =>
response.request().method() === 'POST' && new URL(response.url()).pathname === '/api/scenarios/runs')
await page.getByTestId('confirm-launch-scenario-btn').click()
const response = await started
expect(response.ok(), await response.text()).toBe(true)
const run: ScenarioRunSummary = await response.json()
await expect.poll(async () => (await (await request.get(
`/api/scenarios/runs/${run.scenario_result_id}`,
)).json()).status).toBe('COMPLETED')
const savedResponse = await request.get(`/api/scenarios/runs/${run.scenario_result_id}/launch-configuration`)
expect(savedResponse.ok(), await savedResponse.text()).toBe(true)
const saved: RunScenarioRequest = await savedResponse.json()
expect(saved.target_name).toBe('browser_offline')
expect(saved.objective_scorer).toBeFalsy()
expect(saved.scoring_target_name).toBeFalsy()
expect(saved.adversarial_target_name).toBeFalsy()
expect(saved.techniques).toEqual(['flip'])
expect(await targetSnapshot(request)).toEqual(initialTargets)
})

test(`reports a missing generation target only when launching at ${width}px @seeded`, async ({ page, request }) => {
await page.setViewportSize({ width, height: 900 })
const initialTargets = await targetSnapshot(request)
const before = await (await request.get('/api/scenarios/runs')).json()
await page.goto('/scanner/launch/technique.execute')
await waitForAppReady(page)
await page.getByTestId('scenario-target-select').selectOption('browser_offline')
await page.getByRole('textbox', { name: 'objective', exact: true }).fill('Missing generation role')
await page.getByRole('radio', { name: 'red_teaming', exact: true }).check()
const form = page.getByRole('form', { name: 'Scenario run configuration' })
await expect(form.getByRole('alert')).toHaveCount(0)
await page.getByTestId('launch-scenario-btn').click()
await expect(page.getByRole('dialog').getByText('Use server default')).toBeVisible()
await page.getByTestId('confirm-launch-scenario-btn').click()
await expect(form.getByRole('alert')).toContainText('adversarial_chat')
await expect(page.getByTestId('launch-scenario-btn')).toBeEnabled()
await form.getByRole('alert').scrollIntoViewIfNeeded()
await capture(page, `scan-error-${width}`)
expect((await (await request.get('/api/scenarios/runs')).json()).items).toEqual(before.items)
expect(await targetSnapshot(request)).toEqual(initialTargets)
})

test(`launches suites with configured defaults and reports invalid children at ${width}px @seeded`, async ({ page, request }) => {
await page.setViewportSize({ width, height: 900 })
const initialTargets = await targetSnapshot(request)
const suiteName = `explicit_suite_${width}`
await page.goto('/scanner/launch')
await waitForAppReady(page)
await page.getByTestId(`launch-suite-${suiteName}`).click()
const dialog = page.getByRole('dialog', { name: `Launch ${suiteName}`, exact: true })
await expect(dialog.getByRole('combobox')).toHaveCount(2)
await expect(dialog.getByRole('button', { name: /Create/i })).toHaveCount(0)
await expect(dialog.getByRole('combobox', { name: 'Adversarial Target', exact: true })).toHaveValue('')
await expect(dialog.getByTestId('launch-suite-entries').getByRole('listitem')).toHaveCount(2)
await dialog.getByRole('combobox', { name: 'Target', exact: true }).selectOption('browser_offline')
await capture(page, `suite-configured-${width}`)
const started = page.waitForResponse(response =>
response.request().method() === 'POST' && new URL(response.url()).pathname.endsWith(`/${suiteName}/launch`))
await page.getByTestId('confirm-launch-suite').click()
const launchResponse = await started
expect(launchResponse.ok(), await launchResponse.text()).toBe(true)
const execution = await launchResponse.json()
await expect.poll(async () => (await (await request.get(
`/api/scenario-suites/executions/${execution.suite_execution_id}`,
)).json()).status).toBe('COMPLETED')

const mixedName = `explicit_mixed_${width}`
const before = await (await request.get('/api/scenarios/runs')).json()
await page.goto('/scanner/launch')
await waitForAppReady(page)
await page.getByTestId(`launch-suite-${mixedName}`).click()
const mixedDialog = page.getByRole('dialog', { name: `Launch ${mixedName}`, exact: true })
await mixedDialog.getByRole('combobox', { name: 'Target', exact: true }).selectOption('browser_offline')
await page.getByTestId('confirm-launch-suite').click()
await expect(mixedDialog.getByText(/Target must support input modality/)).toContainText('image')
await expect(page.getByTestId('confirm-launch-suite')).toBeEnabled()
await capture(page, `suite-error-${width}`)
expect((await (await request.get('/api/scenarios/runs')).json()).items).toEqual(before.items)
expect(await targetSnapshot(request)).toEqual(initialTargets)
})
}
9 changes: 4 additions & 5 deletions frontend/e2e/scenario-datasets.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -270,11 +270,10 @@ test('configures, saves, refreshes, estimates without a target, and launches the
await assertNoPreparation(request)

const launch = page.getByRole('button', { name: 'Launch scan', exact: true })
await expect(page.getByTestId('scenario-target-select')).toHaveValue('')
await expect(launch).toBeDisabled()
const issues = page.getByRole('alert', { name: 'Scan configuration issues' })
await expect(issues).toContainText('objective_target is required; no target selected.')
await expect(issues).toContainText('Create and select the target to scan.')
await expect(launch).toBeEnabled()
await launch.focus()
await launch.press('Enter')
await expect(page.getByRole('alert').filter({ hasText: 'Select a target.' })).toHaveText('Select a target.')
await expect(page.getByRole('dialog')).toHaveCount(0)
const rejected = await request.post('/api/scenarios/runs', {
headers: compatibilityHeaders(), data: { ...response.request().postDataJSON(), scenario_name: SCENARIO },
Expand Down
4 changes: 0 additions & 4 deletions frontend/e2e/scenario-history.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -313,10 +313,6 @@ async function mockScenarioAPIs(
});
});

await page.route(new RegExp(`/api/scenarios/catalog/${SCENARIO_NAME.replace(".", "\\.")}/readiness$`), async (route) => {
await route.fulfill({ json: { bindings: [], issues: [] } });
});

await page.route(new RegExp(`/api/scenarios/catalog/${SCENARIO_NAME.replace(".", "\\.")}/estimate$`), async (route) => {
const request = route.request().postDataJSON() as Record<string, unknown>;
estimateRequests.push(request);
Expand Down
1 change: 1 addition & 0 deletions frontend/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
"test:e2e:scenario-hierarchy": "npm run build && playwright test --config playwright.scenario-hierarchy.config.ts",
"test:e2e:scenario-datasets": "npm run build && playwright test --config playwright.scenario-datasets.config.ts",
"test:e2e:scenario-techniques": "npm run build && playwright test --config playwright.scenario-techniques.config.ts",
"test:e2e:scan-launch": "npm run build && playwright test --config playwright.scan-launch.config.ts",
"test:e2e:analytics": "npm run build && playwright test --config playwright.analytics.config.ts",
"test:e2e:paired-outcomes": "npm run build && playwright test --config playwright.paired-outcomes.config.ts",
"test:e2e:seeded": "playwright test --project seeded",
Expand Down
2 changes: 1 addition & 1 deletion frontend/playwright.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ const E2E_BACKEND_URL = `http://127.0.0.1:${E2E_BACKEND_PORT}`;

export default defineConfig({
testDir: "./e2e",
testIgnore: ["**/analytics.spec.ts", "**/paired-outcomes.spec.ts", "**/scenario-hierarchy.spec.ts", "**/scenario-datasets.spec.ts", "**/scenario-techniques.spec.ts"],
testIgnore: ["**/analytics.spec.ts", "**/paired-outcomes.spec.ts", "**/scenario-hierarchy.spec.ts", "**/scenario-datasets.spec.ts", "**/scenario-techniques.spec.ts", "**/scan-launch.seeded.spec.ts"],
fullyParallel: true,
forbidOnly: !!process.env.CI,
retries: process.env.CI ? 2 : 0,
Expand Down
70 changes: 70 additions & 0 deletions frontend/playwright.scan-launch.config.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
import path from 'node:path'
import process from 'node:process'

import { defineConfig, devices } from '@playwright/test'

import type { UserPreferences } from './src/types'

import base from './playwright.config'

const frontendPort = Number(process.env.E2E_FRONTEND_PORT ?? '18723')
const backendPort = Number(process.env.PYRIT_E2E_BACKEND_PORT ?? '18724')
for (const port of [frontendPort, backendPort]) {
if (!Number.isInteger(port) || port < 1 || port > 65535) {
throw new Error('Scan-launch frontend and backend ports must be valid TCP ports.')
}
}
const frontendUrl = `http://127.0.0.1:${frontendPort}`
const backendUrl = `http://127.0.0.1:${backendPort}`

// Missing-role cases need a fresh registry, not the shared seeded shard's named defaults.
export default defineConfig({
...base,
testMatch: 'scan-launch.seeded.spec.ts',
testIgnore: [],
fullyParallel: false,
workers: 1,
use: {
...base.use,
baseURL: frontendUrl,
storageState: {
cookies: [],
origins: [{
origin: frontendUrl,
localStorage: [
{ name: 'pyrit-tour-completed', value: 'true' },
{
name: 'pyrit.userPreferences.v1.local',
value: JSON.stringify({
targets: { adversarial: null },
hiddenTargetRegistryNames: [],
labels: {},
theme: 'system',
chatMarkdown: false,
operation: null,
} satisfies UserPreferences),
},
],
}],
},
},
projects: [{ name: 'scan-launch', use: { ...devices['Desktop Chrome'] } }],
webServer: [
{
command: 'cd .. && uv run --no-sync python -m pyrit.backend.pyrit_backend '
+ `--auth-mode local --host 127.0.0.1 --port ${backendPort} --log-level warning `
+ `--config-file ${path.join('tests', 'end_to_end', 'browser_config.yaml')}`,
env: { PYRIT_DEV_MODE: 'true' },
url: `${backendUrl}/api/health`,
reuseExistingServer: false,
timeout: 120_000,
},
{
command: `npx vite preview --host 127.0.0.1 --port ${frontendPort} --strictPort`,
env: { PYRIT_BACKEND_URL: backendUrl },
url: frontendUrl,
reuseExistingServer: false,
timeout: 120_000,
},
],
})
1 change: 0 additions & 1 deletion frontend/src/App.labels.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,6 @@ jest.mock('@/services/api', () => ({
},
targetsApi: { listTargets: jest.fn(), getTarget: jest.fn() },
scenariosApi: {
readiness: jest.fn().mockResolvedValue({ bindings: [], issues: [] }),
listCatalog: jest.fn(),
listRuns: jest.fn(),
getScenario: jest.fn(),
Expand Down
4 changes: 1 addition & 3 deletions frontend/src/components/Chat/ChatScorerPicker.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ interface ChatScorerPickerProps {
error?: string | null
inherited?: boolean
usage?: 'chat' | 'scenario'
defaultLabel?: string
}

function scorerOptionLabel(scorer: ScorerInstance): string {
Expand All @@ -26,7 +25,6 @@ function scorerOptionLabel(scorer: ScorerInstance): string {

export default function ChatScorerPicker({
value, scorers, onChange, onUseLastUsed, label = 'Chat scorer', disabled, loading, error, inherited = false, usage = 'chat',
defaultLabel,
}: ChatScorerPickerProps) {
const styles = useChatScorerPickerStyles()
const selectedScorer = findRegisteredScorer(value ?? null, scorers, inherited)
Expand All @@ -49,7 +47,7 @@ export default function ChatScorerPicker({
if (scorer) onChange({ registry_name: scorer.scorer_registry_name, identifier_hash: scorer.identifier.hash })
}}>
{onUseLastUsed && <option value="last-used">Use last used</option>}
<option value="">{usage === 'scenario' ? defaultLabel ?? 'Use scenario default' : 'Off'}</option>
<option value="">{usage === 'scenario' ? 'Use scenario default' : 'Off'}</option>
{selected === 'unavailable' && value && <option value="unavailable" hidden disabled>
{selectedScorer ? scorerOptionLabel(selectedScorer) : `Unavailable: ${value.registry_name}`}
</option>}
Expand Down
Loading
Loading