diff --git a/frontend/src/App.test.tsx b/frontend/src/App.test.tsx index edaf5aa9b9..14435d7898 100644 --- a/frontend/src/App.test.tsx +++ b/frontend/src/App.test.tsx @@ -442,6 +442,32 @@ jest.mock("./components/Scenarios/ScenarioCatalog", () => { }; }); +jest.mock("./components/ScenarioPresets/ScenarioPresetLibrary", () => { + const MockScenarioPresetLibrary = () =>
; + MockScenarioPresetLibrary.displayName = "MockScenarioPresetLibrary"; + return { + __esModule: true, + default: MockScenarioPresetLibrary, + }; +}); + +jest.mock("./components/ScenarioPresets/ScenarioPresetEditor", () => { + const { useParams } = jest.requireActual("react-router"); + const MockScenarioPresetEditor = ({ mode }: { mode: string }) => { + const { presetName } = useParams(); + return ( +
+ {presetName ?? ""} +
+ ); + }; + MockScenarioPresetEditor.displayName = "MockScenarioPresetEditor"; + return { + __esModule: true, + default: MockScenarioPresetEditor, + }; +}); + jest.mock("./components/Scenarios/ScenarioDetail", () => { const MockScenarioDetail = ({ defaultObjectiveTarget, @@ -671,6 +697,46 @@ describe("App", () => { expect(screen.getByLabelText("Current URL")).toHaveTextContent(/^\/chat$/); }); + it("renders the preset library as a registry section", () => { + renderApp("/registry/scenario-presets"); + + expect(screen.getByTestId("main-layout")).toHaveAttribute( + "data-current-view", + "registry" + ); + expect(screen.getByTestId("scenario-preset-library")).toBeInTheDocument(); + }); + + it("redirects a bookmarked /scanner/presets to the registry section", async () => { + render( + + + + + ); + + expect(await screen.findByTestId("scenario-preset-library")).toBeInTheDocument(); + expect(screen.getByLabelText("Current URL")).toHaveTextContent( + /^\/registry\/scenario-presets$/ + ); + }); + + it("redirects a bookmarked preset editor URL, keeping the preset it named", async () => { + render( + + + + + ); + + const editor = await screen.findByTestId("scenario-preset-editor"); + expect(editor).toHaveAttribute("data-mode", "edit"); + expect(editor).toHaveTextContent("nightly_probe"); + expect(screen.getByLabelText("Current URL")).toHaveTextContent( + /^\/registry\/scenario-presets\/nightly_probe\/edit$/ + ); + }); + it("renders the converter registry from its direct URL", async () => { renderApp("/registry/converters"); diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 2590741805..b885134e03 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -25,6 +25,9 @@ import ScenarioHistory from './components/History/ScenarioHistory' import ScenarioCatalog from './components/Scenarios/ScenarioCatalog' import ScenarioDetail from './components/Scenarios/ScenarioDetail' import ScenarioRunPage from './components/Scenarios/ScenarioRunPage' +import ScenarioPresetEditor from './components/ScenarioPresets/ScenarioPresetEditor' +import ScenarioPresetLibrary from './components/ScenarioPresets/ScenarioPresetLibrary' +import { presetEditorRoutePath } from './components/ScenarioPresets/presetRoutes' import DatasetCatalog from './components/Datasets/DatasetCatalog' import DatasetSelection from './components/Datasets/DatasetSelection' import FeedbackDialog from './components/Feedback/FeedbackDialog' @@ -127,6 +130,11 @@ function LegacyScenarioRunRedirect() { return } +function LegacyPresetEditorRedirect() { + const { presetName } = useParams<{ presetName: string }>() + return +} + function LegacyScenarioHistoryRedirect() { const location = useLocation() return @@ -815,11 +823,27 @@ function AppContent({ operatorAlias }: { operatorAlias: string | null }) { /> } /> } /> + + } + /> + } /> + } /> } /> } /> } /> } /> + } /> + } /> + } /> Target registry content
} /> Converter registry content} /> Technique registry content} /> + Preset library content} /> + Preset editor content} /> @@ -28,11 +30,12 @@ function renderLayout(initialPath = '/registry/targets') { } describe('RegistryLayout', () => { - it('shows target and converter registry tabs', () => { + it('shows target, converter and preset registry tabs', () => { renderLayout() expect(screen.getByRole('tab', { name: 'Targets' })).toHaveAttribute('aria-selected', 'true') expect(screen.getByRole('tab', { name: 'Converters' })).toHaveAttribute('aria-selected', 'false') + expect(screen.getByRole('tab', { name: 'Scenario presets' })).toHaveAttribute('aria-selected', 'false') expect(screen.getByText('Target registry content')).toBeInTheDocument() }) @@ -46,6 +49,23 @@ describe('RegistryLayout', () => { expect(screen.getByText('Converter registry content')).toBeInTheDocument() }) + it('opens the preset library from its tab', async () => { + const user = userEvent.setup() + renderLayout() + + await user.click(screen.getByRole('tab', { name: 'Scenario presets' })) + + expect(screen.getByRole('tab', { name: 'Scenario presets' })).toHaveAttribute('aria-selected', 'true') + expect(screen.getByText('Preset library content')).toBeInTheDocument() + }) + + it('keeps the preset tab selected on a nested preset route', () => { + renderLayout('/registry/scenario-presets/new') + + expect(screen.getByRole('tab', { name: 'Scenario presets' })).toHaveAttribute('aria-selected', 'true') + expect(screen.getByText('Preset editor content')).toBeInTheDocument() + }) + it('supports the Techniques direct URL and browser history', async () => { const user = userEvent.setup() renderLayout('/registry/techniques') diff --git a/frontend/src/components/Registry/RegistryLayout.tsx b/frontend/src/components/Registry/RegistryLayout.tsx index 18d5d30500..cef3e1f187 100644 --- a/frontend/src/components/Registry/RegistryLayout.tsx +++ b/frontend/src/components/Registry/RegistryLayout.tsx @@ -3,12 +3,25 @@ import { Outlet, useLocation, useNavigate } from 'react-router' import { useRegistryLayoutStyles } from './Registry.styles' +const REGISTRY_TABS = ['targets', 'converters', 'techniques', 'scenario-presets'] as const + +/** + * Keeps the tab selected while a section owns nested routes, which the preset + * editor does — `/registry/scenario-presets/new` is still the presets tab. + */ +function selectedTabFromPath(pathname: string): string { + return ( + REGISTRY_TABS.find( + (tab) => pathname === `/registry/${tab}` || pathname.startsWith(`/registry/${tab}/`), + ) ?? 'targets' + ) +} + export default function RegistryLayout() { const styles = useRegistryLayoutStyles() const location = useLocation() const navigate = useNavigate() - const selectedTab = location.pathname.endsWith('/techniques') ? 'techniques' - : location.pathname.endsWith('/converters') ? 'converters' : 'targets' + const selectedTab = selectedTabFromPath(location.pathname) return (
@@ -22,6 +35,7 @@ export default function RegistryLayout() { Targets Converters Techniques + Scenario presets
diff --git a/frontend/src/components/ScenarioPresets/LaunchPresetDialog.styles.ts b/frontend/src/components/ScenarioPresets/LaunchPresetDialog.styles.ts new file mode 100644 index 0000000000..7f11684d49 --- /dev/null +++ b/frontend/src/components/ScenarioPresets/LaunchPresetDialog.styles.ts @@ -0,0 +1,20 @@ +import { makeStyles, tokens } from '@fluentui/react-components' + +export const useLaunchPresetDialogStyles = makeStyles({ + body: { + display: 'flex', + flexDirection: 'column', + gap: tokens.spacingVerticalM, + }, + summary: { + display: 'flex', + flexDirection: 'column', + gap: tokens.spacingVerticalXXS, + }, + scenarioLine: { + color: tokens.colorNeutralForeground3, + }, + numberInput: { + maxWidth: '10rem', + }, +}) diff --git a/frontend/src/components/ScenarioPresets/LaunchPresetDialog.test.tsx b/frontend/src/components/ScenarioPresets/LaunchPresetDialog.test.tsx new file mode 100644 index 0000000000..9b2f63e1c0 --- /dev/null +++ b/frontend/src/components/ScenarioPresets/LaunchPresetDialog.test.tsx @@ -0,0 +1,352 @@ +import { render, screen, waitFor } from '@testing-library/react' +import userEvent from '@testing-library/user-event' +import { FluentProvider, webLightTheme } from '@fluentui/react-components' +import { MemoryRouter } from 'react-router' + +import { scenarioPresetsApi, scenariosApi } from '@/services/api' +import { makeTarget } from '@/test-utils/targetFixtures' +import type { ScenarioPreset, TargetInstance } from '@/types' + +import LaunchPresetDialog from './LaunchPresetDialog' + +jest.mock('@/services/api', () => ({ + scenarioPresetsApi: { + resolve: jest.fn(), + }, + scenariosApi: { + startRun: jest.fn(), + }, +})) + +const mockResolve = scenarioPresetsApi.resolve as jest.Mock +const mockStartRun = scenariosApi.startRun as jest.Mock + +const mockNavigate = jest.fn() + +jest.mock('react-router', () => ({ + ...jest.requireActual('react-router'), + useNavigate: () => mockNavigate, +})) + +const PRESET: ScenarioPreset = { + name: 'nightly_probe', + scenario_name: 'foundry.red_team_agent', + techniques: ['crescendo'], +} + +const OBJECTIVE_TARGET = makeTarget({ target_registry_name: 'gpt4o' }) +const ADVERSARIAL_TARGET: TargetInstance = makeTarget({ + target_registry_name: 'adversary', + capabilities: { + supports_multi_turn: true, + supports_json_schema: false, + supports_json_output: false, + supports_system_prompt: true, + supported_input_modalities: ['text'], + supported_output_modalities: ['text'], + }, +}) + +interface RenderOptions { + preset?: ScenarioPreset + version?: string + defaultObjectiveTarget?: TargetInstance | null + defaultAdversarialTarget?: TargetInstance | null + labels?: Record +} + +const onDismiss = jest.fn() +const onPresetChanged = jest.fn() + +function renderDialog({ + preset = PRESET, + version = 'v1', + defaultObjectiveTarget = OBJECTIVE_TARGET, + defaultAdversarialTarget = null, + labels = {}, +}: RenderOptions = {}) { + return render( + + + + + , + ) +} + +beforeEach(() => { + jest.clearAllMocks() + mockResolve.mockResolvedValue({ scenario_name: 'foundry.red_team_agent', target_name: 'gpt4o' }) + mockStartRun.mockResolvedValue({ scenario_result_id: 'run-1' }) +}) + +describe('LaunchPresetDialog', () => { + it('reports scenario defaults for the fields the preset leaves unset', () => { + renderDialog() + + expect(screen.getByTestId('launch-preset-summary')).toHaveTextContent( + '1 technique · scenario default datasets', + ) + }) + + it('summarizes the configuration the preset pins', () => { + renderDialog({ + preset: { + name: 'nightly_probe', + scenario_name: 'foundry.red_team_agent', + techniques: ['crescendo', 'flip'], + datasets: { + sources: [{ name: 'harmbench' }, { name: 'advbench' }], + max_total: 25, + }, + include_baseline: true, + }, + }) + + expect(screen.getByTestId('launch-preset-summary')).toHaveTextContent( + '2 techniques · harmbench, advbench · max 25 cases · baseline included', + ) + }) + + it('treats a limit of default as saying nothing about the limit', () => { + renderDialog({ + preset: { + ...PRESET, + datasets: { sources: [{ name: 'harmbench' }], max_total: 'default', max_per_dataset: 'default' }, + }, + }) + + const summary = screen.getByTestId('launch-preset-summary') + expect(summary).toHaveTextContent('harmbench') + expect(summary).not.toHaveTextContent('max') + }) + + it('reports an unbounded limit the flat schema could not express', () => { + renderDialog({ + preset: { ...PRESET, datasets: { sources: [{ name: 'harmbench' }], max_total: 'all' } }, + }) + + expect(screen.getByTestId('launch-preset-summary')).toHaveTextContent('every case') + }) + + it('distinguishes an excluded baseline from an included one', () => { + renderDialog({ preset: { ...PRESET, include_baseline: false } }) + + expect(screen.getByTestId('launch-preset-summary')).toHaveTextContent('baseline excluded') + }) + + it('resolves the preset server-side and navigates to the started run', async () => { + const user = userEvent.setup() + renderDialog({ labels: { op: 'nightly' } }) + + await user.click(screen.getByTestId('confirm-launch-preset')) + + await waitFor(() => expect(mockResolve).toHaveBeenCalledWith('nightly_probe', { + expected_version: 'v1', + target_name: 'gpt4o', + max_concurrency: 10, + max_retries: 0, + labels: { op: 'nightly' }, + })) + expect(mockStartRun).toHaveBeenCalledWith({ + scenario_name: 'foundry.red_team_agent', + target_name: 'gpt4o', + }) + expect(mockNavigate).toHaveBeenCalledWith('/scanner-history/run-1', { + state: { scenarioName: 'foundry.red_team_agent' }, + }) + }) + + it('omits the adversarial target and labels when neither is set', async () => { + const user = userEvent.setup() + renderDialog() + + await user.click(screen.getByTestId('confirm-launch-preset')) + + await waitFor(() => expect(mockResolve).toHaveBeenCalledWith('nightly_probe', { + expected_version: 'v1', + target_name: 'gpt4o', + max_concurrency: 10, + max_retries: 0, + })) + }) + + it('hands a preset edited mid-launch back to the library instead of running it', async () => { + const user = userEvent.setup() + mockResolve.mockRejectedValue({ + isAxiosError: true, + response: { status: 409, data: { detail: "Scenario preset 'nightly_probe' changed since it was read" } }, + }) + + renderDialog() + + await user.click(screen.getByTestId('confirm-launch-preset')) + + await waitFor(() => expect(onPresetChanged).toHaveBeenCalled()) + expect(mockStartRun).not.toHaveBeenCalled() + expect(mockNavigate).not.toHaveBeenCalled() + }) + + it('reports a run-start conflict without claiming the preset changed', async () => { + const user = userEvent.setup() + mockStartRun.mockRejectedValueOnce({ + isAxiosError: true, + response: { status: 409, data: { detail: 'A scenario run is already using this identifier.' } }, + }) + + renderDialog() + + await user.click(screen.getByTestId('confirm-launch-preset')) + + expect(await screen.findByText('A scenario run is already using this identifier.')).toBeInTheDocument() + expect(onPresetChanged).not.toHaveBeenCalled() + expect(mockNavigate).not.toHaveBeenCalled() + expect(screen.getByTestId('confirm-launch-preset')).toBeEnabled() + + await user.click(screen.getByTestId('confirm-launch-preset')) + await waitFor(() => expect(mockNavigate).toHaveBeenCalled()) + expect(onPresetChanged).not.toHaveBeenCalled() + }) + + it('sends the default adversarial target when one is configured', async () => { + const user = userEvent.setup() + renderDialog({ defaultAdversarialTarget: ADVERSARIAL_TARGET }) + + await user.click(screen.getByTestId('confirm-launch-preset')) + + await waitFor(() => expect(mockResolve).toHaveBeenCalledWith( + 'nightly_probe', + expect.objectContaining({ adversarial_target_name: 'adversary' }), + )) + }) + + it('blocks launching until a target is chosen', () => { + renderDialog({ defaultObjectiveTarget: null }) + + expect(screen.getByTestId('confirm-launch-preset')).toBeDisabled() + }) + + it('ignores a default target that this deployment does not have', () => { + renderDialog({ defaultObjectiveTarget: makeTarget({ target_registry_name: 'retired' }) }) + + expect(screen.getByTestId('confirm-launch-preset')).toBeDisabled() + }) + + it('reports a resolve failure and does not start a run', async () => { + const user = userEvent.setup() + mockResolve.mockRejectedValue(new Error('Unknown scenario "foundry.retired".')) + + renderDialog() + + await user.click(screen.getByTestId('confirm-launch-preset')) + + expect(await screen.findByText('Unknown scenario "foundry.retired".')).toBeInTheDocument() + expect(mockStartRun).not.toHaveBeenCalled() + expect(mockNavigate).not.toHaveBeenCalled() + }) + + it('re-enables launch after a failure so the operator can retry', async () => { + const user = userEvent.setup() + mockStartRun.mockRejectedValueOnce(new Error('Target is unavailable.')) + + renderDialog() + + await user.click(screen.getByTestId('confirm-launch-preset')) + + expect(await screen.findByText('Target is unavailable.')).toBeInTheDocument() + expect(screen.getByTestId('confirm-launch-preset')).toBeEnabled() + + await user.click(screen.getByTestId('confirm-launch-preset')) + await waitFor(() => expect(mockNavigate).toHaveBeenCalled()) + }) + + it('dismisses without launching', async () => { + const user = userEvent.setup() + renderDialog() + + await user.click(screen.getByRole('button', { name: 'Cancel' })) + + expect(onDismiss).toHaveBeenCalled() + expect(mockResolve).not.toHaveBeenCalled() + }) + + it('dismisses when the dialog itself is closed', async () => { + const user = userEvent.setup() + renderDialog() + + await user.keyboard('{Escape}') + + expect(onDismiss).toHaveBeenCalled() + }) + + it('sends the target the operator picked instead of the default', async () => { + const user = userEvent.setup() + renderDialog() + + await user.selectOptions(screen.getByRole('combobox', { name: 'Target' }), 'adversary') + await user.click(screen.getByTestId('confirm-launch-preset')) + + await waitFor(() => expect(mockResolve).toHaveBeenCalledWith( + 'nightly_probe', + expect.objectContaining({ target_name: 'adversary' }), + )) + }) + + it('sends the adversarial target the operator picked', async () => { + const user = userEvent.setup() + renderDialog() + + await user.selectOptions(screen.getByRole('combobox', { name: 'Adversarial Target' }), 'adversary') + await user.click(screen.getByTestId('confirm-launch-preset')) + + await waitFor(() => expect(mockResolve).toHaveBeenCalledWith( + 'nightly_probe', + expect.objectContaining({ adversarial_target_name: 'adversary' }), + )) + }) + + it('sends the concurrency and retry limits the operator set', async () => { + const user = userEvent.setup() + renderDialog() + + const concurrency = screen.getByTestId('preset-max-concurrency-input') + await user.clear(concurrency) + await user.type(concurrency, '4') + const retries = screen.getByTestId('preset-max-retries-input') + await user.clear(retries) + await user.type(retries, '2') + + await user.click(screen.getByTestId('confirm-launch-preset')) + + await waitFor(() => expect(mockResolve).toHaveBeenCalledWith( + 'nightly_probe', + expect.objectContaining({ max_concurrency: 4, max_retries: 2 }), + )) + }) + + it('does not resolve twice while a launch is in flight', async () => { + const user = userEvent.setup() + let release: () => void = () => {} + mockResolve.mockReturnValue(new Promise((resolve) => { + release = () => resolve({ scenario_name: 'foundry.red_team_agent', target_name: 'gpt4o' }) + })) + + renderDialog() + + await user.click(screen.getByTestId('confirm-launch-preset')) + expect(screen.getByTestId('confirm-launch-preset')).toBeDisabled() + + release() + await waitFor(() => expect(mockNavigate).toHaveBeenCalled()) + expect(mockResolve).toHaveBeenCalledTimes(1) + }) +}) diff --git a/frontend/src/components/ScenarioPresets/LaunchPresetDialog.tsx b/frontend/src/components/ScenarioPresets/LaunchPresetDialog.tsx new file mode 100644 index 0000000000..4a2e23949d --- /dev/null +++ b/frontend/src/components/ScenarioPresets/LaunchPresetDialog.tsx @@ -0,0 +1,247 @@ +import { type FormEvent, useState } from 'react' + +import { + Button, + Dialog, + DialogActions, + DialogBody, + DialogContent, + DialogSurface, + DialogTitle, + Field, + MessageBar, + MessageBarBody, + Text, +} from '@fluentui/react-components' +import { useNavigate } from 'react-router' + +import TargetSelect from '@/components/Config/TargetSelect' +import SingleStepSpinButton from '@/components/Parameters/SingleStepSpinButton' +import { + DEFAULT_MAX_CONCURRENCY, + DEFAULT_MAX_RETRIES, + MAX_MAX_CONCURRENCY, + MAX_MAX_RETRIES, + MIN_MAX_CONCURRENCY, + MIN_MAX_RETRIES, + resolveSpinButtonValue, +} from '@/components/Scenarios/scenarioRunLimits' +import { scenarioPresetsApi, scenariosApi } from '@/services/api' +import { toApiError } from '@/services/errors' +import type { RunScenarioRequest, ScenarioPreset, TargetInstance } from '@/types' +import { scenarioRunRoutePath } from '@/utils/routeParams' + +import { useLaunchPresetDialogStyles } from './LaunchPresetDialog.styles' + +interface LaunchPresetDialogProps { + preset: ScenarioPreset + version: string | null + targets: TargetInstance[] + defaultObjectiveTarget: TargetInstance | null + defaultAdversarialTarget: TargetInstance | null + labels: Record + onDismiss: () => void + onPresetChanged: () => void +} + +function initialTargetName( + candidate: TargetInstance | null, + available: TargetInstance[], +): string { + if (!candidate) { + return '' + } + const match = available.some( + (target) => target.target_registry_name === candidate.target_registry_name, + ) + return match ? candidate.target_registry_name : '' +} + +/** + * Restates what the preset pins. The dialog covers the library card, so without + * this the operator commits to a run knowing only the preset and scenario name. + * Omitted fields read as scenario defaults because that is what the server does + * with them, and a limit of 'default' is an explicit way of saying the same thing. + */ +function presetSummary(preset: ScenarioPreset): string { + const techniques = preset.techniques ?? [] + const sources = preset.datasets?.sources ?? [] + const parts = [ + techniques.length === 0 + ? 'Scenario default techniques' + : `${techniques.length} technique${techniques.length === 1 ? '' : 's'}`, + sources.length === 0 ? 'scenario default datasets' : sources.map((source) => source.name).join(', '), + ] + const total = preset.datasets?.max_total + if (total != null && total !== 'default') { + parts.push(total === 'all' ? 'every case' : `max ${total} cases`) + } + const perDataset = preset.datasets?.max_per_dataset + if (perDataset != null && perDataset !== 'default') { + parts.push(perDataset === 'all' ? 'every case per dataset' : `max ${perDataset} per dataset`) + } + if (preset.include_baseline != null) { + parts.push(preset.include_baseline ? 'baseline included' : 'baseline excluded') + } + return parts.join(' · ') +} + +/** + * Collects the launch-owned fields a preset deliberately omits, then asks the + * server to merge them with the stored preset. Resolution stays server-side so + * the browser never reimplements the preset-to-run mapping. + * + * The summary above describes the preset as it was read, so the launch sends that + * version back. An edit landing in between is reported rather than launched, which + * is the one case where a run would otherwise differ from what was confirmed. + */ +export default function LaunchPresetDialog({ + preset, + version, + targets, + defaultObjectiveTarget, + defaultAdversarialTarget, + labels, + onDismiss, + onPresetChanged, +}: LaunchPresetDialogProps) { + const styles = useLaunchPresetDialogStyles() + const navigate = useNavigate() + const adversarialTargets = targets.filter( + (target) => target.capabilities?.supports_multi_turn === true, + ) + const [targetName, setTargetName] = useState( + () => initialTargetName(defaultObjectiveTarget, targets), + ) + const [adversarialTargetName, setAdversarialTargetName] = useState( + () => initialTargetName(defaultAdversarialTarget, adversarialTargets), + ) + const [maxConcurrency, setMaxConcurrency] = useState(DEFAULT_MAX_CONCURRENCY) + const [maxRetries, setMaxRetries] = useState(DEFAULT_MAX_RETRIES) + const [submitting, setSubmitting] = useState(false) + const [error, setError] = useState(null) + + const handleSubmit = async (event: FormEvent): Promise => { + event.preventDefault() + if (targetName === '' || submitting) { + return + } + setSubmitting(true) + setError(null) + let request: RunScenarioRequest + try { + request = await scenarioPresetsApi.resolve(preset.name, { + expected_version: version, + target_name: targetName, + ...(adversarialTargetName === '' ? {} : { adversarial_target_name: adversarialTargetName }), + max_concurrency: maxConcurrency, + max_retries: maxRetries, + ...(Object.keys(labels).length > 0 ? { labels } : {}), + }) + } catch (err) { + const apiError = toApiError(err) + if (apiError.status === 409) { + // The summary on screen no longer describes the preset, so re-reading is the + // only way forward; keeping the dialog open would invite a confirm-and-retry loop. + onPresetChanged() + return + } + setError(apiError.detail) + setSubmitting(false) + return + } + try { + const summary = await scenariosApi.startRun(request) + navigate(scenarioRunRoutePath(summary.scenario_result_id), { + state: { scenarioName: preset.scenario_name }, + }) + } catch (err) { + setError(toApiError(err).detail) + setSubmitting(false) + } + } + + return ( + { if (!data.open) onDismiss() }}> + +
+ + Launch {preset.name} + +
+ + Runs {preset.scenario_name} with this preset's saved configuration. + + + {presetSummary(preset)} + +
+ {error && ( + + {error} + + )} + setTargetName(target?.target_registry_name ?? '')} + label="Target" + hint="The registered target this run attacks." + placeholder="Select a target" + disabled={submitting} + /> + setAdversarialTargetName(target?.target_registry_name ?? '')} + label="Adversarial Target" + hint="Only used by scenarios that generate attacks with a second target." + placeholder="Use server default" + disabled={submitting} + /> + + setMaxConcurrency(resolveSpinButtonValue(data, maxConcurrency))} + data-testid="preset-max-concurrency-input" + /> + + + setMaxRetries(resolveSpinButtonValue(data, maxRetries))} + data-testid="preset-max-retries-input" + /> + + + + + + + +
+ + + ) +} diff --git a/frontend/src/components/ScenarioPresets/ScenarioPresetEditor.styles.ts b/frontend/src/components/ScenarioPresets/ScenarioPresetEditor.styles.ts new file mode 100644 index 0000000000..d48adda1f6 --- /dev/null +++ b/frontend/src/components/ScenarioPresets/ScenarioPresetEditor.styles.ts @@ -0,0 +1,70 @@ +import { makeStyles, tokens } from '@fluentui/react-components' + +import { mobileTouchTarget, NARROW_VIEWPORT_QUERY } from '@/styles/touchTargets' +import { WORKSPACE_CANVAS_BACKGROUND } from '@/styles/workspaceBackground' + +export const useScenarioPresetEditorStyles = makeStyles({ + root: { + display: 'flex', + flexDirection: 'column', + height: '100%', + width: '100%', + minWidth: 0, + padding: tokens.spacingVerticalXXL, + overflowX: 'hidden', + overflowY: 'auto', + backgroundColor: WORKSPACE_CANVAS_BACKGROUND, + [NARROW_VIEWPORT_QUERY]: { + padding: `${tokens.spacingVerticalL} ${tokens.spacingHorizontalM}`, + }, + }, + headerText: { + display: 'flex', + flexDirection: 'column', + gap: tokens.spacingVerticalXS, + marginBottom: tokens.spacingVerticalL, + }, + subtitle: { + color: tokens.colorNeutralForeground3, + }, + form: { + display: 'flex', + flexDirection: 'column', + gap: tokens.spacingVerticalL, + maxWidth: '60rem', + }, + section: { + display: 'flex', + flexDirection: 'column', + gap: tokens.spacingVerticalM, + padding: tokens.spacingVerticalL, + borderRadius: tokens.borderRadiusMedium, + border: `${tokens.strokeWidthThin} solid ${tokens.colorNeutralStroke2}`, + backgroundColor: tokens.colorNeutralBackground1, + }, + control: { + maxWidth: '32rem', + }, + dynamicParameters: { + display: 'flex', + flexDirection: 'column', + gap: tokens.spacingVerticalM, + }, + actions: { + display: 'flex', + flexWrap: 'wrap', + gap: tokens.spacingHorizontalS, + alignItems: 'center', + }, + touchTarget: { + ...mobileTouchTarget, + }, + centeredState: { + display: 'flex', + flexDirection: 'column', + alignItems: 'center', + gap: tokens.spacingVerticalM, + padding: tokens.spacingVerticalXXL, + textAlign: 'center', + }, +}) diff --git a/frontend/src/components/ScenarioPresets/ScenarioPresetEditor.test.tsx b/frontend/src/components/ScenarioPresets/ScenarioPresetEditor.test.tsx new file mode 100644 index 0000000000..80397b7a6e --- /dev/null +++ b/frontend/src/components/ScenarioPresets/ScenarioPresetEditor.test.tsx @@ -0,0 +1,735 @@ +import { act, render, screen, waitFor, within } from '@testing-library/react' +import userEvent from '@testing-library/user-event' +import { FluentProvider, webLightTheme } from '@fluentui/react-components' +import { MemoryRouter, Route, Routes } from 'react-router' + +import { scenarioPresetsApi, scenariosApi } from '@/services/api' +import { makeScenario } from '@/test-utils/scenarioFixtures' +import type { ScenarioDatasetSettings, ScenarioPreset, ScenarioRunSizeEstimateResponse } from '@/types' + +import ScenarioPresetEditor from './ScenarioPresetEditor' + +jest.mock('@/services/api', () => ({ + datasetsApi: { + listDatasets: jest.fn().mockResolvedValue({ + items: ['harmbench', 'xstest'].map((name) => ({ + name, selection_key: `dataset:named:${name}`, loaded: true, provider_available: true, + logical_examples: 8, seed_pieces: null, objectives: null, modalities: ['text'], + harm_categories: ['violence'], has_unlabeled_harm_categories: false, + })), + }), + }, + scenarioPresetsApi: { + get: jest.fn(), + create: jest.fn(), + update: jest.fn(), + }, + scenariosApi: { + listCatalog: jest.fn(), + getScenario: jest.fn(), + estimateRun: jest.fn(), + }, +})) + +const mockGet = scenarioPresetsApi.get as jest.Mock +const mockCreate = scenarioPresetsApi.create as jest.Mock +const mockUpdate = scenarioPresetsApi.update as jest.Mock +const mockListCatalog = scenariosApi.listCatalog as jest.Mock +const mockGetScenario = scenariosApi.getScenario as jest.Mock +const mockEstimateRun = scenariosApi.estimateRun as jest.Mock + +const mockNavigate = jest.fn() + +jest.mock('react-router', () => ({ + ...jest.requireActual('react-router'), + useNavigate: () => mockNavigate, +})) + +const SCENARIO = makeScenario() + +const STORED_PRESET: ScenarioPreset = { + name: 'nightly_probe', + scenario_name: 'foundry.red_team_agent', + description: 'Nightly smoke test.', + techniques: ['crescendo'], + include_baseline: false, +} + +function apiError(status: number, detail: string): unknown { + return { + isAxiosError: true, + response: { status, data: { detail } }, + } +} + +/** Drives the shared dataset table's limit popover the same way its own suite does. */ +async function editLimit(user: ReturnType, label: string, value: string): Promise { + await user.click(await screen.findByRole('button', { name: `Edit ${label}` })) + const all = screen.getByRole('checkbox', { name: `All available for ${label}` }) + if (all instanceof HTMLInputElement && all.checked) await user.click(all) + const input = screen.getByRole('textbox', { name: label }) + await user.clear(input) + await user.type(input, value) +} + +function renderCreate() { + return render( + + + + } /> + + + , + ) +} + +function renderEdit(name = 'nightly_probe') { + return render( + + + + } + /> + + + , + ) +} + +beforeEach(() => { + jest.clearAllMocks() + mockListCatalog.mockResolvedValue({ + items: [SCENARIO], + pagination: { has_more: false, next_cursor: null }, + }) + mockGetScenario.mockResolvedValue(SCENARIO) + mockEstimateRun.mockResolvedValue(SCENARIO.default_run_size) + mockGet.mockResolvedValue({ preset: STORED_PRESET, version: 'v1', issues: [] }) + mockCreate.mockResolvedValue({ preset: STORED_PRESET, version: 'v1', issues: [] }) + mockUpdate.mockResolvedValue({ preset: STORED_PRESET, version: 'v2', issues: [] }) +}) + +describe('ScenarioPresetEditor create mode', () => { + it('cannot save before a scenario is chosen', async () => { + renderCreate() + + expect(await screen.findByTestId('scenario-preset-editor')).toBeInTheDocument() + expect(screen.getByTestId('save-preset-btn')).toBeDisabled() + }) + + it('creates a preset that pins nothing the operator left at the scenario default', async () => { + const user = userEvent.setup() + renderCreate() + + await user.type(await screen.findByTestId('preset-name-input'), 'nightly_probe') + await user.click(screen.getByTestId('preset-scenario-select')) + await user.click(await screen.findByRole('option', { name: 'foundry.red_team_agent' })) + + await waitFor(() => expect(screen.getByTestId('save-preset-btn')).toBeEnabled()) + await user.click(screen.getByTestId('save-preset-btn')) + + await waitFor(() => expect(mockCreate).toHaveBeenCalledWith({ + name: 'nightly_probe', + scenario_name: 'foundry.red_team_agent', + })) + expect(mockUpdate).not.toHaveBeenCalled() + expect(mockNavigate).toHaveBeenCalledWith('/registry/scenario-presets') + }) + + it('pins a technique selection the operator moved off the scenario default', async () => { + const user = userEvent.setup() + renderCreate() + + await user.type(await screen.findByTestId('preset-name-input'), 'nightly_probe') + await user.click(screen.getByTestId('preset-scenario-select')) + await user.click(await screen.findByRole('option', { name: 'foundry.red_team_agent' })) + + await user.click(await screen.findByTestId('technique-crescendo')) + await waitFor(() => expect(screen.getByTestId('save-preset-btn')).toBeEnabled()) + await user.click(screen.getByTestId('save-preset-btn')) + + await waitFor(() => expect(mockCreate).toHaveBeenCalledWith({ + name: 'nightly_probe', + scenario_name: 'foundry.red_team_agent', + techniques: ['default_technique', 'crescendo'], + })) + }) + + it('rejects a name the server pattern would reject, without calling the API', async () => { + const user = userEvent.setup() + renderCreate() + + await user.type(await screen.findByTestId('preset-name-input'), 'Nightly-Probe') + await user.click(screen.getByTestId('preset-scenario-select')) + await user.click(await screen.findByRole('option', { name: 'foundry.red_team_agent' })) + + await waitFor(() => expect(screen.getByTestId('save-preset-btn')).toBeEnabled()) + await user.click(screen.getByTestId('save-preset-btn')) + + expect(await screen.findByText(/Use lowercase letters/)).toBeInTheDocument() + expect(mockCreate).not.toHaveBeenCalled() + }) + + it('reports a duplicate name returned by the server', async () => { + const user = userEvent.setup() + mockCreate.mockRejectedValue(apiError(409, 'A preset named "nightly_probe" already exists.')) + renderCreate() + + await user.type(await screen.findByTestId('preset-name-input'), 'nightly_probe') + await user.click(screen.getByTestId('preset-scenario-select')) + await user.click(await screen.findByRole('option', { name: 'foundry.red_team_agent' })) + + await waitFor(() => expect(screen.getByTestId('save-preset-btn')).toBeEnabled()) + await user.click(screen.getByTestId('save-preset-btn')) + + expect( + await screen.findByText('A preset named "nightly_probe" already exists.'), + ).toBeInTheDocument() + expect(mockNavigate).not.toHaveBeenCalled() + }) + + it('reports a missing admin permission', async () => { + const user = userEvent.setup() + mockCreate.mockRejectedValue(apiError(403, 'Admin access required.')) + renderCreate() + + await user.type(await screen.findByTestId('preset-name-input'), 'nightly_probe') + await user.click(screen.getByTestId('preset-scenario-select')) + await user.click(await screen.findByRole('option', { name: 'foundry.red_team_agent' })) + + await waitFor(() => expect(screen.getByTestId('save-preset-btn')).toBeEnabled()) + await user.click(screen.getByTestId('save-preset-btn')) + + expect(await screen.findByText('Admin access required.')).toBeInTheDocument() + }) + + it('reports an unselectable technique set rather than saving an empty one', async () => { + const user = userEvent.setup() + mockGetScenario.mockResolvedValue(makeScenario({ + default_techniques: [], + default_technique: 'all', + aggregate_techniques: ['all'], + })) + renderCreate() + + await user.type(await screen.findByTestId('preset-name-input'), 'nightly_probe') + await user.click(screen.getByTestId('preset-scenario-select')) + await user.click(await screen.findByRole('option', { name: 'foundry.red_team_agent' })) + + await waitFor(() => expect(screen.getByTestId('save-preset-btn')).toBeEnabled()) + await user.click(screen.getByTestId('save-preset-btn')) + + expect(await screen.findByText('Select at least one technique.')).toBeInTheDocument() + expect(mockCreate).not.toHaveBeenCalled() + }) + + it('reports a failure to load the scenario the operator picked', async () => { + const user = userEvent.setup() + mockGetScenario.mockRejectedValue(apiError(503, 'Scenario registry is unavailable.')) + renderCreate() + + await user.click(await screen.findByTestId('preset-scenario-select')) + await user.click(await screen.findByRole('option', { name: 'foundry.red_team_agent' })) + + expect(await screen.findByText('Scenario registry is unavailable.')).toBeInTheDocument() + expect(screen.getByTestId('save-preset-btn')).toBeDisabled() + }) +}) + +describe('ScenarioPresetEditor edit mode', () => { + it('loads the stored preset and pins its name and scenario', async () => { + renderEdit() + + expect(await screen.findByTestId('scenario-preset-editor')).toBeInTheDocument() + expect(screen.getByTestId('preset-name-input')).toBeDisabled() + expect(screen.getByTestId('preset-name-input')).toHaveValue('nightly_probe') + expect(screen.getByTestId('preset-description-input')).toHaveValue('Nightly smoke test.') + }) + + it('updates with the version it read so a concurrent edit is not overwritten', async () => { + const user = userEvent.setup() + renderEdit() + + await user.click(await screen.findByTestId('save-preset-btn')) + + await waitFor(() => expect(mockUpdate).toHaveBeenCalledWith( + 'nightly_probe', + expect.objectContaining({ name: 'nightly_probe', techniques: ['crescendo'] }), + 'v1', + )) + expect(mockCreate).not.toHaveBeenCalled() + expect(mockNavigate).toHaveBeenCalledWith('/registry/scenario-presets') + }) + + it('reports a version conflict instead of navigating away', async () => { + const user = userEvent.setup() + mockUpdate.mockRejectedValue(apiError(409, 'The preset changed since it was loaded.')) + renderEdit() + + await user.click(await screen.findByTestId('save-preset-btn')) + + expect(await screen.findByText('The preset changed since it was loaded.')).toBeInTheDocument() + expect(mockNavigate).not.toHaveBeenCalled() + }) + + it('keeps pinned techniques this deployment does not offer instead of rewriting them', async () => { + const user = userEvent.setup() + mockGet.mockResolvedValue({ + preset: { ...STORED_PRESET, techniques: ['crescendo', 'retired_attack'] }, + version: 'v1', + issues: [{ field: 'techniques', message: "Technique 'retired_attack' is not available." }], + }) + + renderEdit() + + expect(await screen.findByTestId('dropped-techniques-warning')).toHaveTextContent( + 'retired_attack', + ) + await user.click(screen.getByTestId('save-preset-btn')) + + await waitFor(() => expect(mockUpdate).toHaveBeenCalledWith( + 'nightly_probe', + expect.objectContaining({ techniques: ['crescendo', 'retired_attack'] }), + 'v1', + )) + }) + + it('keeps a converter-qualified technique through an unrelated edit', async () => { + const user = userEvent.setup() + mockGet.mockResolvedValue({ + preset: { ...STORED_PRESET, techniques: ['crescendo:converter.translation_spanish'] }, + version: 'v1', + issues: [], + }) + + renderEdit() + + await user.type(await screen.findByTestId('preset-description-input'), ' Updated.') + expect(screen.queryByTestId('dropped-techniques-warning')).not.toBeInTheDocument() + await user.click(screen.getByTestId('save-preset-btn')) + + await waitFor(() => expect(mockUpdate).toHaveBeenCalledWith( + 'nightly_probe', + expect.objectContaining({ techniques: ['crescendo:converter.translation_spanish'] }), + 'v1', + )) + }) + + it('shows modifier validation from the server even when the base technique is available', async () => { + mockGet.mockResolvedValue({ + preset: { ...STORED_PRESET, techniques: ['crescendo:converter.missing'] }, + version: 'v1', + issues: [{ field: 'techniques', message: "Converter 'missing' is not registered." }], + }) + + renderEdit() + + expect(await screen.findByTestId('dropped-techniques-warning')).toHaveTextContent( + "Converter 'missing' is not registered.", + ) + expect(screen.getByTestId('techniques-without-checkbox')).toHaveTextContent('crescendo:converter.missing') + }) + + it('does not invent a technique issue for a token the server accepts', async () => { + mockGet.mockResolvedValue({ + preset: { ...STORED_PRESET, techniques: ['server_extension'] }, + version: 'v1', + issues: [{ field: 'scenario_params', message: 'A parameter is unavailable.' }], + }) + + renderEdit() + + expect(await screen.findByTestId('techniques-without-checkbox')).toHaveTextContent('server_extension') + expect(screen.queryByTestId('dropped-techniques-warning')).not.toBeInTheDocument() + }) + + it('shows a pinned aggregate technique as a tag the operator can remove', async () => { + const user = userEvent.setup() + mockGet.mockResolvedValue({ + preset: { ...STORED_PRESET, techniques: ['all'] }, + version: 'v1', + issues: [], + }) + + renderEdit() + + expect(await screen.findByTestId('techniques-without-checkbox')).toHaveTextContent('all') + expect(screen.getByTestId('technique-crescendo')).not.toBeChecked() + expect(screen.queryByTestId('dropped-techniques-warning')).not.toBeInTheDocument() + + await user.click(screen.getByTestId('technique-crescendo')) + await user.click(screen.getByRole('button', { name: 'Remove all' })) + await user.click(screen.getByTestId('save-preset-btn')) + + await waitFor(() => expect(mockUpdate).toHaveBeenCalledWith( + 'nightly_probe', + expect.objectContaining({ techniques: ['crescendo'] }), + 'v1', + )) + }) + + it('reports a missing preset as not found', async () => { + mockGet.mockRejectedValue(apiError(404, 'No such preset.')) + + renderEdit() + + expect(await screen.findByTestId('editor-error-state')).toHaveTextContent( + 'No preset named "nightly_probe"', + ) + }) + + it('reports an unavailable scenario as unavailable, not as a missing preset', async () => { + mockGetScenario.mockRejectedValue(apiError(404, 'Unknown scenario "foundry.red_team_agent".')) + + renderEdit() + + expect(await screen.findByTestId('scenario-unavailable-warning')).toBeInTheDocument() + expect(screen.queryByTestId('editor-error-state')).not.toBeInTheDocument() + expect(screen.getByTestId('save-preset-btn')).toBeDisabled() + }) + + it('decodes a preset name that was escaped into the route', async () => { + mockGet.mockResolvedValue({ + preset: { ...STORED_PRESET, name: 'a_b' }, + version: 'v1', + issues: [], + }) + + renderEdit('a_b') + + await waitFor(() => expect(mockGet).toHaveBeenCalledWith('a_b')) + }) + + it('saves the edited description', async () => { + const user = userEvent.setup() + renderEdit() + + const description = await screen.findByTestId('preset-description-input') + await user.clear(description) + await user.type(description, 'Weekly smoke test.') + await user.click(screen.getByTestId('save-preset-btn')) + + await waitFor(() => expect(mockUpdate).toHaveBeenCalledWith( + 'nightly_probe', + expect.objectContaining({ description: 'Weekly smoke test.' }), + 'v1', + )) + }) + + it('saves the techniques and baseline choice the operator changed', async () => { + const user = userEvent.setup() + renderEdit() + + await user.click(await screen.findByTestId('technique-default_technique')) + await user.click(screen.getByTestId('baseline-checkbox')) + await user.click(screen.getByTestId('save-preset-btn')) + + await waitFor(() => expect(mockUpdate).toHaveBeenCalledWith( + 'nightly_probe', + expect.objectContaining({ + techniques: ['crescendo', 'default_technique'], + include_baseline: true, + }), + 'v1', + )) + }) + + it('saves the dataset selection the operator made', async () => { + const user = userEvent.setup() + renderEdit() + + await user.click(await screen.findByRole('combobox', { name: 'Add dataset' })) + await user.click(screen.getByRole('option', { name: 'xstest (unknown)', exact: true })) + await editLimit(user, 'Maximum objectives', '25') + await user.click(screen.getByTestId('save-preset-btn')) + + await waitFor(() => expect(mockUpdate).toHaveBeenCalled()) + const saved = mockUpdate.mock.calls.at(-1)?.[1] as ScenarioPreset + expect(saved.datasets?.sources?.map((source) => source.name)).toEqual(['harmbench', 'xstest']) + expect(saved.datasets?.max_total).toBe(25) + }) + + it('refuses to save while the dataset control reports an invalid limit', async () => { + const user = userEvent.setup() + renderEdit() + + await editLimit(user, 'Maximum objectives', '0') + await user.keyboard('{Escape}') + await user.click(screen.getByTestId('save-preset-btn')) + + await waitFor(() => expect(screen.getByTestId('save-preset-btn')).toBeEnabled()) + expect(mockUpdate).not.toHaveBeenCalled() + }) + + it('offers no objective cap on a scenario that sizes itself by prompt generation', async () => { + const generativeScenario = makeScenario({ + default_run_size: { + dataset_size: { kind: 'indeterminate', detail: 'Generated at run time.' }, + dataset_limit: { state: 'not_applicable' }, + estimated_attack_count: null, + components: [], + datasets: [], + note: null, + }, + }) + mockGetScenario.mockResolvedValue(generativeScenario) + renderEdit() + + expect(await screen.findByRole('combobox', { name: 'Add dataset' })).toBeInTheDocument() + expect(screen.queryByRole('button', { name: 'Edit Maximum objectives' })).not.toBeInTheDocument() + }) + + it('leaves the editor without saving when cancelled', async () => { + const user = userEvent.setup() + renderEdit() + + await user.click(await screen.findByRole('button', { name: 'Cancel' })) + + expect(mockUpdate).not.toHaveBeenCalled() + expect(mockNavigate).toHaveBeenCalledWith('/registry/scenario-presets') + }) + + it('returns to the library from the not-found state', async () => { + const user = userEvent.setup() + mockGet.mockRejectedValue(apiError(404, 'No such preset.')) + + renderEdit() + + await user.click(await screen.findByRole('button', { name: 'Back to presets' })) + + expect(mockNavigate).toHaveBeenCalledWith('/registry/scenario-presets') + }) + + it('reports a catalog failure that is not a missing preset as an error', async () => { + mockListCatalog.mockRejectedValue(apiError(503, 'Preset storage is not configured.')) + + renderEdit() + + expect(await screen.findByTestId('editor-error-state')).toHaveTextContent( + 'Preset storage is not configured.', + ) + }) +}) + +describe('ScenarioPresetEditor dataset projection', () => { + const defaults: ScenarioDatasetSettings = { + sources: [{ name: 'harmbench', max_size: 5, fetch: 'if_missing', configured_provider: false }], + max_per_dataset: 5, + max_total: 'all', + fetch: 'if_missing', + filters: {}, + } + + function projection(limit: number | 'all'): ScenarioRunSizeEstimateResponse { + return { + ...SCENARIO.default_run_size, + dataset_defaults: defaults, + effective_datasets: { + ...defaults, + max_per_dataset: limit, + sources: [{ ...defaults.sources[0], max_size: limit }], + }, + } + } + + beforeEach(() => { + jest.useFakeTimers() + mockGetScenario.mockResolvedValue(makeScenario({ dataset_defaults: defaults })) + }) + + afterEach(() => { + jest.useRealTimers() + }) + + it.each([ + { limit: 2, text: 'Up to 2 of 8' }, + { limit: 'all', text: '8 examples' }, + ] as const)('shows the effective size for max_per_dataset=$limit', async ({ limit, text }) => { + const datasets = { max_per_dataset: limit } + mockGet.mockResolvedValue({ + preset: { ...STORED_PRESET, datasets }, + version: 'v1', + issues: [], + }) + mockEstimateRun.mockResolvedValue(projection(limit)) + + renderEdit() + + const row = await screen.findByRole('row', { name: /harmbench/ }) + expect(await within(row).findByText(text)).toBeInTheDocument() + expect(mockEstimateRun).toHaveBeenCalledWith( + STORED_PRESET.scenario_name, + expect.objectContaining({ techniques: ['crescendo'], datasets, include_baseline: false }), + expect.any(AbortSignal), + ) + }) + + it('uses projected limits for a source that is not in the scenario defaults', async () => { + const datasets = { sources: [{ name: 'xstest' }], max_per_dataset: 2 } + mockGet.mockResolvedValue({ + preset: { ...STORED_PRESET, datasets }, + version: 'v1', + issues: [], + }) + mockEstimateRun.mockResolvedValue({ + ...projection(2), + effective_datasets: { + ...defaults, + max_per_dataset: 2, + sources: [{ name: 'xstest', max_size: 2, fetch: 'if_missing', configured_provider: false }], + }, + }) + + renderEdit() + + const row = await screen.findByRole('row', { name: /xstest/ }) + expect(await within(row).findByText('Up to 2 of 8')).toBeInTheDocument() + }) + + it('preserves a source-specific cap while projecting the inherited per-dataset cap', async () => { + const sourceDefaults: ScenarioDatasetSettings = { + ...defaults, + sources: [ + { ...defaults.sources[0], max_size: 3 }, + { ...defaults.sources[0], name: 'xstest' }, + ], + } + mockGetScenario.mockResolvedValue(makeScenario({ dataset_defaults: sourceDefaults })) + mockGet.mockResolvedValue({ + preset: { ...STORED_PRESET, datasets: { max_per_dataset: 2 } }, + version: 'v1', + issues: [], + }) + mockEstimateRun.mockResolvedValue({ + ...projection(2), + dataset_defaults: sourceDefaults, + effective_datasets: { + ...sourceDefaults, + max_per_dataset: 2, + sources: [sourceDefaults.sources[0], { ...sourceDefaults.sources[1], max_size: 2 }], + }, + }) + + renderEdit() + + const inherited = await screen.findByRole('row', { name: /xstest/ }) + expect(await within(inherited).findByText('Up to 2 of 8')).toBeInTheDocument() + const pinned = screen.getByRole('row', { name: /harmbench/ }) + expect(within(pinned).getByText('Up to 3 of 8')).toBeInTheDocument() + }) + + it('reports projection errors without preventing a preset from being saved', async () => { + const user = userEvent.setup({ advanceTimers: jest.advanceTimersByTime }) + mockEstimateRun.mockRejectedValue(new Error('Projection unavailable.')) + + renderEdit() + + expect(await screen.findByText( + 'Dataset preview could not be updated: Projection unavailable.', + )).toBeInTheDocument() + await user.click(screen.getByTestId('save-preset-btn')) + await waitFor(() => expect(mockUpdate).toHaveBeenCalled()) + }) + + it('ignores a pending projection after the operator resets the datasets', async () => { + const user = userEvent.setup({ advanceTimers: jest.advanceTimersByTime }) + let complete: (value: ScenarioRunSizeEstimateResponse) => void = () => {} + mockGet.mockResolvedValue({ + preset: { ...STORED_PRESET, datasets: { max_per_dataset: 2 } }, + version: 'v1', + issues: [], + }) + mockEstimateRun + .mockReturnValueOnce(new Promise((resolve) => { complete = resolve })) + .mockResolvedValue(projection(5)) + + renderEdit() + + await waitFor(() => expect(mockEstimateRun).toHaveBeenCalledTimes(1)) + const signal: AbortSignal = mockEstimateRun.mock.calls[0][2] + await user.click(screen.getByRole('button', { name: 'Reset datasets' })) + await waitFor(() => expect(mockEstimateRun).toHaveBeenCalledTimes(2)) + expect(signal.aborted).toBe(true) + + await act(async () => { complete(projection(2)) }) + + const row = screen.getByRole('row', { name: /harmbench/ }) + expect(within(row).getByText('Up to 5 of 8')).toBeInTheDocument() + expect(within(row).queryByText('Up to 2 of 8')).not.toBeInTheDocument() + }) + + it('stops the preview spinner when an invalid edit cancels the pending request', async () => { + const user = userEvent.setup({ advanceTimers: jest.advanceTimersByTime }) + mockEstimateRun.mockReturnValueOnce(new Promise(() => {})) + + renderEdit() + + await waitFor(() => expect(mockEstimateRun).toHaveBeenCalledTimes(1)) + expect(screen.getByText('Updating dataset preview...')).toBeInTheDocument() + const signal: AbortSignal = mockEstimateRun.mock.calls[0][2] + + await user.click(screen.getByTestId('technique-crescendo')) + + expect(signal.aborted).toBe(true) + expect(screen.queryByText('Updating dataset preview...')).not.toBeInTheDocument() + await user.click(screen.getByTestId('save-preset-btn')) + expect(await screen.findByText('Select at least one technique.')).toBeInTheDocument() + expect(mockUpdate).not.toHaveBeenCalled() + }) +}) + +describe('ScenarioPresetEditor dynamic parameters', () => { + const SCENARIO_WITH_PARAM = makeScenario({ + supported_parameters: [ + { + name: 'max_turns', + type_name: 'int', + required: false, + default: '5', + description: 'Turn budget.', + }, + ], + }) + + it('stores a scenario-specific parameter the operator changed', async () => { + const user = userEvent.setup() + mockGetScenario.mockResolvedValue(SCENARIO_WITH_PARAM) + renderEdit() + + const field = await screen.findByTestId('preset-param-max_turns') + await user.clear(field) + await user.type(field, '9') + await user.click(screen.getByTestId('save-preset-btn')) + + await waitFor(() => expect(mockUpdate).toHaveBeenCalledWith( + 'nightly_probe', + expect.objectContaining({ scenario_params: { max_turns: 9 } }), + 'v1', + )) + }) + + it('preserves and surfaces stored parameters this editor renders no field for', async () => { + const user = userEvent.setup() + mockGetScenario.mockResolvedValue(SCENARIO_WITH_PARAM) + mockGet.mockResolvedValue({ + preset: { ...STORED_PRESET, scenario_params: { max_turns: 5, max_concurrency: 4 } }, + version: 'v1', + issues: [], + }) + + renderEdit() + + expect(await screen.findByTestId('carried-params-notice')).toHaveTextContent('max_concurrency') + + await user.click(screen.getByTestId('save-preset-btn')) + + await waitFor(() => expect(mockUpdate).toHaveBeenCalledWith( + 'nightly_probe', + expect.objectContaining({ scenario_params: { max_turns: 5, max_concurrency: 4 } }), + 'v1', + )) + }) +}) diff --git a/frontend/src/components/ScenarioPresets/ScenarioPresetEditor.tsx b/frontend/src/components/ScenarioPresets/ScenarioPresetEditor.tsx new file mode 100644 index 0000000000..2cabb296b3 --- /dev/null +++ b/frontend/src/components/ScenarioPresets/ScenarioPresetEditor.tsx @@ -0,0 +1,503 @@ +import { type FormEvent, useCallback, useEffect, useMemo, useState } from 'react' + +import { + Button, + Field, + Input, + MessageBar, + MessageBarBody, + Option, + Spinner, + Combobox, + Text, + Textarea, +} from '@fluentui/react-components' +import { useNavigate, useParams } from 'react-router' + +import ParameterField from '@/components/Parameters/ParameterField' +import type { ParameterFormValue } from '@/components/Parameters/parameterForm' +import ScenarioDatasetTable from '@/components/Scenarios/ScenarioDatasetTable' +import ScenarioTechniqueSelector from '@/components/Scenarios/ScenarioTechniqueSelector' +import { scenarioDatasetDefaults, scenarioDatasetOptions } from '@/components/Scenarios/datasetPresentation' +import { + buildScenarioConfig, + dynamicScenarioParameters, + initialScenarioConfigState, + uniqueTechniqueOptions, + type ScenarioConfigFormState, +} from '@/components/Scenarios/scenarioConfigForm' +import { useScenarioDatasets } from '@/hooks/useScenarioDatasets' +import { scenarioPresetsApi, scenariosApi } from '@/services/api' +import { toApiError } from '@/services/errors' +import type { + RegisteredScenario, + ScenarioPresetResponse, + ScenarioRunSizeEstimateRequest, + ScenarioRunSizeEstimateResponse, +} from '@/types' +import { fetchAllPages } from '@/utils/fetchAllPages' +import { routerPathParamValue } from '@/utils/routeParams' + +import { useScenarioPresetEditorStyles } from './ScenarioPresetEditor.styles' +import { PRESETS_ROUTE } from './presetRoutes' +import { + configToPreset, + presetToConfigState, + uneditableScenarioParams, + validatePresetName, +} from './scenarioPresetForm' + +/** Items requested per catalog page while paging the scenario picker's options. */ +const CATALOG_PAGE_SIZE = 200 +const DATASET_PROJECTION_DEBOUNCE_MS = 250 + +type LoadStatus = 'loading' | 'ready' | 'not-found' | 'error' + +interface DatasetProjectionResult { + request: ScenarioRunSizeEstimateRequest + scenarioName: string + error: string | null +} + +export default function ScenarioPresetEditor({ mode }: { mode: 'create' | 'edit' }) { + const { presetName } = useParams<{ presetName: string }>() + // Remount on navigation between two presets so every field resets to the + // newly loaded document rather than retaining the previous one's edits. + return +} + +interface ScenarioPresetEditorContentProps { + mode: 'create' | 'edit' + presetName: string | undefined +} + +function ScenarioPresetEditorContent({ mode, presetName }: ScenarioPresetEditorContentProps) { + const styles = useScenarioPresetEditorStyles() + const navigate = useNavigate() + const decodedName = presetName === undefined ? '' : routerPathParamValue(presetName) + + const [status, setStatus] = useState('loading') + const [loadError, setLoadError] = useState(null) + const [catalog, setCatalog] = useState([]) + const [loaded, setLoaded] = useState(null) + + const [name, setName] = useState(mode === 'edit' ? decodedName : '') + const [description, setDescription] = useState('') + const [scenario, setScenario] = useState(null) + const [scenarioLoading, setScenarioLoading] = useState(false) + const [config, setConfig] = useState(null) + const [scenarioUnavailable, setScenarioUnavailable] = useState(null) + const [datasetError, setDatasetError] = useState(null) + const [datasetProjection, setDatasetProjection] = useState(null) + const [datasetProjectionResult, setDatasetProjectionResult] = useState(null) + + const [saving, setSaving] = useState(false) + const [validationError, setValidationError] = useState(null) + const [saveError, setSaveError] = useState(null) + + useEffect(() => { + let cancelled = false + + const load = async (): Promise => { + try { + const scenarios = await fetchAllPages( + (cursor) => scenariosApi.listCatalog(CATALOG_PAGE_SIZE, cursor, false), + undefined, + (entry) => entry.scenario_name, + ) + if (cancelled) return + setCatalog(scenarios) + + if (mode === 'create') { + setStatus('ready') + return + } + + const response = await scenarioPresetsApi.get(decodedName) + if (cancelled) return + setLoaded(response) + setName(response.preset.name) + setDescription(response.preset.description ?? '') + setStatus('ready') + + // The list endpoint omits run-size estimates, so the preset's scenario is + // re-fetched in full to get the configured dataset caps the form shows. + // A preset may legitimately name a scenario this deployment lacks; that + // is an unavailable scenario, not a missing preset, so it must not be + // reported as a 404 on the preset itself. + try { + const full = await scenariosApi.getScenario(response.preset.scenario_name) + if (cancelled) return + setScenario(full) + setConfig(presetToConfigState(full, response.preset)) + } catch (scenarioErr) { + if (cancelled) return + setScenarioUnavailable(toApiError(scenarioErr).detail) + } + } catch (err) { + if (cancelled) return + const apiError = toApiError(err) + setLoadError(apiError.detail) + setStatus(apiError.status === 404 ? 'not-found' : 'error') + } + } + + void load() + return () => { + cancelled = true + } + }, [decodedName, mode]) + + const handleScenarioChange = useCallback(async (nextScenarioName: string): Promise => { + setScenarioLoading(true) + setSaveError(null) + setValidationError(null) + setDatasetProjection(null) + setDatasetProjectionResult(null) + try { + const full = await scenariosApi.getScenario(nextScenarioName) + setScenario(full) + setConfig(initialScenarioConfigState(full)) + } catch (err) { + setSaveError(toApiError(err).detail) + } finally { + setScenarioLoading(false) + } + }, []) + + const datasetMetadata = useScenarioDatasets() + const datasetBaseline = useMemo( + () => (scenario + ? { + defaults: datasetProjection?.dataset_defaults ?? scenarioDatasetDefaults(scenario), + options: datasetProjection?.dataset_options ?? scenarioDatasetOptions(scenario), + } + : null), + [datasetProjection, scenario], + ) + + const dynamicParameters = useMemo( + () => (scenario ? dynamicScenarioParameters(scenario) : []), + [scenario], + ) + const techniqueOptions = useMemo( + () => (scenario ? uniqueTechniqueOptions(scenario).techniques : []), + [scenario], + ) + const carriedScenarioParams = useMemo( + () => (scenario ? Object.keys(uneditableScenarioParams(scenario, loaded?.preset ?? null)) : []), + [loaded, scenario], + ) + const techniqueIssues = loaded?.issues.filter((issue) => issue.field === 'techniques') ?? [] + const builtConfig = useMemo( + () => (config ? buildScenarioConfig({ + techniques: config.techniques, + dynamicParameters, + scenarioParamValues: config.scenarioParamValues, + datasets: config.datasets, + includeBaseline: config.includeBaseline, + }) : null), + [config, dynamicParameters], + ) + const scenarioName = scenario?.scenario_name + const projectionRequest = !scenarioLoading && builtConfig?.ok ? builtConfig.config : null + const projectionIsCurrent = datasetProjectionResult?.request === projectionRequest + && datasetProjectionResult?.scenarioName === scenarioName + const datasetProjectionLoading = Boolean(scenarioName && projectionRequest && !projectionIsCurrent) + const datasetProjectionError = projectionIsCurrent ? datasetProjectionResult?.error : null + + useEffect(() => { + if (!scenarioName || !projectionRequest) return + + const controller = new AbortController() + const timer = window.setTimeout(() => { + void scenariosApi.estimateRun(scenarioName, projectionRequest, controller.signal) + .then((response) => { + if (controller.signal.aborted) return + setDatasetProjection(response) + setDatasetProjectionResult({ request: projectionRequest, scenarioName, error: null }) + }) + .catch((err: unknown) => { + if (controller.signal.aborted) return + setDatasetProjectionResult({ request: projectionRequest, scenarioName, error: toApiError(err).detail }) + }) + }, DATASET_PROJECTION_DEBOUNCE_MS) + + return () => { + window.clearTimeout(timer) + controller.abort() + } + }, [projectionRequest, scenarioName]) + + const updateConfig = (patch: Partial): void => { + setConfig((current) => (current ? { ...current, ...patch } : current)) + setValidationError(null) + } + + const updateScenarioParam = useCallback((parameterName: string, value: ParameterFormValue) => { + setConfig((current) => (current + ? { ...current, scenarioParamValues: { ...current.scenarioParamValues, [parameterName]: value } } + : current)) + setValidationError(null) + }, []) + + const handleSubmit = async (event: FormEvent): Promise => { + event.preventDefault() + if (saving) { + return + } + setSaveError(null) + + const trimmedName = name.trim() + const nameError = validatePresetName(trimmedName) + if (nameError) { + setValidationError(nameError) + return + } + if (!scenario || !config) { + setValidationError('Select a scenario.') + return + } + if (!builtConfig?.ok) { + setValidationError(builtConfig?.error ?? 'Select a scenario.') + return + } + if (datasetError) { + setValidationError(datasetError) + return + } + setValidationError(null) + + const preset = configToPreset( + { name: trimmedName, scenarioName: scenario.scenario_name, description }, + builtConfig.config, + { scenario, previous: loaded?.preset ?? null }, + ) + + setSaving(true) + try { + if (loaded === null) { + await scenarioPresetsApi.create(preset) + } else if (loaded.version === null) { + setSaveError(`Preset '${loaded.preset.name}' is defined in code and cannot be edited here.`) + setSaving(false) + return + } else { + await scenarioPresetsApi.update(loaded.preset.name, preset, loaded.version) + } + navigate(PRESETS_ROUTE) + } catch (err) { + setSaveError(toApiError(err).detail) + setSaving(false) + } + } + + if (status === 'loading') { + return ( +
+
+ +
+
+ ) + } + + if (status !== 'ready') { + return ( +
+
+ + + {status === 'not-found' ? `No preset named "${decodedName}".` : loadError} + + + +
+
+ ) + } + + const editing = loaded !== null + + return ( +
+
+ + {editing ? `Edit ${decodedName}` : 'New preset'} + + + A preset stores what to test. The target, concurrency and retries are chosen at launch. + +
+ +
+ {saveError && ( + + {saveError} + + )} + {validationError && ( + + {validationError} + + )} + {techniqueIssues.length > 0 && ( + + + {techniqueIssues.map((issue) => issue.message).join(' ')} Stored techniques + are preserved on save unless you remove them below. + + + )} + {carriedScenarioParams.length > 0 && ( + + + This preset carries parameters this editor has no field for + ({carriedScenarioParams.join(', ')}). They are preserved as-is on save. + + + )} + {scenarioUnavailable && ( + + + This preset targets {loaded?.preset.scenario_name}, which is not available here + ({scenarioUnavailable}). Its configuration cannot be edited in this deployment. + + + )} + +
+ + { + setName(data.value) + setValidationError(null) + }} + data-testid="preset-name-input" + /> + + +