diff --git a/doc/gui/0_gui.md b/doc/gui/0_gui.md index 4a8658af25..3897992c31 100644 --- a/doc/gui/0_gui.md +++ b/doc/gui/0_gui.md @@ -977,6 +977,32 @@ pairs. For example, four pairs with one of each transition give 50% success on each side and a 0 pp difference, while still showing one failure becoming a success and one success becoming a failure. +Each transition row can be expanded to inspect the matched logical objectives +behind that count. Expanding loads only the selected transition's pairs, with +the baseline and attack outcomes shown side by side and links to each saved +conversation. Conversation links retain the originating Scenario run context. +When a compound result has no conversation of its own, the panel links to one +of its recorded child conversations and labels the child's outcome separately. +It prefers the last linked conversation matching the compound outcome, falling +back to the last linked conversation when none matches. The displayed pair +outcome and observation count still belong to the top-level result. +Use **Load more** to inspect additional pairs, or collapse the row to hide its +details. Large transitions are loaded in bounded pages; a failed request can +be retried from the expanded row. Switching selected groups clears the details. +All results use the same decided-pair cohort and most recent top-level attempt +as the transition counts, so retries and child attempts are never additional +pairs. + +The following focused screenshots show one expanded transition and its conversation links. + +Expanded transition details on desktop: + +![Paired outcomes with an expanded transition, objective, and baseline and attack conversation links](./images/paired-outcomes-desktop.png) + +The same transition details stack vertically on a narrow screen: + +![Expanded paired outcomes at a mobile viewport width](./images/paired-outcomes-mobile.png) + **Decided pairs** shows the number with success or failure on both sides, followed by the number planned on both sides. **Excluded pairs** counts the remaining shared pairs. Missing outcomes, errors, and undetermined outcomes diff --git a/doc/gui/images/paired-outcomes-desktop.png b/doc/gui/images/paired-outcomes-desktop.png new file mode 100644 index 0000000000..ce3a198ba0 Binary files /dev/null and b/doc/gui/images/paired-outcomes-desktop.png differ diff --git a/doc/gui/images/paired-outcomes-mobile.png b/doc/gui/images/paired-outcomes-mobile.png new file mode 100644 index 0000000000..ade41f0d46 Binary files /dev/null and b/doc/gui/images/paired-outcomes-mobile.png differ diff --git a/frontend/e2e/fixtures/paired_outcomes_backend.py b/frontend/e2e/fixtures/paired_outcomes_backend.py index 31e68731a5..d6f300612e 100644 --- a/frontend/e2e/fixtures/paired_outcomes_backend.py +++ b/frontend/e2e/fixtures/paired_outcomes_backend.py @@ -24,12 +24,15 @@ async def _seed_async(memory: SQLiteMemory) -> None: AttackIdentifier, AttackOutcome, AttackResult, + Conversation, + MessagePiece, ScenarioIdentifier, ScenarioResult, ScenarioRunPlan, ScenarioRunPlanAtomicGroup, ScenarioRunPlanGroupKind, ScenarioRunPlanSeedGroup, + ScenarioRunPlanSeedPrompt, ScenarioRunState, TargetIdentifier, ) @@ -45,28 +48,51 @@ async def _seed_async(memory: SQLiteMemory) -> None: objective_target=target, ) ) - plan = ScenarioRunPlan( - atomic_groups=[ - ScenarioRunPlanAtomicGroup( - id=name, - atomic_attack_name=name, - display_group=name.title(), - technique_eval_hash=f"{name}-eval", - kind=kind, - seed_group_ids=shared + [extra], - ) - for name, kind, extra in [ - ("baseline", ScenarioRunPlanGroupKind.BASELINE, "base-only"), - ("attack", ScenarioRunPlanGroupKind.ATTACK, "attack-only"), - ] - ], - seed_groups=[ - ScenarioRunPlanSeedGroup(id=seed, objective=seed, objective_sha256=to_sha256(seed)) - for seed in shared + ["base-only", "attack-only"] - ], - ) - for index, state in enumerate([ScenarioRunState.COMPLETED, ScenarioRunState.COMPLETED, ScenarioRunState.FAILED]): + for index, state in enumerate( + [ + ScenarioRunState.COMPLETED, + ScenarioRunState.COMPLETED, + ScenarioRunState.FAILED, + ScenarioRunState.COMPLETED, + ScenarioRunState.COMPLETED, + ] + ): scenario_id = UUID(int=9100 + index) + page_seeds = ( + [f"page-{number:02d}" + ("x" * 900 if index == 4 and number == 23 else "") for number in range(30)] + if index in (3, 4) + else [] + ) + selected_seeds = shared + page_seeds + plan = ScenarioRunPlan( + atomic_groups=[ + ScenarioRunPlanAtomicGroup( + id=name, + atomic_attack_name=name, + display_group=name.title(), + technique_eval_hash=f"{name}-eval", + kind=kind, + seed_group_ids=selected_seeds + [extra], + ) + for name, kind, extra in [ + ("baseline", ScenarioRunPlanGroupKind.BASELINE, "base-only"), + ("attack", ScenarioRunPlanGroupKind.ATTACK, "attack-only"), + ] + ], + seed_groups=[ + ScenarioRunPlanSeedGroup( + id=seed, + objective=seed, + objective_sha256=to_sha256(seed), + prompts=[ + ScenarioRunPlanSeedPrompt(value="LongPrompt" * 80, sequence=0, data_type="text", role="user") + ] + if index == 4 and seed == "a" + else [], + ) + for seed in selected_seeds + ["base-only", "attack-only"] + ], + ) rows: dict[str, list[AttackResult]] = {"baseline": [], "attack": []} for name, outcomes in [ ("baseline", [AttackOutcome.FAILURE, AttackOutcome.SUCCESS, AttackOutcome.FAILURE, AttackOutcome.SUCCESS]), @@ -89,6 +115,24 @@ async def _seed_async(memory: SQLiteMemory) -> None: }, ) ) + for seed in page_seeds: + for name, outcome in [("baseline", AttackOutcome.FAILURE), ("attack", AttackOutcome.SUCCESS)]: + rows[name].append( + AttackResult( + attack_result_id=str(uuid4()), + conversation_id=str(uuid4()), + objective=seed, + outcome=outcome, + timestamp=timestamp, + atomic_attack_identifier=identifier, + attribution_parent_id=str(scenario_id), + attribution_data={ + "seed_group_id": seed, + "parent_eval_hash": f"{name}-eval", + "parent_collection": name, + }, + ) + ) for seed, outcome in [ ("undetermined", AttackOutcome.UNDETERMINED), ("error", AttackOutcome.ERROR), @@ -128,6 +172,42 @@ async def _seed_async(memory: SQLiteMemory) -> None: }, ) ) + if index == 4: + parent = rows["attack"][0] + child = parent.model_copy(deep=True) + child.attack_result_id = str(uuid4()) + child.conversation_id = str(uuid4()) + assert child.attribution_data is not None + child.attribution_data["attempt_index"] = 1 + parent.conversation_id = "" + parent.metadata["child_attack_result_ids"] = [child.attack_result_id] + rows["attack"].append(child) + for attack_result in (item for group in rows.values() for item in group): + if not attack_result.conversation_id: + continue + await memory.add_conversation_to_memory_async( + conversation=Conversation( + conversation_id=attack_result.conversation_id, + target_identifier=target, + attack_result_id=attack_result.attack_result_id, + ) + ) + await memory.add_message_pieces_to_memory_async( + message_pieces=[ + MessagePiece( + role="user", + conversation_id=attack_result.conversation_id, + sequence=0, + original_value=f"Offline objective: {attack_result.objective}", + ), + MessagePiece( + role="assistant", + conversation_id=attack_result.conversation_id, + sequence=1, + original_value=f"Offline {attack_result.outcome.value} example; no model was called.", + ), + ] + ) await memory.add_attack_results_to_memory_async( attack_results=[row for group in rows.values() for row in group] ) diff --git a/frontend/e2e/paired-outcomes.spec.ts b/frontend/e2e/paired-outcomes.spec.ts index 4170ae90f0..0c5a219d43 100644 --- a/frontend/e2e/paired-outcomes.spec.ts +++ b/frontend/e2e/paired-outcomes.spec.ts @@ -1,9 +1,29 @@ -import { expect, test } from '@playwright/test' +import { expect, test, type Locator, type Page } from '@playwright/test' import { scenarioRunRoutePath } from '../src/utils/routeParams' const run = scenarioRunRoutePath('00000000-0000-0000-0000-00000000238c') +const paginatedRun = scenarioRunRoutePath('00000000-0000-0000-0000-00000000238f') -test('compares persisted outcomes through the real API, including retries and exclusions', async ({ page }) => { +async function captureExpandedTransition(page: Page, table: Locator, details: Locator, path: string): Promise { + const header = table.getByRole('row').first() + await header.evaluate((element) => element.scrollIntoView({ block: 'start' })) + await expect(details.getByRole('link', { name: 'Open baseline conversation' })).toBeInViewport({ ratio: 1 }) + await expect(details.getByRole('link', { name: 'Open attack conversation' })).toBeInViewport({ ratio: 1 }) + const headerBox = await header.boundingBox() + const detailsBox = await details.boundingBox() + const viewport = page.viewportSize() + if (!headerBox || !detailsBox || !viewport) throw new Error('Missing screenshot bounds') + const clip = { + x: headerBox.x, y: headerBox.y, width: headerBox.width, + height: detailsBox.y + detailsBox.height - headerBox.y + 8, + } + expect(clip.y).toBeGreaterThanOrEqual(0) + expect(clip.y + clip.height).toBeLessThanOrEqual(viewport.height) + expect(clip.height).toBeLessThanOrEqual(700) + await page.screenshot({ path, clip }) +} + +test('compares persisted outcomes through the real API, including retries and exclusions', async ({ page }, testInfo) => { const errors: string[] = [] page.on('pageerror', (error) => errors.push(error.message)) await page.goto(run) @@ -30,13 +50,53 @@ test('compares persisted outcomes through the real API, including retries and ex for (const label of ['Failure → Success', 'Success → Failure', 'Failure → Failure', 'Success → Success']) { await expect(comparison.getByRole('row', { name: `${label} 1` })).toBeVisible() } + const detailRequests: string[] = [] + page.on('request', (request) => { + if (request.url().includes('/paired-outcomes/details?')) detailRequests.push(request.url()) + }) + const expand = comparison.getByRole('button', { name: 'Failure → Success' }) + await expect(expand).toHaveAttribute('aria-expanded', 'false') + expect(detailRequests).toEqual([]) + const detailsResponse = page.waitForResponse((res) => res.url().includes('/paired-outcomes/details?')) + await expand.click() + const detailResult = await detailsResponse + expect(detailResult.status()).toBe(200) + expect(await detailResult.json()).toMatchObject({ + baseline_group_id: 'baseline', attack_group_id: 'attack', transition: 'failure_to_success', + total_pairs: 1, pairs: [{ seed_group_id: 'a', objective: 'a', baseline: { outcome: 'failure' }, attack: { outcome: 'success' } }], + }) + const details = panel.getByRole('region', { name: 'Failure → Success pair details' }) + await expect(details).toContainText('Logical seed: a') + await expect(details).toContainText('Outcome: failure') + await expect(details).toContainText('Outcome: success') + const baselineLink = details.getByRole('link', { name: 'Open baseline conversation' }) + const attackLink = details.getByRole('link', { name: 'Open attack conversation' }) + await expect(baselineLink).toHaveAttribute('href', /scenarioResultId=00000000-0000-0000-0000-00000000238c/) + await expect(attackLink).toHaveAttribute('href', /scenarioResultId=00000000-0000-0000-0000-00000000238c/) + await captureExpandedTransition(page, comparison.getByRole('table', { name: 'Outcome transitions' }), details, + testInfo.outputPath('paired-outcomes-desktop.png')) + await attackLink.click() + await expect(page).toHaveURL(/\/attacks\/[^/]+\/conversations\/[^/]+\?scenarioResultId=/) + await expect(page.getByText('Offline objective: a')).toBeVisible() + await page.goto(run) + const restored = page.getByRole('region', { name: 'Paired outcomes', exact: true }) + await restored.getByLabel('Baseline group').selectOption('baseline') + await restored.getByLabel('Attack group').selectOption('attack') + await restored.getByRole('button', { name: 'Compare outcomes' }).click() + const restoredExpand = restored.getByRole('button', { name: 'Failure → Success' }) + await restoredExpand.click() + await expect(restored.getByRole('region', { name: 'Failure → Success pair details' })).toBeVisible() + await restoredExpand.click() + await expect(restored.getByRole('region', { name: 'Failure → Success pair details' })).toHaveCount(0) + await restoredExpand.click() + await expect(restored.getByRole('region', { name: 'Failure → Success pair details' })).toContainText('Logical seed: a') await panel.getByLabel('Attack group').selectOption('') await expect(comparison).toHaveCount(0) await expect(panel.getByRole('button', { name: 'Compare outcomes' })).toBeDisabled() expect(errors).toEqual([]) }) -test('keeps comparisons usable at a narrow viewport and with keyboard controls', async ({ page }, testInfo) => { +test('keeps expanded pairs usable at a narrow viewport and with keyboard controls', async ({ page }, testInfo) => { await page.setViewportSize({ width: 390, height: 844 }) await page.goto(run) const panel = page.getByRole('region', { name: 'Paired outcomes', exact: true }) @@ -50,13 +110,51 @@ test('keeps comparisons usable at a narrow viewport and with keyboard controls', await expect(button).toBeFocused() await page.keyboard.press('Enter') await expect(panel.getByRole('region', { name: 'Paired comparison' })).toBeVisible() + const baselineMetric = await panel.getByRole('group', { name: 'Baseline success rate' }).boundingBox() + const attackMetric = await panel.getByRole('group', { name: 'Attack success rate' }).boundingBox() + expect(baselineMetric?.y).toBe(attackMetric?.y) const overflow = await panel.evaluate((element) => element.scrollWidth > element.clientWidth) expect(overflow).toBe(false) const size = await button.boundingBox() expect(size?.height).toBeGreaterThanOrEqual(44) await panel.getByRole('table', { name: 'Outcome transitions' }).scrollIntoViewIfNeeded() await expect(panel.getByRole('row', { name: 'Failure → Success 1' })).toBeVisible() - await page.screenshot({ path: testInfo.outputPath('paired-outcomes-mobile.png') }) + const expand = panel.getByRole('button', { name: 'Failure → Success' }) + await expand.focus() + await page.keyboard.press('Enter') + await expect(expand).toHaveAttribute('aria-expanded', 'true') + expect((await expand.boundingBox())?.height).toBeLessThanOrEqual(44) + await expect(panel.getByRole('region', { name: 'Failure → Success pair details' })).toContainText('Logical seed: a') + expect(await panel.evaluate((element) => element.scrollWidth > element.clientWidth)).toBe(false) + await captureExpandedTransition(page, panel.getByRole('table', { name: 'Outcome transitions' }), + panel.getByRole('region', { name: 'Failure → Success pair details' }), testInfo.outputPath('paired-outcomes-mobile.png')) +}) + +test('paginates matching SQLite pairs', async ({ page }) => { + await page.goto(paginatedRun) + const panel = page.getByRole('region', { name: 'Paired outcomes', exact: true }) + await panel.getByLabel('Baseline group').selectOption('baseline') + await panel.getByLabel('Attack group').selectOption('attack') + await panel.getByRole('button', { name: 'Compare outcomes' }).click() + await expect(panel.getByRole('row', { name: 'Failure → Success 31' })).toBeVisible() + const expand = panel.getByRole('button', { name: 'Failure → Success' }) + await expand.click() + const details = panel.getByRole('region', { name: 'Failure → Success pair details' }) + await expect(details).toContainText('25 of 31 pairs shown') + await expect(details.getByRole('article')).toHaveCount(25) + const nextPage = page.waitForResponse((response) => response.url().includes('/paired-outcomes/details?') && response.url().includes('cursor=')) + await details.getByRole('button', { name: 'Load more Failure → Success pairs' }).focus() + await page.keyboard.press('Enter') + const result = await nextPage + expect(result.status()).toBe(200) + expect(await result.json()).toMatchObject({ total_pairs: 31, next_cursor: null }) + await expect(details).toContainText('31 of 31 pairs shown') + await expect(details.getByRole('article')).toHaveCount(31) + const firstNewPair = details.getByRole('article').nth(25) + await expect(firstNewPair).toBeFocused() + await page.keyboard.press('Tab') + await expect(firstNewPair.getByRole('link', { name: 'Open baseline conversation' })).toBeFocused() + await expect(details.getByRole('button', { name: 'Load more Failure → Success pairs' })).toHaveCount(0) }) test('explains why legacy or failed runs cannot be compared', async ({ page }) => { @@ -65,3 +163,50 @@ test('explains why legacy or failed runs cannot be compared', async ({ page }) = await page.goto(scenarioRunRoutePath('00000000-0000-0000-0000-00000000238e')) await expect(page.getByRole('region', { name: 'Paired outcomes', exact: true })).toContainText('Complete the run') }) + +test('opens a compound child conversation and pages long seed IDs on mobile', async ({ page }) => { + await page.setViewportSize({ width: 320, height: 844 }) + await page.goto(scenarioRunRoutePath('00000000-0000-0000-0000-000000002390')) + const panel = page.getByRole('region', { name: 'Paired outcomes', exact: true }) + await panel.getByLabel('Baseline group').selectOption('baseline') + await panel.getByLabel('Attack group').selectOption('attack') + await panel.getByRole('button', { name: 'Compare outcomes' }).click() + await panel.getByRole('button', { name: 'Failure → Success' }).click() + const details = panel.getByRole('region', { name: 'Failure → Success pair details' }) + await expect(details).toContainText('25 of 31 pairs shown') + await expect(details).toContainText('Linked child outcome: success') + expect(await panel.evaluate((element) => element.scrollWidth > element.clientWidth)).toBe(false) + const nextPage = page.waitForResponse((response) => response.url().includes('/paired-outcomes/details?') && response.url().includes('cursor=')) + await details.getByRole('button', { name: 'Load more Failure → Success pairs' }).click() + expect((await nextPage).status()).toBe(200) + await expect(details).toContainText('31 of 31 pairs shown') + const baseline = details.getByRole('link', { name: 'Open baseline conversation', exact: true }).first() + const baselinePath = await baseline.getAttribute('href') + const child = details.getByRole('link', { name: 'Open attack child conversation', exact: true }) + await child.click() + await expect(page).toHaveURL(/\/attacks\/[^/]+\/conversations\/[^/]+\?scenarioResultId=00000000-0000-0000-0000-000000002390/) + await expect(page.getByText('Offline objective: a')).toBeVisible() + if (!baselinePath) throw new Error('Missing baseline conversation link') + await page.goto(baselinePath) + await expect(page.getByText('Offline objective: a')).toBeVisible() +}) + +test('offers a usable mobile retry after a real network failure', async ({ page, context }) => { + await page.setViewportSize({ width: 320, height: 844 }) + await page.goto(run) + const panel = page.getByRole('region', { name: 'Paired outcomes', exact: true }) + await panel.getByLabel('Baseline group').selectOption('baseline') + await panel.getByLabel('Attack group').selectOption('attack') + await panel.getByRole('button', { name: 'Compare outcomes' }).click() + const expand = panel.getByRole('button', { name: 'Failure → Success' }) + await expect(expand).toBeVisible() + await context.setOffline(true) + await expand.click() + const retry = panel.getByRole('button', { name: 'Retry Failure → Success pairs' }) + await expect(retry).toBeVisible() + expect(await panel.evaluate((element) => element.scrollWidth > element.clientWidth)).toBe(false) + expect((await retry.boundingBox())?.height).toBeGreaterThanOrEqual(44) + await context.setOffline(false) + await retry.click() + await expect(panel.getByRole('region', { name: 'Failure → Success pair details' })).toContainText('Logical seed: a') +}) diff --git a/frontend/src/components/Scenarios/PairedOutcomes.styles.ts b/frontend/src/components/Scenarios/PairedOutcomes.styles.ts index fca3f35478..c94f3644ef 100644 --- a/frontend/src/components/Scenarios/PairedOutcomes.styles.ts +++ b/frontend/src/components/Scenarios/PairedOutcomes.styles.ts @@ -35,7 +35,79 @@ export const usePairedOutcomesStyles = makeStyles({ }, metrics: { display: 'grid', + overflowWrap: 'anywhere', gridTemplateColumns: 'repeat(auto-fit, minmax(9rem, 1fr))', gap: tokens.spacingHorizontalL, + [NARROW_VIEWPORT_QUERY]: { gridTemplateColumns: 'repeat(2, minmax(0, 1fr))' }, + }, + transitions: { tableLayout: 'fixed', width: '100%' }, + countCell: { width: '3.5rem', textAlign: 'end', whiteSpace: 'nowrap' }, + expandButton: { + minHeight: MINIMUM_TOUCH_TARGET_SIZE, + maxWidth: '100%', + textAlign: 'start', + whiteSpace: 'normal', + justifyContent: 'flex-start', + }, + detailsCell: { + padding: tokens.spacingVerticalM, + minWidth: 0, + [NARROW_VIEWPORT_QUERY]: { padding: tokens.spacingVerticalS }, + }, + details: { + display: 'flex', + flexDirection: 'column', + gap: tokens.spacingVerticalM, + minWidth: 0, + }, + pairCard: { + minWidth: 0, + display: 'flex', + flexDirection: 'column', + gap: tokens.spacingVerticalM, + border: `1px solid ${tokens.colorNeutralStroke2}`, + borderRadius: tokens.borderRadiusMedium, + padding: tokens.spacingHorizontalM, + }, + pairHeader: { minWidth: 0 }, + wrappingText: { overflowWrap: 'anywhere', whiteSpace: 'pre-wrap' }, + promptValue: { overflowWrap: 'anywhere', whiteSpace: 'pre-wrap' }, + seedPrompts: { display: 'flex', flexDirection: 'column', gap: tokens.spacingVerticalXS, minWidth: 0 }, + sides: { + display: 'grid', + gridTemplateColumns: 'repeat(2, minmax(0, 1fr))', + gap: tokens.spacingHorizontalM, + [NARROW_VIEWPORT_QUERY]: { gridTemplateColumns: 'minmax(0, 1fr)' }, + }, + side: { + display: 'flex', + flexDirection: 'column', + gap: tokens.spacingVerticalXS, + minWidth: 0, + overflowWrap: 'anywhere', + }, + conversationLink: { + display: 'inline-flex', + alignItems: 'center', + minHeight: MINIMUM_TOUCH_TARGET_SIZE, + alignSelf: 'flex-start', + color: tokens.colorBrandForegroundLink, + ':focus-visible': { outline: `2px solid ${tokens.colorStrokeFocus2}`, outlineOffset: '2px' }, + }, + feedback: { + display: 'flex', + flexDirection: 'column', + alignItems: 'flex-start', + gap: tokens.spacingVerticalS, + '& button': { minHeight: MINIMUM_TOUCH_TARGET_SIZE }, + }, + errorMessage: { width: '100%', minWidth: 0, overflowWrap: 'anywhere' }, + pageControls: { + display: 'flex', + alignItems: 'center', + justifyContent: 'space-between', + flexWrap: 'wrap', + gap: tokens.spacingHorizontalS, + '& button': { minHeight: MINIMUM_TOUCH_TARGET_SIZE }, }, }) diff --git a/frontend/src/components/Scenarios/PairedOutcomes.test.tsx b/frontend/src/components/Scenarios/PairedOutcomes.test.tsx index 895e14580d..73b6637cba 100644 --- a/frontend/src/components/Scenarios/PairedOutcomes.test.tsx +++ b/frontend/src/components/Scenarios/PairedOutcomes.test.tsx @@ -1,12 +1,15 @@ import { act, render, screen, within } from '@testing-library/react' import userEvent from '@testing-library/user-event' import { FluentProvider, webLightTheme } from '@fluentui/react-components' +import { MemoryRouter } from 'react-router' import { scenariosApi } from '@/services/api' -import type { PairedOutcomeStatistics, ScenarioRunPlan } from '@/types' +import type { PairedOutcomeDetail, PairedOutcomeDetailPage, PairedOutcomeStatistics, ScenarioRunPlan } from '@/types' import PairedOutcomes from './PairedOutcomes' -jest.mock('@/services/api', () => ({ scenariosApi: { getPairedOutcomes: jest.fn() } })) +jest.mock('@/services/api', () => ({ scenariosApi: { getPairedOutcomes: jest.fn(), getPairedOutcomeDetails: jest.fn() } })) const request = scenariosApi.getPairedOutcomes as jest.Mock +const detailsRequest = scenariosApi.getPairedOutcomeDetails as jest.Mock +const runId = '00000000-0000-0000-0000-00000000238c' const plan: ScenarioRunPlan = { version: 1, seed_groups: [], atomic_groups: [ { id: 'base', kind: 'baseline', atomic_attack_name: 'baseline', display_group: 'Baseline', technique_eval_hash: 'b', seed_group_ids: [], tags: [] }, @@ -22,8 +25,15 @@ const report: PairedOutcomeStatistics = { baseline_errors: 0, attack_errors: 1, baseline_undetermined: 0, attack_undetermined: 0, unmatched_baseline_seeds: 1, unmatched_attack_seeds: 0, unattributed_attempts: 0, } -function mount(props: Partial> = {}) { - return render() +function mount(props: Partial> = {}, reactStrictMode: boolean = false) { + return render( + + + + + , + { reactStrictMode }, + ) } async function compare() { const user = userEvent.setup() @@ -32,7 +42,42 @@ async function compare() { await user.click(screen.getByRole('button', { name: 'Compare outcomes' })) return user } -beforeEach(() => { request.mockReset(); request.mockResolvedValue(report) }) +const pair: PairedOutcomeDetail = { + seed_group_id: 'seed-1', objective: 'Ask a clear question', + seed_prompts: [{ sequence: 1, value: 'Example prompt', data_type: 'text', role: 'user', parameters: [] }], + baseline: { + outcome: 'failure', attack_result_id: 'baseline-result', conversation_id: 'baseline-conversation', + outcome_reason: 'Baseline failed', timestamp: '2026-10-20T00:00:00Z', + }, + attack: { + outcome: 'success', attack_result_id: 'attack-result', conversation_id: 'attack-conversation', + outcome_reason: null, timestamp: '2026-10-20T00:01:00Z', + }, +} +function detailPage(pairs: PairedOutcomeDetail[] = [pair], next_cursor: string | null = null, + total_pairs = 1): PairedOutcomeDetailPage { + return { + baseline_group_id: 'base', attack_group_id: 'attack', transition: 'failure_to_success', + pairs, next_cursor, total_pairs, + } +} +beforeEach(() => { + request.mockReset(); request.mockResolvedValue(report) + detailsRequest.mockReset(); detailsRequest.mockResolvedValue(detailPage()) +}) + +test('ignores the cancelled Strict Mode request without duplicating pair details', async () => { + let resolve!: (value: PairedOutcomeDetailPage) => void + detailsRequest.mockReturnValueOnce(new Promise((done) => { resolve = done })) + mount({}, true) + const user = await compare() + await user.click(await screen.findByRole('button', { name: 'Failure → Success' })) + expect(await screen.findByText('1 of 1 pairs shown')).toBeInTheDocument() + expect((detailsRequest.mock.calls[0][2] as AbortSignal).aborted).toBe(true) + await act(async () => resolve(detailPage([{ ...pair, objective: 'Stale cancelled objective' }]))) + expect(screen.queryByText('Stale cancelled objective')).not.toBeInTheDocument() + expect(screen.getAllByText('Logical seed: seed-1')).toHaveLength(1) +}) test('requires explicit groups and shows backend transitions, paired rates and exclusions', async () => { mount() @@ -48,7 +93,136 @@ test('requires explicit groups and shows backend transitions, paired rates and e } expect(within(results).getByRole('group', { name: 'Excluded pairs' })).toHaveTextContent('2') expect(within(results).getByText(/not establish causation/)).toBeInTheDocument() - expect(request).toHaveBeenCalledWith('run', { baseline_group_id: 'base', attack_group_id: 'attack' }, expect.any(AbortSignal)) + expect(request).toHaveBeenCalledWith(runId, { baseline_group_id: 'base', attack_group_id: 'attack' }, expect.any(AbortSignal)) + expect(detailsRequest).not.toHaveBeenCalled() +}) +test('expands on keyboard, lazily retrieves the selected transition, and links both conversations in Scenario context', async () => { + mount() + const user = await compare() + const expand = await screen.findByRole('button', { name: 'Failure → Success' }) + expect(expand).toHaveAttribute('aria-expanded', 'false') + expect(detailsRequest).not.toHaveBeenCalled() + expand.focus() + await user.keyboard('{Enter}') + expect(expand).toHaveAttribute('aria-expanded', 'true') + const details = await screen.findByRole('region', { name: 'Failure → Success pair details' }) + expect(within(details).getByText('Ask a clear question')).toBeInTheDocument() + expect(within(details).getByText('Logical seed: seed-1')).toBeInTheDocument() + expect(within(details).getByText('Example prompt')).toBeInTheDocument() + expect(within(details).getByText('Outcome: failure')).toBeInTheDocument() + expect(within(details).getByText('Outcome: success')).toBeInTheDocument() + expect(within(details).getByText('Baseline failed')).toBeInTheDocument() + expect(within(details).getByRole('link', { name: 'Open baseline conversation' })) + .toHaveAttribute('href', `/attacks/baseline-result/conversations/baseline-conversation?scenarioResultId=${runId}`) + expect(within(details).getByRole('link', { name: 'Open attack conversation' })) + .toHaveAttribute('href', `/attacks/attack-result/conversations/attack-conversation?scenarioResultId=${runId}`) + expect(detailsRequest).toHaveBeenCalledWith(runId, { + baseline_group_id: 'base', attack_group_id: 'attack', transition: 'failure_to_success', limit: 25, + }, expect.any(AbortSignal)) + const firstSignal = detailsRequest.mock.calls[0][2] as AbortSignal + await user.click(expand) + expect(expand).toHaveAttribute('aria-expanded', 'false') + expect(screen.queryByRole('region', { name: 'Failure → Success pair details' })).not.toBeInTheDocument() + expect(firstSignal.aborted).toBe(true) + await user.click(expand) + expect(detailsRequest).toHaveBeenCalledTimes(2) + expect(await screen.findByText('Ask a clear question')).toBeInTheDocument() +}) + +test('paginates with bounded requests, stable cursors and no duplicate pairs', async () => { + const firstPage = Array.from({ length: 25 }, (_, index) => ({ + ...pair, seed_group_id: `seed-${String(index + 1).padStart(2, '0')}`, + })) + const secondPage = Array.from({ length: 6 }, (_, index) => ({ + ...pair, seed_group_id: `seed-${index + 26}`, + })) + request.mockResolvedValue({ ...report, failure_to_success: 31, decided_pairs: 34 }) + detailsRequest.mockResolvedValueOnce(detailPage(firstPage, 'opaque-page-1', 31)) + .mockResolvedValueOnce(detailPage(secondPage, null, 31)) + mount() + const user = await compare() + await user.click(await screen.findByRole('button', { name: 'Failure → Success' })) + expect(await screen.findByText('25 of 31 pairs shown')).toBeInTheDocument() + await user.click(screen.getByRole('button', { name: 'Load more Failure → Success pairs' })) + expect(await screen.findByText('31 of 31 pairs shown')).toBeInTheDocument() + expect(screen.getAllByRole('article')[25]).toHaveFocus() + expect(screen.getAllByText('Logical seed: seed-01')).toHaveLength(1) + expect(screen.queryByRole('button', { name: 'Load more Failure → Success pairs' })).not.toBeInTheDocument() + expect(detailsRequest).toHaveBeenNthCalledWith(2, runId, { + baseline_group_id: 'base', attack_group_id: 'attack', transition: 'failure_to_success', + cursor: 'opaque-page-1', limit: 25, + }, expect.any(AbortSignal)) +}) + +test('retries a failed first page and a failed additional page without losing loaded pairs', async () => { + request.mockResolvedValue({ ...report, failure_to_success: 2, decided_pairs: 5 }) + detailsRequest.mockRejectedValueOnce(new Error('Connection lost')) + .mockResolvedValueOnce(detailPage([pair], 'page-2', 2)) + .mockRejectedValueOnce(new Error('Second page unavailable')) + .mockResolvedValueOnce(detailPage([{ ...pair, seed_group_id: 'seed-2' }], null, 2)) + mount() + const user = await compare() + await user.click(await screen.findByRole('button', { name: 'Failure → Success' })) + expect(await screen.findByText('Connection lost')).toBeInTheDocument() + await user.click(screen.getByRole('button', { name: 'Retry Failure → Success pairs' })) + expect(await screen.findByText('1 of 2 pairs shown')).toBeInTheDocument() + await user.click(screen.getByRole('button', { name: 'Load more Failure → Success pairs' })) + expect(await screen.findByText('Second page unavailable')).toBeInTheDocument() + expect(screen.getByText('Logical seed: seed-1')).toBeInTheDocument() + await user.click(screen.getByRole('button', { name: 'Retry Failure → Success pairs' })) + expect(await screen.findByText('2 of 2 pairs shown')).toBeInTheDocument() + expect(screen.getAllByText('Logical seed: seed-1')).toHaveLength(1) + expect(detailsRequest.mock.calls[2][1]).toMatchObject({ cursor: 'page-2' }) + expect(detailsRequest.mock.calls[3][1]).toMatchObject({ cursor: 'page-2' }) +}) + +test('zero-count transitions show empty state without making a detail request', async () => { + request.mockResolvedValue({ ...report, failure_to_success: 0, decided_pairs: 3 }) + mount() + const user = await compare() + await user.click(await screen.findByRole('button', { name: 'Failure → Success' })) + expect(screen.getByText('No matching pairs in this transition.')).toBeInTheDocument() + expect(detailsRequest).not.toHaveBeenCalled() +}) + +test('collapse cancels requests and ignores late responses, then re-expansion starts fresh', async () => { + let resolve!: (value: PairedOutcomeDetailPage) => void + detailsRequest.mockReturnValueOnce(new Promise((done) => { resolve = done })) + mount() + const user = await compare() + const expand = await screen.findByRole('button', { name: 'Failure → Success' }) + await user.click(expand) + const signal = detailsRequest.mock.calls[0][2] as AbortSignal + await user.click(expand) + expect(signal.aborted).toBe(true) + await act(async () => resolve(detailPage())) + expect(screen.queryByText('Ask a clear question')).not.toBeInTheDocument() + await user.click(expand) + expect(detailsRequest).toHaveBeenCalledTimes(2) + expect(await screen.findByText('Ask a clear question')).toBeInTheDocument() +}) + +test('selection changes cancel detail requests and suppress stale pairs', async () => { + let resolve!: (value: PairedOutcomeDetailPage) => void + detailsRequest.mockReturnValueOnce(new Promise((done) => { resolve = done })) + mount() + const user = await compare() + await user.click(await screen.findByRole('button', { name: 'Failure → Success' })) + const signal = detailsRequest.mock.calls[0][2] as AbortSignal + await user.selectOptions(screen.getByLabelText('Attack group'), 'other') + expect(signal.aborted).toBe(true) + await act(async () => resolve(detailPage())) + expect(screen.queryByRole('region', { name: 'Paired comparison' })).not.toBeInTheDocument() + expect(screen.queryByText('Ask a clear question')).not.toBeInTheDocument() +}) + +test('rejects an inconsistent detail count instead of presenting mismatched observations', async () => { + detailsRequest.mockResolvedValueOnce(detailPage([pair], null, 99)) + mount() + const user = await compare() + await user.click(await screen.findByRole('button', { name: 'Failure → Success' })) + expect(await screen.findByText(/Pair details changed/)).toBeInTheDocument() + expect(screen.queryByText('Ask a clear question')).not.toBeInTheDocument() }) test('shows unavailable rates when there are no decided pairs', async () => { request.mockResolvedValue({ ...report, decided_pairs: 0, excluded_pairs: 6, @@ -99,3 +273,55 @@ test('cancels requests on unmount', async () => { view.unmount() expect(signal.aborted).toBe(true) }) + +test('links a compound outcome to its linked child conversation in Scenario context', async () => { + const compoundPair = { + ...pair, + attack: { + ...pair.attack, + conversation_id: '', + child_conversation: { + attack_result_id: 'linked-child', conversation_id: 'child-conversation', outcome: 'success' as const, + }, + }, + } + detailsRequest.mockResolvedValueOnce(detailPage([compoundPair])) + mount() + const user = await compare() + await user.click(await screen.findByRole('button', { name: 'Failure → Success' })) + const link = await screen.findByRole('link', { name: 'Open attack child conversation' }) + expect(link).toHaveAttribute('href', `/attacks/linked-child/conversations/child-conversation?scenarioResultId=${runId}`) + expect(screen.getByText('Linked child outcome: success')).toBeInTheDocument() + expect(screen.getByText('1 of 1 pairs shown')).toBeInTheDocument() +}) + +test('rejects a truncated final page instead of silently hiding remaining pairs', async () => { + request.mockResolvedValue({ ...report, failure_to_success: 2 }) + detailsRequest.mockResolvedValueOnce(detailPage([pair], null, 2)) + mount() + const user = await compare() + await user.click(await screen.findByRole('button', { name: 'Failure → Success' })) + expect(await screen.findByText(/could not be paged consistently/)).toBeInTheDocument() + expect(screen.queryByText('1 of 2 pairs shown')).not.toBeInTheDocument() +}) + +test('renders multimodal prompt pieces sharing a sequence without duplicate React keys', async () => { + const consoleError = jest.spyOn(console, 'error') + detailsRequest.mockResolvedValueOnce(detailPage([{ + ...pair, + seed_prompts: [ + { sequence: 0, value: 'Text piece', role: 'user', data_type: 'text', parameters: [] }, + { sequence: 0, value: 'Image piece', role: 'user', data_type: 'image_path', parameters: [] }, + ], + }])) + try { + mount() + const user = await compare() + await user.click(await screen.findByRole('button', { name: 'Failure → Success' })) + expect(await screen.findByText('Text piece')).toBeInTheDocument() + expect(screen.getByText('Image piece')).toBeInTheDocument() + expect(consoleError).not.toHaveBeenCalled() + } finally { + consoleError.mockRestore() + } +}) diff --git a/frontend/src/components/Scenarios/PairedOutcomes.tsx b/frontend/src/components/Scenarios/PairedOutcomes.tsx index 96236cfe48..f9ee8b1a92 100644 --- a/frontend/src/components/Scenarios/PairedOutcomes.tsx +++ b/frontend/src/components/Scenarios/PairedOutcomes.tsx @@ -1,4 +1,5 @@ -import { useEffect, useId, useState } from 'react' +import { useEffect, useId, useRef, useState } from 'react' +import { Link } from 'react-router' import { Button, @@ -15,10 +16,19 @@ import { TableRow, Text, } from '@fluentui/react-components' +import { ChevronDownRegular, ChevronRightRegular } from '@fluentui/react-icons' import { scenariosApi } from '@/services/api' import { toApiError } from '@/services/errors' -import type { PairedOutcomeStatistics, ScenarioRunPlan, ScenarioRunState } from '@/types' +import type { + PairedOutcomeDetail, + PairedOutcomeDetailSide, + PairedOutcomeStatistics, + PairedOutcomeTransition, + ScenarioRunPlan, + ScenarioRunState, +} from '@/types' +import { attackConversationRoutePath } from '@/utils/routeParams' import { usePairedOutcomesStyles } from './PairedOutcomes.styles' @@ -34,6 +44,15 @@ interface ComparisonProps { readonly attackId: string } +const DETAIL_PAGE_SIZE = 25 + +const TRANSITIONS: ReadonlyArray<{ label: string; id: PairedOutcomeTransition }> = [ + { label: 'Failure → Success', id: 'failure_to_success' }, + { label: 'Success → Failure', id: 'success_to_failure' }, + { label: 'Failure → Failure', id: 'failure_to_failure' }, + { label: 'Success → Success', id: 'success_to_success' }, +] + /** Explicitly select two planned groups; the backend owns all matching and counting. */ export default function PairedOutcomes({ scenarioResultId, status, plan }: PairedOutcomesProps) { const styles = usePairedOutcomesStyles() @@ -144,19 +163,15 @@ function Comparison({ scenarioResultId, baselineId, attackId }: ComparisonProps) if (state.status === 'loading') return if (state.status === 'error') { - return {state.error} + return + {state.error} + } const report = state.report const difference = report.paired_difference === null ? 'Unavailable' : `${report.paired_difference > 0 ? '+' : ''}${(report.paired_difference * 100).toFixed(1)} pp` - const transitions = [ - ['Failure → Success', report.failure_to_success], - ['Success → Failure', report.success_to_failure], - ['Failure → Failure', report.failure_to_failure], - ['Success → Success', report.success_to_success], - ] as const const exclusions = [ ['Missing baseline', report.missing_baseline], ['Missing attack', report.missing_attack], @@ -183,19 +198,24 @@ function Comparison({ scenarioResultId, baselineId, attackId }: ComparisonProps) Rates use the same decided pairs. Difference = attack − baseline, in percentage points (pp). - +
Baseline → Attack - Pairs + Pairs - {transitions.map(([label, count]) => ( - - {label} - {count.toLocaleString()} - + {TRANSITIONS.map(({ label, id }) => ( + ))}
@@ -215,6 +235,209 @@ function Comparison({ scenarioResultId, baselineId, attackId }: ComparisonProps) ) } +interface TransitionRowProps extends ComparisonProps { + readonly label: string + readonly transition: PairedOutcomeTransition + readonly count: number +} + +function TransitionRow({ label, transition, count, scenarioResultId, baselineId, attackId }: TransitionRowProps) { + const styles = usePairedOutcomesStyles() + const detailsId = useId() + const [expanded, setExpanded] = useState(false) + + return ( + <> + + + + + {count.toLocaleString()} + + {expanded && ( + + +
+ +
+
+
+ )} + + ) +} + +interface DetailState { + pairs: PairedOutcomeDetail[] + pageStart: number + nextCursor: string | null + status: 'loading' | 'ready' | 'error' + error?: string +} + +function TransitionDetails({ label, transition, count, scenarioResultId, baselineId, attackId }: TransitionRowProps) { + const styles = usePairedOutcomesStyles() + const [state, setState] = useState({ pairs: [], pageStart: 0, nextCursor: null, status: 'loading' }) + const [request, setRequest] = useState<{ cursor?: string; version: number }>({ version: 0 }) + const detailsRegion = useRef(null) + const firstNewPair = useRef(null) + + useEffect(() => { + if (count === 0) return + const controller = new AbortController() + void scenariosApi.getPairedOutcomeDetails(scenarioResultId, { + baseline_group_id: baselineId, + attack_group_id: attackId, + transition, + limit: DETAIL_PAGE_SIZE, + ...(request.cursor ? { cursor: request.cursor } : {}), + }, controller.signal).then((page) => { + if (controller.signal.aborted) return + setState((previous) => { + if (page.total_pairs !== count || page.baseline_group_id !== baselineId + || page.attack_group_id !== attackId || page.transition !== transition + || (page.next_cursor && page.next_cursor === request.cursor)) { + return { ...previous, status: 'error', error: 'Pair details changed. Compare outcomes again to refresh.' } + } + const pairs = [...previous.pairs, ...page.pairs] + if (pairs.length > count || new Set(pairs.map((pair) => pair.seed_group_id)).size !== pairs.length + || (page.next_cursor === null && pairs.length !== count) + || (page.next_cursor !== null && (page.pairs.length === 0 || pairs.length === count))) { + return { ...previous, status: 'error', error: 'Pair details could not be paged consistently. Compare outcomes again.' } + } + return { pairs, pageStart: previous.pairs.length, nextCursor: page.next_cursor, status: 'ready' } + }) + }).catch((error: unknown) => { + if (!controller.signal.aborted) { + setState((previous) => ({ ...previous, status: 'error', error: toApiError(error).detail })) + } + }) + return () => controller.abort() + }, [scenarioResultId, baselineId, attackId, transition, count, request]) + + useEffect(() => { + if (state.status === 'ready' && document.activeElement === detailsRegion.current) { + // Restore focus lost with the loading button, without interrupting navigation elsewhere. + firstNewPair.current?.focus() + } + }, [state.status, state.pairs]) + + const fetchPage = (cursor?: string) => { + if (document.activeElement instanceof HTMLButtonElement) detailsRegion.current?.focus({ preventScroll: true }) + setState((previous) => ({ ...previous, status: 'loading', error: undefined })) + setRequest((previous) => ({ cursor, version: previous.version + 1 })) + } + + return ( +
+ {count === 0 && No matching pairs in this transition.} + {state.pairs.map((pair, position) => ( +
+
+ Objective + {pair.objective || 'Objective unavailable'} + Logical seed: {pair.seed_group_id} +
+ {pair.seed_prompts.length > 0 && ( +
+ Seed prompts + {/* Multimodal pieces share sequences; positions are fixed in this saved prompt list. */} + {pair.seed_prompts.map((prompt, position) => ( +
+ + {`#${prompt.sequence}${prompt.role ? ` · ${prompt.role}` : ''}${prompt.data_type ? ` · ${prompt.data_type}` : ''}`} + + {prompt.value} +
+ ))} +
+ )} +
+ + +
+
+ ))} + {state.status === 'loading' && count > 0 && ( + + )} + {state.status === 'error' && ( +
+ + {state.error} + + +
+ )} + {state.status === 'ready' && ( +
+ {`${state.pairs.length} of ${count.toLocaleString()} pairs shown`} + {state.nextCursor && ( + + )} +
+ )} +
+ ) +} + +function OutcomeSide({ label, side, scenarioResultId }: { + label: string + side: PairedOutcomeDetailSide + scenarioResultId: string +}) { + const styles = usePairedOutcomesStyles() + return ( +
+ {label} + {`Outcome: ${side.outcome}`} + {side.outcome_reason && {side.outcome_reason}} + {new Date(side.timestamp).toLocaleString()} + {side.attack_result_id && side.conversation_id ? ( + + {`Open ${label.toLowerCase()} conversation`} + + ) : side.child_conversation ? ( + <> + {`Linked child outcome: ${side.child_conversation.outcome}`} + + {`Open ${label.toLowerCase()} child conversation`} + + + ) : Conversation unavailable} +
+ ) +} + function rate(value: number | null): string { return value === null ? 'Unavailable' : `${(value * 100).toFixed(1)}%` } diff --git a/frontend/src/components/Scenarios/ScenarioDetail.test.tsx b/frontend/src/components/Scenarios/ScenarioDetail.test.tsx index 75ba749ab8..2421e4e599 100644 --- a/frontend/src/components/Scenarios/ScenarioDetail.test.tsx +++ b/frontend/src/components/Scenarios/ScenarioDetail.test.tsx @@ -1734,7 +1734,7 @@ describe('ScenarioDetail', () => { expect(within(preview).getByRole('alert')).toHaveTextContent('prevent older runs from resuming') await user.click(within(preview).getByRole('button', { name: 'Cancel' })) expect(mockStartRun).not.toHaveBeenCalled() - await user.click(screen.getByRole('button', { name: 'Launch scan' })) + await user.click(await screen.findByRole('button', { name: 'Launch scan' })) const confirmed = await screen.findByRole('dialog', { name: 'Run preview' }) await user.click(within(confirmed).getByRole('button', { name: 'Launch scan' })) expect(mockStartRun).toHaveBeenCalledTimes(1) diff --git a/frontend/src/services/api.ts b/frontend/src/services/api.ts index 2bd21495cd..9adffe0b5a 100644 --- a/frontend/src/services/api.ts +++ b/frontend/src/services/api.ts @@ -54,6 +54,8 @@ import type { ScenarioRunListResponse, ScenarioRunProgress, PairedOutcomeStatistics, + PairedOutcomeDetailPage, + PairedOutcomeTransition, ScenarioQueueSnapshot, ScenarioRunState, ConfigurationFileContent, @@ -695,6 +697,24 @@ export const scenariosApi = { return response.data }, + getPairedOutcomeDetails: async ( + scenarioResultId: string, + params: { + baseline_group_id: string + attack_group_id: string + transition: PairedOutcomeTransition + cursor?: string + limit: number + }, + signal?: AbortSignal, + ): Promise => { + const response = await apiClient.get( + `/scenarios/runs/${encodeURIComponent(scenarioResultId)}/paired-outcomes/details`, + { params, signal }, + ) + return response.data + }, + getQueue: async (signal?: AbortSignal): Promise => { const response = await apiClient.get('/scenarios/runs/queue', { signal }) return response.data diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts index 2946cb8bf5..a4cbc8d1c1 100644 --- a/frontend/src/types/index.ts +++ b/frontend/src/types/index.ts @@ -1686,6 +1686,43 @@ export interface PairedOutcomeStatistics { unattributed_attempts: number } +/** Detail pages use the aggregate's exact pairing, exclusions, and latest top-level outcome. */ +export type PairedOutcomeTransition = + | 'failure_to_success' + | 'success_to_failure' + | 'failure_to_failure' + | 'success_to_success' + +export interface PairedOutcomeDetailSide { + outcome: AttackOutcome + attack_result_id: string + conversation_id: string + timestamp: string + outcome_reason: string | null + child_conversation?: { + attack_result_id: string + conversation_id: string + outcome: AttackOutcome + } | null +} + +export interface PairedOutcomeDetail { + seed_group_id: string + objective: string + seed_prompts: ScenarioRunPlanSeedPrompt[] + baseline: PairedOutcomeDetailSide + attack: PairedOutcomeDetailSide +} + +export interface PairedOutcomeDetailPage { + baseline_group_id: string + attack_group_id: string + transition: PairedOutcomeTransition + pairs: PairedOutcomeDetail[] + total_pairs: number + next_cursor: string | null +} + export interface ScenarioRunPlanAtomicGroup { kind?: 'baseline' | 'attack' | 'adaptive' | 'unknown' | null id: string diff --git a/pyrit/analytics/__init__.py b/pyrit/analytics/__init__.py index 89a5ec08be..6b33af413d 100644 --- a/pyrit/analytics/__init__.py +++ b/pyrit/analytics/__init__.py @@ -12,7 +12,16 @@ from pyrit.analytics.attack_result_analytics import AttackResultAnalytics from pyrit.analytics.conversation_analytics import ConversationAnalytics from pyrit.analytics.outcome_statistics import combine_outcome_statistics, compute_outcome_statistics - from pyrit.analytics.paired_outcome_statistics import PairedOutcomeStatistics, compute_paired_outcome_statistics + from pyrit.analytics.paired_outcome_statistics import ( + PairedOutcomeChildConversation, + PairedOutcomeDetail, + PairedOutcomeDetailPage, + PairedOutcomeDetailSide, + PairedOutcomeStatistics, + PairedOutcomeTransition, + compute_paired_outcome_statistics, + get_paired_outcome_details, + ) from pyrit.analytics.result_analysis import ( AttackStats, analyze_results, @@ -35,12 +44,18 @@ "combine_outcome_statistics": "pyrit.analytics.outcome_statistics", "compute_outcome_statistics": "pyrit.analytics.outcome_statistics", "compute_paired_outcome_statistics": "pyrit.analytics.paired_outcome_statistics", + "get_paired_outcome_details": "pyrit.analytics.paired_outcome_statistics", "compute_producer_counts": "pyrit.analytics.scenario_statistics", "compute_scenario_statistics": "pyrit.analytics.scenario_statistics", "compute_technique_stats_async": "pyrit.analytics.technique_analysis", "count_execution_units": "pyrit.analytics.scenario_statistics", "ConversationAnalytics": "pyrit.analytics.conversation_analytics", "PairedOutcomeStatistics": "pyrit.analytics.paired_outcome_statistics", + "PairedOutcomeChildConversation": "pyrit.analytics.paired_outcome_statistics", + "PairedOutcomeDetail": "pyrit.analytics.paired_outcome_statistics", + "PairedOutcomeDetailPage": "pyrit.analytics.paired_outcome_statistics", + "PairedOutcomeDetailSide": "pyrit.analytics.paired_outcome_statistics", + "PairedOutcomeTransition": "pyrit.analytics.paired_outcome_statistics", "ExactTextMatching": "pyrit.analytics.text_matching", "get_cached_results_for_technique": "pyrit.analytics.result_analysis", "get_cached_results_for_technique_async": "pyrit.analytics.result_analysis", diff --git a/pyrit/analytics/paired_outcome_statistics.py b/pyrit/analytics/paired_outcome_statistics.py index 4291259736..f152644239 100644 --- a/pyrit/analytics/paired_outcome_statistics.py +++ b/pyrit/analytics/paired_outcome_statistics.py @@ -5,8 +5,13 @@ from __future__ import annotations +import base64 +import binascii +import hashlib +import json from dataclasses import dataclass from datetime import datetime +from enum import Enum from typing import TYPE_CHECKING from pyrit.analytics.scenario_statistics import ( @@ -15,7 +20,7 @@ resolve_attack_result_attempt, ) from pyrit.common.utils import to_sha256 -from pyrit.models import AttackOutcome, ScenarioExecutionUnit, ScenarioRunPlanGroupKind +from pyrit.models import AttackOutcome, ScenarioExecutionUnit, ScenarioRunPlanGroupKind, ScenarioRunPlanSeedPrompt if TYPE_CHECKING: from collections.abc import Iterator @@ -60,6 +65,244 @@ class PairedOutcomeStatistics: unattributed_attempts: int +class PairedOutcomeTransition(str, Enum): + """One decided baseline-to-attack outcome transition.""" + + FAILURE_TO_SUCCESS = "failure_to_success" + SUCCESS_TO_FAILURE = "success_to_failure" + FAILURE_TO_FAILURE = "failure_to_failure" + SUCCESS_TO_SUCCESS = "success_to_success" + + +_TRANSITION_OUTCOMES = { + PairedOutcomeTransition.FAILURE_TO_SUCCESS: (AttackOutcome.FAILURE, AttackOutcome.SUCCESS), + PairedOutcomeTransition.SUCCESS_TO_FAILURE: (AttackOutcome.SUCCESS, AttackOutcome.FAILURE), + PairedOutcomeTransition.FAILURE_TO_FAILURE: (AttackOutcome.FAILURE, AttackOutcome.FAILURE), + PairedOutcomeTransition.SUCCESS_TO_SUCCESS: (AttackOutcome.SUCCESS, AttackOutcome.SUCCESS), +} + + +@dataclass(frozen=True, slots=True) +class PairedOutcomeChildConversation: + """One linked child conversation for a compound completion without its own conversation.""" + + attack_result_id: str + conversation_id: str + outcome: AttackOutcome + + +@dataclass(frozen=True, slots=True) +class PairedOutcomeDetailSide: + """The effective top-level completion for one side of a logical pair.""" + + outcome: AttackOutcome + attack_result_id: str + conversation_id: str + timestamp: datetime + outcome_reason: str | None + child_conversation: PairedOutcomeChildConversation | None = None + + +@dataclass(frozen=True, slots=True) +class PairedOutcomeDetail: + """A shared logical seed and the effective outcomes for both selected groups.""" + + seed_group_id: str + objective: str + seed_prompts: tuple[ScenarioRunPlanSeedPrompt, ...] + baseline: PairedOutcomeDetailSide + attack: PairedOutcomeDetailSide + + +@dataclass(frozen=True, slots=True) +class PairedOutcomeDetailPage: + """A bounded, seed-ID-ordered page of matching decided pairs.""" + + baseline_group_id: str + attack_group_id: str + transition: PairedOutcomeTransition + pairs: tuple[PairedOutcomeDetail, ...] + total_pairs: int + next_cursor: str | None + + +def _detail_side(result: AttackResult, *, attempts_by_id: dict[str, AttackResult]) -> PairedOutcomeDetailSide: + """ + Copy only the effective top-level outcome context needed by the GUI. + + Returns: + PairedOutcomeDetailSide: Selected top-level result context. + """ + child_conversation = None + if not result.conversation_id: + fallback = matching = None + for linked in _linked_outcome_results(result, attempts_by_id=attempts_by_id): + if linked.conversation_id: + fallback = linked + if linked.outcome == result.outcome: + matching = linked + selected = matching or fallback + if selected is not None: + child_conversation = PairedOutcomeChildConversation( + attack_result_id=selected.attack_result_id, + conversation_id=selected.conversation_id, + outcome=selected.outcome, + ) + return PairedOutcomeDetailSide( + outcome=result.outcome, + attack_result_id=result.attack_result_id, + conversation_id=result.conversation_id, + timestamp=result.timestamp, + outcome_reason=result.outcome_reason, + child_conversation=child_conversation, + ) + + +def _encode_detail_cursor( + *, run_id: str, baseline_group_id: str, attack_group_id: str, transition: PairedOutcomeTransition, seed_id: str +) -> str: + """ + Encode a stable keyset position scoped to this run and comparison. + + Returns: + str: Portable, canonical cursor that can be decoded by any worker. + """ + payload = { + "v": 1, + "scope": _detail_cursor_scope( + run_id=run_id, + baseline_group_id=baseline_group_id, + attack_group_id=attack_group_id, + transition=transition, + ), + "after": to_sha256(seed_id), + } + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + return base64.urlsafe_b64encode(encoded).decode().rstrip("=") + + +def _detail_cursor_scope( + *, run_id: str, baseline_group_id: str, attack_group_id: str, transition: PairedOutcomeTransition +) -> str: + """ + Bind a cursor to a particular saved run and ordered group selection. + + Returns: + str: Stable digest of the run, selected groups, and transition. + """ + payload = json.dumps([run_id, baseline_group_id, attack_group_id, transition.value], separators=(",", ":")) + return hashlib.sha256(payload.encode()).hexdigest() + + +def _decode_detail_cursor( + cursor: str, *, run_id: str, baseline_group_id: str, attack_group_id: str, transition: PairedOutcomeTransition +) -> str: + """ + Reject malformed, mismatched, or noncanonical pagination cursors. + + Returns: + str: Validated last seed digest; membership is verified against current matches. + + Raises: + ValueError: If the cursor is malformed or belongs to another selection. + """ + try: + if not cursor or len(cursor) > 1024: + raise ValueError + raw = base64.b64decode(cursor + "=" * (-len(cursor) % 4), altchars=b"-_", validate=True) + if base64.urlsafe_b64encode(raw).decode().rstrip("=") != cursor: + raise ValueError + payload = json.loads(raw) + if not isinstance(payload, dict) or set(payload) != {"v", "scope", "after"}: + raise ValueError + if ( + type(payload["v"]) is not int + or payload["v"] != 1 + or payload["scope"] + != _detail_cursor_scope( + run_id=run_id, + baseline_group_id=baseline_group_id, + attack_group_id=attack_group_id, + transition=transition, + ) + or not isinstance(payload["after"], str) + or len(payload["after"]) != 64 + or any(character not in "0123456789abcdef" for character in payload["after"]) + ): + raise ValueError + return payload["after"] + except (ValueError, TypeError, UnicodeDecodeError, binascii.Error) as exc: + raise ValueError("Invalid paired-outcome detail cursor for the selected run, groups, and transition.") from exc + + +def get_paired_outcome_details( + scenario_result: ScenarioResult, + *, + baseline_group_id: str, + attack_group_id: str, + transition: PairedOutcomeTransition, + limit: int = 25, + cursor: str | None = None, +) -> PairedOutcomeDetailPage: + """ + Page through one transition using the aggregate's exact pairing and compatibility checks. + + Returns: + PairedOutcomeDetailPage: Deterministically ordered effective top-level pairs. + + Raises: + ValueError: If the selection, evidence, page size, or cursor is invalid. + """ + if not isinstance(transition, PairedOutcomeTransition): + raise ValueError("Unknown paired-outcome transition.") + if isinstance(limit, bool) or not isinstance(limit, int) or not 1 <= limit <= 100: + raise ValueError("Paired-outcome detail limit must be between 1 and 100.") + after = ( + _decode_detail_cursor( + cursor, + run_id=str(scenario_result.id), + baseline_group_id=baseline_group_id, + attack_group_id=attack_group_id, + transition=transition, + ) + if cursor is not None + else None + ) + matches: list[PairedOutcomeDetail] = [] + _compute_paired_outcome_statistics( + scenario_result, + baseline_group_id=baseline_group_id, + attack_group_id=attack_group_id, + selected_transition=transition, + matching_details=matches, + ) + matches.sort(key=lambda item: item.seed_group_id) + ids = [to_sha256(item.seed_group_id) for item in matches] + if after is not None and after not in ids: + raise ValueError("Invalid paired-outcome detail cursor: seed is not in the selected transition.") + start = ids.index(after) + 1 if after is not None else 0 + page = matches[start : start + limit] + next_cursor = ( + _encode_detail_cursor( + run_id=str(scenario_result.id), + baseline_group_id=baseline_group_id, + attack_group_id=attack_group_id, + transition=transition, + seed_id=page[-1].seed_group_id, + ) + if start + limit < len(matches) + else None + ) + return PairedOutcomeDetailPage( + baseline_group_id=baseline_group_id, + attack_group_id=attack_group_id, + transition=transition, + pairs=tuple(page), + total_pairs=len(matches), + next_cursor=next_cursor, + ) + + @dataclass(frozen=True, slots=True) class _ScoringEvidence: """Available scoring metadata for a persisted outcome.""" @@ -260,6 +503,28 @@ def compute_paired_outcome_statistics( ValueError: If the run plan, selection, attribution, objectives, or available scorer identities cannot support an unambiguous comparison. """ + return _compute_paired_outcome_statistics( + scenario_result, baseline_group_id=baseline_group_id, attack_group_id=attack_group_id + ) + + +def _compute_paired_outcome_statistics( + scenario_result: ScenarioResult, + *, + baseline_group_id: str, + attack_group_id: str, + selected_transition: PairedOutcomeTransition | None = None, + matching_details: list[PairedOutcomeDetail] | None = None, +) -> PairedOutcomeStatistics: + """ + Compute aggregates and optionally collect matching details from the same validated pairs. + + Returns: + PairedOutcomeStatistics: Validated aggregate of effective pair outcomes. + + Raises: + ValueError: If the plan, selected results, or score evidence are incompatible. + """ baseline, attack, plan, lookup = _selected_groups( scenario_result=scenario_result, baseline_group_id=baseline_group_id, @@ -272,6 +537,7 @@ def compute_paired_outcome_statistics( # The plan validates unique seed IDs. The same run carries the scenario-level # target/scorer identity; per-result identities add stronger checks. + seed_by_id = {seed.id: seed for seed in plan.seed_groups} objectives = {seed.id: seed.objective_sha256 for seed in plan.seed_groups} relevant_names = {baseline.atomic_attack_name, attack.atomic_attack_name} selected_ids = {baseline.id, attack.id} @@ -387,6 +653,21 @@ def compute_paired_outcome_statistics( raise ValueError("Decided pairs contain incompatible score types.") observed_score_type = score_type transitions[(before[0], after[0])] += 1 + if ( + matching_details is not None + and selected_transition is not None + and _TRANSITION_OUTCOMES[selected_transition] == (before[0], after[0]) + ): + seed = seed_by_id[seed_id] + matching_details.append( + PairedOutcomeDetail( + seed_group_id=seed_id, + objective=seed.objective, + seed_prompts=tuple(seed.prompts), + baseline=_detail_side(before[1], attempts_by_id=before[2]), + attack=_detail_side(after[1], attempts_by_id=after[2]), + ) + ) decided = sum(transitions.values()) baseline_successes = ( diff --git a/pyrit/backend/routes/scenarios.py b/pyrit/backend/routes/scenarios.py index 011a471514..3c00f55ea7 100644 --- a/pyrit/backend/routes/scenarios.py +++ b/pyrit/backend/routes/scenarios.py @@ -17,7 +17,13 @@ from fastapi import APIRouter, HTTPException, Query, Request, status from starlette.concurrency import run_in_threadpool -from pyrit.analytics import PairedOutcomeStatistics, compute_paired_outcome_statistics +from pyrit.analytics import ( + PairedOutcomeDetailPage, + PairedOutcomeStatistics, + PairedOutcomeTransition, + compute_paired_outcome_statistics, + get_paired_outcome_details, +) from pyrit.backend.models.common import MAX_ITEMS, CursorStr, IdentifierStr, LabelFilterStr, ProblemDetail from pyrit.backend.models.datasets import ScenarioDatasetLoadResponse from pyrit.backend.models.scenarios import ( @@ -441,6 +447,45 @@ async def get_scenario_run_results(scenario_result_id: IdentifierStr) -> Scenari return result +@router.get( + "/runs/{scenario_result_id}/paired-outcomes/details", + response_model=PairedOutcomeDetailPage, + responses={ + 400: {"model": ProblemDetail, "description": "Invalid selection, pairing evidence, or detail cursor"}, + 404: {"model": ProblemDetail, "description": "Run not found"}, + 409: {"model": ProblemDetail, "description": "Run not yet completed"}, + }, +) +async def get_paired_outcome_details_async( + *, + scenario_result_id: IdentifierStr, + baseline_group_id: IdentifierStr = Query(..., description="Saved baseline atomic group ID"), + attack_group_id: IdentifierStr = Query(..., description="Saved attack atomic group ID"), + transition: PairedOutcomeTransition = Query(..., description="Decided outcome transition"), + limit: int = Query(25, ge=1, le=100, description="Maximum matching pairs per page"), + cursor: CursorStr | None = Query(None, description="Opaque filter-bound detail cursor"), +) -> PairedOutcomeDetailPage: + """ + Load a bounded page of objective and conversation details for one transition. + + Returns: + PairedOutcomeDetailPage: Pairs selected by the same SDK semantics as aggregate analytics. + """ + result = await get_scenario_run_results(scenario_result_id=scenario_result_id) + try: + return await run_in_threadpool( + get_paired_outcome_details, + result, + baseline_group_id=baseline_group_id, + attack_group_id=attack_group_id, + transition=transition, + limit=limit, + cursor=cursor, + ) + except ValueError as exc: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from None + + @router.get( "/runs/{scenario_result_id}/paired-outcomes", response_model=PairedOutcomeStatistics, diff --git a/tests/unit/analytics/test_paired_outcome_statistics.py b/tests/unit/analytics/test_paired_outcome_statistics.py index 7809e50bf3..889c4d0735 100644 --- a/tests/unit/analytics/test_paired_outcome_statistics.py +++ b/tests/unit/analytics/test_paired_outcome_statistics.py @@ -7,7 +7,11 @@ import pytest -from pyrit.analytics.paired_outcome_statistics import compute_paired_outcome_statistics +from pyrit.analytics.paired_outcome_statistics import ( + PairedOutcomeTransition, + compute_paired_outcome_statistics, + get_paired_outcome_details, +) from pyrit.common.utils import to_sha256 from pyrit.executor.attack import PromptSendingAttack from pyrit.executor.attack.compound import SequenceCompletionPolicy, SequentialAttackResult @@ -19,6 +23,7 @@ AttackResult, AttackSeedGroup, ComponentIdentifier, + ScenarioResult, ScenarioRunPlan, ScenarioRunPlanAtomicGroup, ScenarioRunPlanGroupKind, @@ -900,3 +905,193 @@ def test_paired_outcomes_reject_malformed_child_result_ids() -> None: _plan(("a",), ("a",)), {"baseline": [_result("a", AttackOutcome.FAILURE)], "attack": [result]}, ) + + +def _detail_scenario() -> ScenarioResult: + """Construct pairs with a retry, a child attempt, and an excluded outcome.""" + return make_scenario_result( + metadata={SCENARIO_RUN_PLAN_METADATA_KEY: _plan().model_dump(mode="json")}, + attack_results={ + "baseline": [ + _result("d", AttackOutcome.FAILURE), + _result("c", AttackOutcome.SUCCESS), + _result("b", AttackOutcome.FAILURE), + _result("a", AttackOutcome.FAILURE), + ], + "attack": [ + _result("b", AttackOutcome.SUCCESS), + _result("a", AttackOutcome.FAILURE, seconds=-2), + _result("d", AttackOutcome.ERROR), + _result("a", AttackOutcome.SUCCESS, seconds=1), + _result("a", AttackOutcome.FAILURE, seconds=2, attempt_index=1), + _result("c", AttackOutcome.SUCCESS), + ], + }, + ) + + +def test_paired_outcome_details_paginate_same_decided_pairs_as_summary() -> None: + saved = _detail_scenario() + summary = compute_paired_outcome_statistics(saved, baseline_group_id="base", attack_group_id="attack") + assert (summary.failure_to_success, summary.success_to_success, summary.excluded_pairs) == (2, 1, 1) + + first = get_paired_outcome_details( + saved, + baseline_group_id="base", + attack_group_id="attack", + transition=PairedOutcomeTransition.FAILURE_TO_SUCCESS, + limit=1, + ) + assert first.total_pairs == summary.failure_to_success + assert [row.seed_group_id for row in first.pairs] == ["a"] + assert first.pairs[0].objective == "a" + assert first.pairs[0].attack.outcome == AttackOutcome.SUCCESS + assert first.pairs[0].attack.attack_result_id == saved.attack_results["attack"][3].attack_result_id + assert first.next_cursor is not None + + second = get_paired_outcome_details( + saved, + baseline_group_id="base", + attack_group_id="attack", + transition=PairedOutcomeTransition.FAILURE_TO_SUCCESS, + limit=1, + cursor=first.next_cursor, + ) + assert [row.seed_group_id for row in second.pairs] == ["b"] + assert second.next_cursor is None + saved.attack_results["attack"].reverse() + again = get_paired_outcome_details( + saved, + baseline_group_id="base", + attack_group_id="attack", + transition=PairedOutcomeTransition.FAILURE_TO_SUCCESS, + limit=1, + cursor=first.next_cursor, + ) + assert again == second + + +def test_paired_outcome_details_reject_wrong_scope_and_nonmember_cursors() -> None: + from pyrit.analytics.paired_outcome_statistics import _encode_detail_cursor + + saved = _detail_scenario() + valid = get_paired_outcome_details( + saved, + baseline_group_id="base", + attack_group_id="attack", + transition=PairedOutcomeTransition.FAILURE_TO_SUCCESS, + limit=1, + ).next_cursor + assert valid + with pytest.raises(ValueError, match="cursor"): + get_paired_outcome_details( + saved, + baseline_group_id="base", + attack_group_id="attack", + transition=PairedOutcomeTransition.SUCCESS_TO_SUCCESS, + cursor=valid, + ) + with pytest.raises(ValueError, match="cursor"): + get_paired_outcome_details( + saved, + baseline_group_id="base", + attack_group_id="attack", + transition=PairedOutcomeTransition.FAILURE_TO_SUCCESS, + cursor="!malformed!", + ) + nonmember = _encode_detail_cursor( + run_id=str(saved.id), + baseline_group_id="base", + attack_group_id="attack", + transition=PairedOutcomeTransition.FAILURE_TO_SUCCESS, + seed_id="c", + ) + with pytest.raises(ValueError, match="seed"): + get_paired_outcome_details( + saved, + baseline_group_id="base", + attack_group_id="attack", + transition=PairedOutcomeTransition.FAILURE_TO_SUCCESS, + cursor=nonmember, + ) + + +@pytest.mark.parametrize("limit", [0, -1, 101, True]) +def test_paired_outcome_details_reject_invalid_page_size(limit: int) -> None: + with pytest.raises(ValueError, match="limit"): + get_paired_outcome_details( + _detail_scenario(), + baseline_group_id="base", + attack_group_id="attack", + transition=PairedOutcomeTransition.FAILURE_TO_SUCCESS, + limit=limit, + ) + + +def test_paired_outcome_details_preserve_compatibility_checks() -> None: + saved = _detail_scenario() + saved.attack_results["attack"][0].objective = "different objective" + with pytest.raises(ValueError, match="objective"): + get_paired_outcome_details( + saved, + baseline_group_id="base", + attack_group_id="attack", + transition=PairedOutcomeTransition.FAILURE_TO_SUCCESS, + ) + + +@pytest.mark.parametrize("seed_id", ["a" + "x" * 900, "a" + "雪" * 200], ids=["long-ascii", "long-unicode"]) +def test_paired_outcome_details_can_continue_after_long_seed_ids(seed_id: str) -> None: + seeds = (seed_id, "z") + saved = make_scenario_result( + metadata={SCENARIO_RUN_PLAN_METADATA_KEY: _plan(seeds, seeds).model_dump(mode="json")}, + attack_results={ + "baseline": [_result(seed, AttackOutcome.FAILURE) for seed in seeds], + "attack": [_result(seed, AttackOutcome.SUCCESS) for seed in seeds], + }, + ) + first = get_paired_outcome_details( + saved, + baseline_group_id="base", + attack_group_id="attack", + transition=PairedOutcomeTransition.FAILURE_TO_SUCCESS, + limit=1, + ) + assert first.next_cursor is not None + assert len(first.next_cursor) <= 1024 + second = get_paired_outcome_details( + saved, + baseline_group_id="base", + attack_group_id="attack", + transition=PairedOutcomeTransition.FAILURE_TO_SUCCESS, + limit=1, + cursor=first.next_cursor, + ) + assert [pair.seed_group_id for pair in first.pairs + second.pairs] == [seed_id, "z"] + assert second.next_cursor is None + + +@pytest.mark.parametrize("live", [False, True]) +def test_paired_outcome_details_link_nested_compound_conversation_without_counting_children(live: bool) -> None: + outer, inner, leaf = _nested_compound_outcome() + outer.conversation_id = inner.conversation_id = "" + unrelated = _result("a", AttackOutcome.SUCCESS, seconds=100, group="attack", attempt_index=2) + results = [outer, unrelated] if live else _saved_compound_results([outer, inner, leaf, unrelated]) + saved = make_scenario_result( + metadata={SCENARIO_RUN_PLAN_METADATA_KEY: _plan(("a",), ("a",)).model_dump(mode="json")}, + attack_results={"baseline": [_result("a", AttackOutcome.FAILURE)], "attack": results}, + ) + page = get_paired_outcome_details( + saved, + baseline_group_id="base", + attack_group_id="attack", + transition=PairedOutcomeTransition.FAILURE_TO_SUCCESS, + ) + assert page.total_pairs == len(page.pairs) == 1 + side = page.pairs[0].attack + assert side.attack_result_id == outer.attack_result_id + assert side.outcome == AttackOutcome.SUCCESS + assert side.child_conversation is not None + assert side.child_conversation.attack_result_id == leaf.attack_result_id + assert side.child_conversation.conversation_id == leaf.conversation_id + assert side.child_conversation.outcome == AttackOutcome.SUCCESS diff --git a/tests/unit/backend/test_paired_outcome_routes.py b/tests/unit/backend/test_paired_outcome_routes.py index 1bb605147a..47f1f7a3b9 100644 --- a/tests/unit/backend/test_paired_outcome_routes.py +++ b/tests/unit/backend/test_paired_outcome_routes.py @@ -154,3 +154,53 @@ def test_paired_outcomes_rejects_legacy_plan(client: TestClient) -> None: ) assert response.status_code == 400 assert "saved scenario run plan" in response.json()["detail"] + + +def test_paired_detail_endpoint_returns_selected_seed_and_conversation_ids(client: TestClient) -> None: + saved = _saved_result() + params = {"baseline_group_id": "baseline", "attack_group_id": "attack", "transition": "failure_to_success"} + with patch("pyrit.backend.routes.scenarios.get_scenario_run_service") as service: + service.return_value.get_run_results_async = AsyncMock(return_value=saved) + response = client.get("/api/scenarios/runs/saved/paired-outcomes/details", params=params) + assert response.status_code == 200 + body = response.json() + assert body["total_pairs"] == 1 + assert body["next_cursor"] is None + assert body["pairs"][0]["seed_group_id"] == "a" + assert body["pairs"][0]["objective"] == "a" + assert body["pairs"][0]["baseline"]["conversation_id"] == saved.attack_results["baseline"][0].conversation_id + assert body["pairs"][0]["attack"]["conversation_id"] == saved.attack_results["attack"][0].conversation_id + + +@pytest.mark.parametrize( + ("params", "status_code"), + [ + ({"baseline_group_id": "baseline", "attack_group_id": "attack", "transition": "bad"}, 422), + ( + { + "baseline_group_id": "baseline", + "attack_group_id": "attack", + "transition": "failure_to_success", + "limit": 101, + }, + 422, + ), + ( + { + "baseline_group_id": "baseline", + "attack_group_id": "attack", + "transition": "failure_to_success", + "cursor": "invalid!", + }, + 400, + ), + ({"baseline_group_id": "attack", "attack_group_id": "baseline", "transition": "failure_to_success"}, 400), + ], +) +def test_paired_detail_endpoint_validates_filters_and_cursors( + client: TestClient, params: dict[str, str | int], status_code: int +) -> None: + with patch("pyrit.backend.routes.scenarios.get_scenario_run_service") as service: + service.return_value.get_run_results_async = AsyncMock(return_value=_saved_result()) + response = client.get("/api/scenarios/runs/saved/paired-outcomes/details", params=params) + assert response.status_code == status_code