Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions doc/gui/0_gui.md
Original file line number Diff line number Diff line change
Expand Up @@ -977,6 +977,32 @@ pairs. For example, four pairs with one of each transition give 50% success
on each side and a 0 pp difference, while still showing one failure becoming
a success and one success becoming a failure.

Each transition row can be expanded to inspect the matched logical objectives
behind that count. Expanding loads only the selected transition's pairs, with
the baseline and attack outcomes shown side by side and links to each saved
conversation. Conversation links retain the originating Scenario run context.
When a compound result has no conversation of its own, the panel links to one
of its recorded child conversations and labels the child's outcome separately.
It prefers the last linked conversation matching the compound outcome, falling
back to the last linked conversation when none matches. The displayed pair
outcome and observation count still belong to the top-level result.
Use **Load more** to inspect additional pairs, or collapse the row to hide its
details. Large transitions are loaded in bounded pages; a failed request can
be retried from the expanded row. Switching selected groups clears the details.
All results use the same decided-pair cohort and most recent top-level attempt
as the transition counts, so retries and child attempts are never additional
pairs.

The following focused screenshots show one expanded transition and its conversation links.

Expanded transition details on desktop:

![Paired outcomes with an expanded transition, objective, and baseline and attack conversation links](./images/paired-outcomes-desktop.png)

The same transition details stack vertically on a narrow screen:

![Expanded paired outcomes at a mobile viewport width](./images/paired-outcomes-mobile.png)

**Decided pairs** shows the number with success or failure on both sides,
followed by the number planned on both sides. **Excluded pairs** counts the
remaining shared pairs. Missing outcomes, errors, and undetermined outcomes
Expand Down
Binary file added doc/gui/images/paired-outcomes-desktop.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added doc/gui/images/paired-outcomes-mobile.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
122 changes: 101 additions & 21 deletions frontend/e2e/fixtures/paired_outcomes_backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,15 @@ async def _seed_async(memory: SQLiteMemory) -> None:
AttackIdentifier,
AttackOutcome,
AttackResult,
Conversation,
MessagePiece,
ScenarioIdentifier,
ScenarioResult,
ScenarioRunPlan,
ScenarioRunPlanAtomicGroup,
ScenarioRunPlanGroupKind,
ScenarioRunPlanSeedGroup,
ScenarioRunPlanSeedPrompt,
ScenarioRunState,
TargetIdentifier,
)
Expand All @@ -45,28 +48,51 @@ async def _seed_async(memory: SQLiteMemory) -> None:
objective_target=target,
)
)
plan = ScenarioRunPlan(
atomic_groups=[
ScenarioRunPlanAtomicGroup(
id=name,
atomic_attack_name=name,
display_group=name.title(),
technique_eval_hash=f"{name}-eval",
kind=kind,
seed_group_ids=shared + [extra],
)
for name, kind, extra in [
("baseline", ScenarioRunPlanGroupKind.BASELINE, "base-only"),
("attack", ScenarioRunPlanGroupKind.ATTACK, "attack-only"),
]
],
seed_groups=[
ScenarioRunPlanSeedGroup(id=seed, objective=seed, objective_sha256=to_sha256(seed))
for seed in shared + ["base-only", "attack-only"]
],
)
for index, state in enumerate([ScenarioRunState.COMPLETED, ScenarioRunState.COMPLETED, ScenarioRunState.FAILED]):
for index, state in enumerate(
[
ScenarioRunState.COMPLETED,
ScenarioRunState.COMPLETED,
ScenarioRunState.FAILED,
ScenarioRunState.COMPLETED,
ScenarioRunState.COMPLETED,
]
):
scenario_id = UUID(int=9100 + index)
page_seeds = (
[f"page-{number:02d}" + ("x" * 900 if index == 4 and number == 23 else "") for number in range(30)]
if index in (3, 4)
else []
)
selected_seeds = shared + page_seeds
plan = ScenarioRunPlan(
atomic_groups=[
ScenarioRunPlanAtomicGroup(
id=name,
atomic_attack_name=name,
display_group=name.title(),
technique_eval_hash=f"{name}-eval",
kind=kind,
seed_group_ids=selected_seeds + [extra],
)
for name, kind, extra in [
("baseline", ScenarioRunPlanGroupKind.BASELINE, "base-only"),
("attack", ScenarioRunPlanGroupKind.ATTACK, "attack-only"),
]
],
seed_groups=[
ScenarioRunPlanSeedGroup(
id=seed,
objective=seed,
objective_sha256=to_sha256(seed),
prompts=[
ScenarioRunPlanSeedPrompt(value="LongPrompt" * 80, sequence=0, data_type="text", role="user")
]
if index == 4 and seed == "a"
else [],
)
for seed in selected_seeds + ["base-only", "attack-only"]
],
)
rows: dict[str, list[AttackResult]] = {"baseline": [], "attack": []}
for name, outcomes in [
("baseline", [AttackOutcome.FAILURE, AttackOutcome.SUCCESS, AttackOutcome.FAILURE, AttackOutcome.SUCCESS]),
Expand All @@ -89,6 +115,24 @@ async def _seed_async(memory: SQLiteMemory) -> None:
},
)
)
for seed in page_seeds:
for name, outcome in [("baseline", AttackOutcome.FAILURE), ("attack", AttackOutcome.SUCCESS)]:
rows[name].append(
AttackResult(
attack_result_id=str(uuid4()),
conversation_id=str(uuid4()),
objective=seed,
outcome=outcome,
timestamp=timestamp,
atomic_attack_identifier=identifier,
attribution_parent_id=str(scenario_id),
attribution_data={
"seed_group_id": seed,
"parent_eval_hash": f"{name}-eval",
"parent_collection": name,
},
)
)
for seed, outcome in [
("undetermined", AttackOutcome.UNDETERMINED),
("error", AttackOutcome.ERROR),
Expand Down Expand Up @@ -128,6 +172,42 @@ async def _seed_async(memory: SQLiteMemory) -> None:
},
)
)
if index == 4:
parent = rows["attack"][0]
child = parent.model_copy(deep=True)
child.attack_result_id = str(uuid4())
child.conversation_id = str(uuid4())
assert child.attribution_data is not None
child.attribution_data["attempt_index"] = 1
parent.conversation_id = ""
parent.metadata["child_attack_result_ids"] = [child.attack_result_id]
rows["attack"].append(child)
for attack_result in (item for group in rows.values() for item in group):
if not attack_result.conversation_id:
continue
await memory.add_conversation_to_memory_async(
conversation=Conversation(
conversation_id=attack_result.conversation_id,
target_identifier=target,
attack_result_id=attack_result.attack_result_id,
)
)
await memory.add_message_pieces_to_memory_async(
message_pieces=[
MessagePiece(
role="user",
conversation_id=attack_result.conversation_id,
sequence=0,
original_value=f"Offline objective: {attack_result.objective}",
),
MessagePiece(
role="assistant",
conversation_id=attack_result.conversation_id,
sequence=1,
original_value=f"Offline {attack_result.outcome.value} example; no model was called.",
),
]
)
await memory.add_attack_results_to_memory_async(
attack_results=[row for group in rows.values() for row in group]
)
Expand Down
Loading