Add severity-weighted anytime risk discovery

README.md

@@ -236,6 +236,7 @@ PowerAgentBench returns per-case and aggregate metrics, including:
 - duplicate validation requests,
 - explicit submission and auto-finalization indicators,
 - validation budget use,
+- severity-weighted anytime risk-discovery AUC and discovery at 25%, 50%, 75%, and 100% of the validation budget,
 - completed and requested case counts.
 
 These metrics distinguish answer quality, tool evidence, search quality, mitigation quality, safety behavior, and workflow compliance.

benchmarks/steady/level_2/README.md

@@ -157,6 +157,23 @@ The runner returns per-case and aggregate CSV files with:
 
 These fields separate search quality, evidence quality, tool compliance, budget use, mitigation, and workflow completion.
 
+### Severity-Weighted Anytime Risk Discovery
+
+The evaluator also reports severity-weighted anytime risk-discovery metrics:
+
+- `anytime_risk_auc`,
+- `anytime_risk_at_25`,
+- `anytime_risk_at_50`,
+- `anytime_risk_at_75`,
+- `anytime_risk_at_100`.
+
+These metrics evaluate how quickly an agent discovers hidden dangerous contingency severity as its validation budget is consumed. For a validation fraction \(\alpha\), the risk-discovery fraction is the severity discovered up to that point divided by the total hidden severity of dangerous contingencies. `anytime_risk_auc` is the normalized trapezoidal area under this discovery curve.
+
+The metrics are computed by the hidden evaluator after the agent has completed its validation trajectory. Hidden oracle severity values are therefore not exposed to the agent.
+
+For a fixed validation budget, a higher `anytime_risk_auc` indicates earlier discovery of severity-weighted risk. The `at_*` metrics report discovery at 25%, 50%, 75%, and 100% of the available validation budget. These metrics complement final discovery, evidence, safety, mitigation, and action-cost metrics rather than replacing them.
+
+
 ## Evaluation Regimes
 
 - **Open**: users can inspect public files and debug agents locally.

poweragentbench/steady_state_agentic.py

@@ -965,6 +965,103 @@ def execute(self, tool: str, args: MutableMapping[str, Any] | None) -> Tuple[Dic
         return {"error": f"Unknown tool '{tool}'.", "allowed_tools": ["case_summary", "rank_base_loading", "rank_lodf", "validate", "redispatch", "submit"]}, False, None
 
 
+
+def compute_anytime_risk_metrics(
+    output: AgentOutput,
+    oracle_values: Dict[Contingency, float],
+    dangerous: set[Contingency],
+) -> Dict[str, float]:
+    """Measure risk discovered as the validation budget is consumed.
+
+    The hidden oracle is used only by the evaluator.  The agent sees neither
+    the hidden severity values nor the resulting anytime-risk metrics.
+    """
+    budget = int(output.validation_budget)
+
+    if budget <= 0 or not dangerous:
+        return {
+            "anytime_risk_auc": 0.0,
+            "anytime_risk_at_25": 0.0,
+            "anytime_risk_at_50": 0.0,
+            "anytime_risk_at_75": 0.0,
+            "anytime_risk_at_100": 0.0,
+        }
+
+    total_risk = sum(
+        max(0.0, float(oracle_values.get(c, 0.0)))
+        for c in dangerous
+    )
+
+    if total_risk <= 1e-12:
+        return {
+            "anytime_risk_auc": 0.0,
+            "anytime_risk_at_25": 0.0,
+            "anytime_risk_at_50": 0.0,
+            "anytime_risk_at_75": 0.0,
+            "anytime_risk_at_100": 0.0,
+        }
+
+    # AgentOutput.validated preserves insertion order.  This gives us a
+    # common validation trajectory for both deterministic and LLM agents.
+    validation_order = list(output.validated.keys())
+
+    points = [(0, 0.0)]
+    discovered = set()
+
+    for i, contingency in enumerate(validation_order, start=1):
+        if i > budget:
+            break
+
+        discovered.add(contingency)
+
+        discovered_risk = sum(
+            max(0.0, float(oracle_values.get(c, 0.0)))
+            for c in discovered
+            if c in dangerous
+        )
+
+        risk_fraction = min(1.0, discovered_risk / total_risk)
+        points.append((i, risk_fraction))
+
+    # If the agent used less than the available budget, hold its final
+    # discovery level for the unused portion of the budget.
+    if points[-1][0] < budget:
+        points.append((budget, points[-1][1]))
+
+    def risk_at(target: int) -> float:
+        if target <= 0:
+            return 0.0
+
+        for i in range(1, len(points)):
+            x0, y0 = points[i - 1]
+            x1, y1 = points[i]
+
+            if target <= x1:
+                if x1 == x0:
+                    return float(y1)
+
+                alpha = (target - x0) / float(x1 - x0)
+                return float(y0 + alpha * (y1 - y0))
+
+        return float(points[-1][1])
+
+    # Normalized trapezoidal area under the risk-discovery curve.
+    auc = 0.0
+
+    for (x0, y0), (x1, y1) in zip(points[:-1], points[1:]):
+        auc += 0.5 * (y0 + y1) * (x1 - x0)
+
+    auc /= float(budget)
+
+    return {
+        "anytime_risk_auc": float(auc),
+        "anytime_risk_at_25": risk_at(int(round(0.25 * budget))),
+        "anytime_risk_at_50": risk_at(int(round(0.50 * budget))),
+        "anytime_risk_at_75": risk_at(int(round(0.75 * budget))),
+        "anytime_risk_at_100": risk_at(budget),
+    }
+
+
 def score_agent(
     original_case: GridCase,
     output: AgentOutput,
@@ -1039,6 +1136,12 @@ def score_agent(
     unvalidated_claims = reported - found
     unvalidated_claim_rate = len(unvalidated_claims) / max(1, len(reported))
 
+    anytime_metrics = compute_anytime_risk_metrics(
+        output=output,
+        oracle_values=oracle_values,
+        dangerous=dangerous,
+    )
+
     eval_case = output.mitigated_case if output.mitigated_case is not None else original_case
 
     pre_top_values = [dc_power_flow(original_case, c).severity for c in oracle_top_list]
@@ -1094,6 +1197,7 @@ def score_agent(
             if output.validation_budget
             else 0.0
         ),
+        **anytime_metrics,
     }
 
 

tests/test_anytime_risk.py

@@ -0,0 +1,157 @@
+import unittest
+
+from poweragentbench.steady_state_agentic import (
+    AgentOutput,
+    compute_anytime_risk_metrics,
+)
+
+
+class TestAnytimeRiskMetrics(unittest.TestCase):
+    def setUp(self):
+        self.A, self.B, self.C, self.D = (0, 1), (0, 2), (0, 3), (0, 4)
+        self.oracle = {
+            self.A: 4.0,
+            self.B: 3.0,
+            self.C: 2.0,
+            self.D: 1.0,
+        }
+        self.dangerous = set(self.oracle)
+
+    def output(self, order, budget=4):
+        return AgentOutput(
+            name="toy",
+            validated={c: self.oracle[c] for c in order},
+            reported=[],
+            validation_budget=float(budget),
+        )
+
+    def metrics(self, order, budget=4, oracle=None, dangerous=None):
+        return compute_anytime_risk_metrics(
+            self.output(order, budget),
+            self.oracle if oracle is None else oracle,
+            self.dangerous if dangerous is None else dangerous,
+        )
+
+    def test_known_early_trajectory(self):
+        result = self.metrics([self.A, self.B, self.C, self.D])
+
+        self.assertAlmostEqual(result["anytime_risk_auc"], 0.625)
+        self.assertAlmostEqual(result["anytime_risk_at_25"], 0.40)
+        self.assertAlmostEqual(result["anytime_risk_at_50"], 0.70)
+        self.assertAlmostEqual(result["anytime_risk_at_75"], 0.90)
+        self.assertAlmostEqual(result["anytime_risk_at_100"], 1.00)
+
+    def test_early_discovery_beats_late_discovery(self):
+        early = self.metrics([self.A, self.B, self.C, self.D])
+        late = self.metrics([self.D, self.C, self.B, self.A])
+
+        self.assertAlmostEqual(early["anytime_risk_auc"], 0.625)
+        self.assertAlmostEqual(late["anytime_risk_auc"], 0.375)
+        self.assertEqual(early["anytime_risk_at_100"], 1.0)
+        self.assertEqual(late["anytime_risk_at_100"], 1.0)
+        self.assertGreater(
+            early["anytime_risk_auc"],
+            late["anytime_risk_auc"],
+        )
+
+    def test_validation_budget_limits_trajectory(self):
+        result = self.metrics(
+            [self.A, self.B, self.C, self.D],
+            budget=2,
+        )
+
+        self.assertAlmostEqual(result["anytime_risk_at_100"], 0.70)
+
+    def test_empty_or_zero_risk_cases_return_zero(self):
+        no_validation = self.metrics([], budget=0)
+        no_danger = self.metrics([self.A], dangerous=set())
+
+        zero_oracle = {self.A: 0.0, self.B: 0.0}
+        zero_risk = self.metrics(
+            [self.A, self.B],
+            budget=2,
+            oracle=zero_oracle,
+            dangerous={self.A, self.B},
+        )
+
+        for result in (no_validation, no_danger, zero_risk):
+            self.assertTrue(all(value == 0.0 for value in result.values()))
+
+    def test_metrics_are_bounded(self):
+        result = self.metrics([self.B, self.A, self.D, self.C])
+
+        for name, value in result.items():
+            with self.subTest(metric=name):
+                self.assertGreaterEqual(value, 0.0)
+                self.assertLessEqual(value, 1.0)
+
+
+if __name__ == "__main__":
+    unittest.main()
+
+
+class TestScoreAgentAnytimeRiskIntegration(unittest.TestCase):
+
+    def test_score_agent_exposes_anytime_risk_metrics(self):
+        from unittest.mock import patch
+
+        from poweragentbench.steady_state_agentic import PFResult, score_agent
+
+        A = (0, 1)
+        B = (0, 2)
+        C = (0, 3)
+        D = (0, 4)
+
+        oracle_values = {
+            A: 4.0,
+            B: 3.0,
+            C: 2.0,
+            D: 1.0,
+        }
+
+        output = AgentOutput(
+            name="integration",
+            validated={
+                A: 4.0,
+                B: 3.0,
+                C: 2.0,
+                D: 1.0,
+            },
+            reported=[A, B, C, D],
+            validation_budget=4.0,
+        )
+
+        fake_pf = PFResult(
+            feasible=True,
+            flows={},
+            loading={},
+            severity=1.0,
+            island_penalty=0.0,
+            outage=(),
+        )
+
+        with patch(
+            "poweragentbench.steady_state_agentic.dc_power_flow",
+            return_value=fake_pf,
+        ):
+            result = score_agent(
+                original_case=None,
+                output=output,
+                oracle_values=oracle_values,
+                top_m=4,
+                danger_threshold=1.0,
+            )
+
+        for key in (
+            "anytime_risk_auc",
+            "anytime_risk_at_25",
+            "anytime_risk_at_50",
+            "anytime_risk_at_75",
+            "anytime_risk_at_100",
+        ):
+            self.assertIn(key, result)
+            self.assertGreaterEqual(result[key], 0.0)
+            self.assertLessEqual(result[key], 1.0)
+
+        self.assertAlmostEqual(result["anytime_risk_auc"], 0.625)
+        self.assertAlmostEqual(result["anytime_risk_at_100"], 1.0)