diff --git a/vero/src/vero/evaluation/scoring/error_taxonomy.py b/vero/src/vero/evaluation/scoring/error_taxonomy.py index 8637ee2d..20102dd0 100644 --- a/vero/src/vero/evaluation/scoring/error_taxonomy.py +++ b/vero/src/vero/evaluation/scoring/error_taxonomy.py @@ -43,6 +43,11 @@ class ErrorCategory(str, Enum): #: Authentication/authorization failed (e.g. a wrong or cycled key). Does #: not heal on retry: terminate loudly. AUTH_FAILURE = "auth_failure" + #: The requested model is not deployed on the configured upstream. A + #: permanent misconfiguration, not the candidate's doing: never scored as a + #: sample, and terminating, because every remaining case will fail the same + #: way and the run has nothing left to measure. + UPSTREAM_MODEL_UNAVAILABLE = "upstream_model_unavailable" #: Transient external infrastructure (rate limit, timeout, connection, 5xx, #: overloaded). Retryable; if it persists it renders the aggregate invalid. TRANSIENT_INFRA = "transient_infra" @@ -91,6 +96,16 @@ class CategoryPolicy: is_informative_sample=False, diagnostic_code="auth_failure", ), + ErrorCategory.UPSTREAM_MODEL_UNAVAILABLE: CategoryPolicy( + retryable=False, + terminating=True, + # Unlike auth, keep counting these toward invalidity: if the terminating + # path is ever bypassed, the aggregate must still come out invalid + # rather than silently averaging a shrinking set of survivors. + counts_toward_invalidity=True, + is_informative_sample=False, + diagnostic_code="upstream_model_unavailable", + ), ErrorCategory.TRANSIENT_INFRA: CategoryPolicy( retryable=True, terminating=False, @@ -149,6 +164,21 @@ def policy(category: ErrorCategory) -> CategoryPolicy: ), ErrorCategory.AUTH_FAILURE, ), + ( + # A model that is configured but not provisioned upstream. Left + # unclassified this is the worst kind of silent failure: the agent's + # every call 404s, it writes no answer, and the case is recorded as an + # informative task failure: the harness blaming the candidate for a + # model that does not exist. Matched on the provider's own error codes + # and on the exact Azure sentence, never on a bare "does not exist", + # which would also swallow "loading container: file does not exist". + re.compile( + r"deploymentnotfound|model_not_found|" + r"the api deployment for this resource does not exist", + re.IGNORECASE, + ), + ErrorCategory.UPSTREAM_MODEL_UNAVAILABLE, + ), ( re.compile( r"rate.?limit|too.?many.?requests|time(?:d.?)?out|connection|" @@ -205,6 +235,9 @@ def classify_case(signals: list[str]) -> ErrorCategory: for category in ( ErrorCategory.INFERENCE_BUDGET_EXHAUSTED, ErrorCategory.AUTH_FAILURE, + # Ahead of infra: a case that saw both a dead sandbox and a missing + # deployment should report the deterministic, operator-fixable one. + ErrorCategory.UPSTREAM_MODEL_UNAVAILABLE, ErrorCategory.TRANSIENT_INFRA, ): if category in categories: diff --git a/vero/tests/test_v05_error_taxonomy.py b/vero/tests/test_v05_error_taxonomy.py index bace7eda..1cdb742a 100644 --- a/vero/tests/test_v05_error_taxonomy.py +++ b/vero/tests/test_v05_error_taxonomy.py @@ -95,3 +95,62 @@ def test_classify_case_treats_environment_loss_as_infra_not_task_failure(): ) # A candidate's own bug is still an informative task failure, unchanged. assert classify_case(["ValueError"]) is ErrorCategory.TASK_FAILURE + + +def test_a_missing_upstream_deployment_is_not_a_task_failure(): + """The 2026-07-25 swe-atlas-qna run's exact terminal exceptions. + + 145 of its 469 cases died on a model that is not provisioned. Classified as + a task failure, each was recorded as an informative score of 0.0: the + harness blaming the candidate for a model that does not exist. + """ + azure = ( + "Error code: 404 - {'error': {'type': 'invalid_request_error', " + "'code': 'DeploymentNotFound', 'message': 'The API deployment for this " + "resource does not exist. If you created the deployment within the " + "last 5 minutes, please wait a moment and try again.'}}" + ) + assert ( + classify_signal(azure) is ErrorCategory.UPSTREAM_MODEL_UNAVAILABLE + ) + assert ( + classify_signal("model_not_found") is ErrorCategory.UPSTREAM_MODEL_UNAVAILABLE + ) + assert ( + classify_case(["NotFoundError", azure]) + is ErrorCategory.UPSTREAM_MODEL_UNAVAILABLE + ) + + unavailable = policy(ErrorCategory.UPSTREAM_MODEL_UNAVAILABLE) + assert not unavailable.is_informative_sample + assert not unavailable.retryable + assert unavailable.terminating + assert unavailable.counts_toward_invalidity + + # It outranks a co-occurring sandbox death: the deterministic, + # operator-fixable cause is the one worth reporting. + assert ( + classify_case( + [ + "NotFoundError", + "Modal Sandbox with container ID ta-01KY not found.", + azure, + ] + ) + is ErrorCategory.UPSTREAM_MODEL_UNAVAILABLE + ) + + +def test_missing_deployment_pattern_does_not_swallow_container_load_failures(): + """`FetchSpec failed: loading container: file does not exist` is infra. + + It is 102 of that same run's cases, and it also contains "does not exist", + so the deployment pattern must not be written loosely enough to claim it. + """ + fetchspec = "FetchSpec failed: loading container: file does not exist\n" + assert classify_signal(fetchspec) is ErrorCategory.TRANSIENT_INFRA + assert classify_case(["RuntimeError", fetchspec]) is ErrorCategory.TRANSIENT_INFRA + assert ( + classify_case(["AddTestsDirError", "Failed to add tests directory to environment."]) + is ErrorCategory.TRANSIENT_INFRA + )