From 10515d12ad0e8c489752f035fb13c94c6ea82917 Mon Sep 17 00:00:00 2001 From: Shehab Yasser Date: Sat, 25 Jul 2026 09:14:32 +0300 Subject: [PATCH] fix: a missing upstream deployment is not a task failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #53 stopped a dead sandbox being scored as an informative zero. The same masking survives one layer up, for a cause that is even easier to fix. In the 2026-07-25 swe-atlas-qna run, 145 of 469 cases ended on: NotFoundError: Error code: 404 - {'error': {'type': 'invalid_request_error', 'code': 'DeploymentNotFound', 'message': 'The API deployment for this resource does not exist. ...'}} The configured eval model is not provisioned on the Azure resource. The agent's every call 404s, it writes no answer, and `classify_case` — finding nothing in the signal it recognises — returns TASK_FAILURE, whose policy is `is_informative_sample=True`. So each case was recorded as a genuine score of 0.0 and the harness reported that the candidate failed the task. Add UPSTREAM_MODEL_UNAVAILABLE: never an informative sample, never retried, terminating (every remaining case fails identically, so there is nothing left to measure), and counted toward invalidity so the aggregate still comes out invalid if the terminating path is ever bypassed. It is ranked above TRANSIENT_INFRA, so a case that saw both a dead sandbox and a missing deployment reports the deterministic, operator-fixable cause. The pattern deliberately matches the provider error codes and the exact Azure sentence rather than a bare "does not exist": 102 cases in that same run failed with `FetchSpec failed: loading container: file does not exist`, which is genuine infrastructure and must stay TRANSIENT_INFRA. There is a test pinning both sides. Refs #51. Co-Authored-By: Claude Opus 5 (1M context) --- .../vero/evaluation/scoring/error_taxonomy.py | 33 +++++++++++ vero/tests/test_v05_error_taxonomy.py | 59 +++++++++++++++++++ 2 files changed, 92 insertions(+) diff --git a/vero/src/vero/evaluation/scoring/error_taxonomy.py b/vero/src/vero/evaluation/scoring/error_taxonomy.py index 8637ee2d..20102dd0 100644 --- a/vero/src/vero/evaluation/scoring/error_taxonomy.py +++ b/vero/src/vero/evaluation/scoring/error_taxonomy.py @@ -43,6 +43,11 @@ class ErrorCategory(str, Enum): #: Authentication/authorization failed (e.g. a wrong or cycled key). Does #: not heal on retry: terminate loudly. AUTH_FAILURE = "auth_failure" + #: The requested model is not deployed on the configured upstream. A + #: permanent misconfiguration, not the candidate's doing: never scored as a + #: sample, and terminating, because every remaining case will fail the same + #: way and the run has nothing left to measure. + UPSTREAM_MODEL_UNAVAILABLE = "upstream_model_unavailable" #: Transient external infrastructure (rate limit, timeout, connection, 5xx, #: overloaded). Retryable; if it persists it renders the aggregate invalid. TRANSIENT_INFRA = "transient_infra" @@ -91,6 +96,16 @@ class CategoryPolicy: is_informative_sample=False, diagnostic_code="auth_failure", ), + ErrorCategory.UPSTREAM_MODEL_UNAVAILABLE: CategoryPolicy( + retryable=False, + terminating=True, + # Unlike auth, keep counting these toward invalidity: if the terminating + # path is ever bypassed, the aggregate must still come out invalid + # rather than silently averaging a shrinking set of survivors. + counts_toward_invalidity=True, + is_informative_sample=False, + diagnostic_code="upstream_model_unavailable", + ), ErrorCategory.TRANSIENT_INFRA: CategoryPolicy( retryable=True, terminating=False, @@ -149,6 +164,21 @@ def policy(category: ErrorCategory) -> CategoryPolicy: ), ErrorCategory.AUTH_FAILURE, ), + ( + # A model that is configured but not provisioned upstream. Left + # unclassified this is the worst kind of silent failure: the agent's + # every call 404s, it writes no answer, and the case is recorded as an + # informative task failure: the harness blaming the candidate for a + # model that does not exist. Matched on the provider's own error codes + # and on the exact Azure sentence, never on a bare "does not exist", + # which would also swallow "loading container: file does not exist". + re.compile( + r"deploymentnotfound|model_not_found|" + r"the api deployment for this resource does not exist", + re.IGNORECASE, + ), + ErrorCategory.UPSTREAM_MODEL_UNAVAILABLE, + ), ( re.compile( r"rate.?limit|too.?many.?requests|time(?:d.?)?out|connection|" @@ -205,6 +235,9 @@ def classify_case(signals: list[str]) -> ErrorCategory: for category in ( ErrorCategory.INFERENCE_BUDGET_EXHAUSTED, ErrorCategory.AUTH_FAILURE, + # Ahead of infra: a case that saw both a dead sandbox and a missing + # deployment should report the deterministic, operator-fixable one. + ErrorCategory.UPSTREAM_MODEL_UNAVAILABLE, ErrorCategory.TRANSIENT_INFRA, ): if category in categories: diff --git a/vero/tests/test_v05_error_taxonomy.py b/vero/tests/test_v05_error_taxonomy.py index bace7eda..1cdb742a 100644 --- a/vero/tests/test_v05_error_taxonomy.py +++ b/vero/tests/test_v05_error_taxonomy.py @@ -95,3 +95,62 @@ def test_classify_case_treats_environment_loss_as_infra_not_task_failure(): ) # A candidate's own bug is still an informative task failure, unchanged. assert classify_case(["ValueError"]) is ErrorCategory.TASK_FAILURE + + +def test_a_missing_upstream_deployment_is_not_a_task_failure(): + """The 2026-07-25 swe-atlas-qna run's exact terminal exceptions. + + 145 of its 469 cases died on a model that is not provisioned. Classified as + a task failure, each was recorded as an informative score of 0.0: the + harness blaming the candidate for a model that does not exist. + """ + azure = ( + "Error code: 404 - {'error': {'type': 'invalid_request_error', " + "'code': 'DeploymentNotFound', 'message': 'The API deployment for this " + "resource does not exist. If you created the deployment within the " + "last 5 minutes, please wait a moment and try again.'}}" + ) + assert ( + classify_signal(azure) is ErrorCategory.UPSTREAM_MODEL_UNAVAILABLE + ) + assert ( + classify_signal("model_not_found") is ErrorCategory.UPSTREAM_MODEL_UNAVAILABLE + ) + assert ( + classify_case(["NotFoundError", azure]) + is ErrorCategory.UPSTREAM_MODEL_UNAVAILABLE + ) + + unavailable = policy(ErrorCategory.UPSTREAM_MODEL_UNAVAILABLE) + assert not unavailable.is_informative_sample + assert not unavailable.retryable + assert unavailable.terminating + assert unavailable.counts_toward_invalidity + + # It outranks a co-occurring sandbox death: the deterministic, + # operator-fixable cause is the one worth reporting. + assert ( + classify_case( + [ + "NotFoundError", + "Modal Sandbox with container ID ta-01KY not found.", + azure, + ] + ) + is ErrorCategory.UPSTREAM_MODEL_UNAVAILABLE + ) + + +def test_missing_deployment_pattern_does_not_swallow_container_load_failures(): + """`FetchSpec failed: loading container: file does not exist` is infra. + + It is 102 of that same run's cases, and it also contains "does not exist", + so the deployment pattern must not be written loosely enough to claim it. + """ + fetchspec = "FetchSpec failed: loading container: file does not exist\n" + assert classify_signal(fetchspec) is ErrorCategory.TRANSIENT_INFRA + assert classify_case(["RuntimeError", fetchspec]) is ErrorCategory.TRANSIENT_INFRA + assert ( + classify_case(["AddTestsDirError", "Failed to add tests directory to environment."]) + is ErrorCategory.TRANSIENT_INFRA + )