From fa7f6df6eed1d3a809923cfc08e9d3b9325f64b7 Mon Sep 17 00:00:00 2001 From: Happy <78135550+Createyouracccount@users.noreply.github.com> Date: Sun, 6 Sep 2026 21:26:12 +0900 Subject: [PATCH] =?UTF-8?q?model=20call=20Unknown=20class=20=EB=85=B8?= =?UTF-8?q?=EC=B6=9C=EA=B3=BC=20provider=20=EB=8F=99=EC=9E=91=20=EA=B3=A0?= =?UTF-8?q?=EC=A0=95=20=ED=85=8C=EC=8A=A4=ED=8A=B8,=20CI=20no-fail-fast?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RecoveryReason::ModelCallUnknown이 journal의 ModelCallUnknownReason을 버려 timeout, transport_unavailable, interrupted가 모두 model_call_unknown 하나로 보고됐다. 로컬 27B에서 planner timeout을 진단할 때 저널을 직접 열어야 했던 원인이다. 이제 model_call_unknown.로 보고하며, durable Unknown 상태·process 경계 뒤 남은 reservation(interrupted)·planner port failure(timeout, transport_unavailable) 세 경로가 같은 class로 수렴한다. 실측으로 잡은 provider 동작을 hermetic 테스트로 고정한다: assistant message의 reasoning/reasoning_content 필드는 planner와 probe 양쪽에서 무시되고, 응답을 멈춘 provider는 client 예산(1초)에서 PlannerUnavailable(Timeout)으로 닫히며 journal에 ModelCallBecameUnknown(timeout)을 남기고 재시도하지 않는다. CI와 release quality job의 cargo test에 --no-fail-fast를 추가한다. 한 test binary가 실패하면 나머지 binary 결과가 보고되지 않아 45개 중 14개만 보이던 문제를 없앤다. --- .github/workflows/ci.yml | 2 +- .github/workflows/release.yml | 2 +- crates/xgeny-cli/src/composition.rs | 115 +++++++++++++++--- crates/xgeny-provider-openai/src/lib.rs | 35 ++++++ .../tests/http_contract.rs | 82 ++++++++++++- docs/getting-started.md | 5 +- 6 files changed, 216 insertions(+), 25 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f965817..544b9e2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -136,7 +136,7 @@ jobs: run: rustup show active-toolchain - name: Test workspace - run: cargo test --workspace --locked + run: cargo test --workspace --locked --no-fail-fast - name: Lint platform-specific filesystem adapter run: cargo clippy --locked -p xgeny-adapter-filesystem --all-targets -- -D warnings diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 026cff6..81205eb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -120,7 +120,7 @@ jobs: run: cargo clippy --workspace --all-targets --locked -- -D warnings - name: Test workspace - run: cargo test --workspace --locked + run: cargo test --workspace --locked --no-fail-fast - name: Check third-party license notices run: sh scripts/check-third-party-licenses.sh --check diff --git a/crates/xgeny-cli/src/composition.rs b/crates/xgeny-cli/src/composition.rs index fd3166c..3170494 100644 --- a/crates/xgeny-cli/src/composition.rs +++ b/crates/xgeny-cli/src/composition.rs @@ -46,9 +46,9 @@ use xgeny_runtime::{ PlanningConstraint, ProposalRejection, RequiredRouteFeatures, RouteRequest, }; use xgeny_workgraph::{ - CompletionOutputRecord, ModelCallRejectionReason, ModelCallStatus, PlannedExecutionProfile, - ReconstructableMaterialReference, RunEvent, RunEventBody, RunState, StepStatus, - derive_frontier, + CompletionOutputRecord, ModelCallRejectionReason, ModelCallStatus, ModelCallUnknownReason, + PlannedExecutionProfile, ReconstructableMaterialReference, RunEvent, RunEventBody, RunState, + StepStatus, derive_frontier, }; use crate::allow_file::{ALLOW_FILE_PROVIDER_ID, AllowFileCatalog}; @@ -508,7 +508,8 @@ const fn proposal_rejection_code(rejection: ProposalRejection) -> &'static str { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum RecoveryReason { - ModelCallUnknown, + /// A model call is durably Unknown. The class is the same value the journal records. + ModelCallUnknown(ModelCallUnknownReason), EffectOutcomeUnknown, } @@ -516,7 +517,13 @@ impl RecoveryReason { #[must_use] pub const fn code(self) -> &'static str { match self { - Self::ModelCallUnknown => "model_call_unknown", + Self::ModelCallUnknown(ModelCallUnknownReason::Timeout) => "model_call_unknown.timeout", + Self::ModelCallUnknown(ModelCallUnknownReason::TransportUnavailable) => { + "model_call_unknown.transport_unavailable" + } + Self::ModelCallUnknown(ModelCallUnknownReason::Interrupted) => { + "model_call_unknown.interrupted" + } Self::EffectOutcomeUnknown => "effect_outcome_unknown", } } @@ -880,19 +887,20 @@ where summary: output.summary().to_owned(), }); } - if has_unknown_model_call(&state) { + if let Some(reason) = unknown_model_call_reason(&state) { return Ok(LocalCommandResult::RecoveryRequired { run_id: state.run_id, - reason: RecoveryReason::ModelCallUnknown, + reason: RecoveryReason::ModelCallUnknown(reason), }); } if has_reserved_model_call(&state) { drop(store); let mut store = reopen_writable_verified(&layout, &manifest, &state)?; mark_reserved_model_call_unknown(&mut store, &lease, &manifest)?; + // A reservation found without its outcome after a process boundary is Interrupted. return Ok(LocalCommandResult::RecoveryRequired { run_id: state.run_id, - reason: RecoveryReason::ModelCallUnknown, + reason: RecoveryReason::ModelCallUnknown(ModelCallUnknownReason::Interrupted), }); } if let Some(step_id) = executing_step_id(&state) { @@ -1238,8 +1246,12 @@ fn classify_durable_boundary_after_driver_error( .map_err(|_| PublicRunError::Integrity)? .ok_or(PublicRunError::Integrity)?; verify_manifest_state(manifest, &state)?; - let reason = if has_unknown_model_call(&state) || has_reserved_model_call(&state) { - Some(RecoveryReason::ModelCallUnknown) + let reason = if let Some(reason) = unknown_model_call_reason(&state) { + Some(RecoveryReason::ModelCallUnknown(reason)) + } else if has_reserved_model_call(&state) { + Some(RecoveryReason::ModelCallUnknown( + ModelCallUnknownReason::Interrupted, + )) } else if executing_step_id(&state).is_some() || has_effect_uncertainty(&state) { Some(RecoveryReason::EffectOutcomeUnknown) } else { @@ -1379,13 +1391,17 @@ fn verify_manifest_state(manifest: &RunManifest, state: &RunState) -> Result<(), Ok(()) } -fn has_unknown_model_call(state: &RunState) -> bool { +/// The durable Unknown class of the active model call, if any. +fn unknown_model_call_reason(state: &RunState) -> Option { state .agent_loop .as_ref() .and_then(|agent| agent.model_calls.as_ref()) .and_then(|lifecycle| lifecycle.active_call.as_ref()) - .is_some_and(|call| matches!(call.status, ModelCallStatus::Unknown { .. })) + .and_then(|call| match call.status { + ModelCallStatus::Unknown { reason } => Some(reason), + ModelCallStatus::Reserved => None, + }) } fn has_reserved_model_call(state: &RunState) -> bool { @@ -1628,10 +1644,18 @@ fn map_layout_create(error: crate::run_layout::RunLayoutError) -> PublicRunError /// durable `ModelCallRejectionReason` class so callers never see a bare `model_rejected`. fn map_planner_unavailable(run_id: String, failure: PlannerPortFailure) -> LocalCommandResult { let reason = match failure { - PlannerPortFailure::Timeout | PlannerPortFailure::Unavailable => { + PlannerPortFailure::Timeout => { return LocalCommandResult::RecoveryRequired { run_id, - reason: RecoveryReason::ModelCallUnknown, + reason: RecoveryReason::ModelCallUnknown(ModelCallUnknownReason::Timeout), + }; + } + PlannerPortFailure::Unavailable => { + return LocalCommandResult::RecoveryRequired { + run_id, + reason: RecoveryReason::ModelCallUnknown( + ModelCallUnknownReason::TransportUnavailable, + ), }; } PlannerPortFailure::InvalidResponse => ModelCallRejectionReason::PlannerInvalidResponse, @@ -1717,10 +1741,12 @@ fn map_driver_outcome( reason: RejectionReason::MaterialRejected, }, DriverOutcome::PlannerUnavailable(failure) => map_planner_unavailable(run_id, failure), - DriverOutcome::ModelCallRecoveryRequired { .. } => LocalCommandResult::RecoveryRequired { - run_id, - reason: RecoveryReason::ModelCallUnknown, - }, + DriverOutcome::ModelCallRecoveryRequired { reason, .. } => { + LocalCommandResult::RecoveryRequired { + run_id, + reason: RecoveryReason::ModelCallUnknown(reason), + } + } DriverOutcome::ModelCallRejected(reason) => LocalCommandResult::Rejected { run_id, reason: RejectionReason::ModelRejected(reason), @@ -2561,6 +2587,59 @@ mod tests { ); } + #[test] + fn model_call_unknown_class_reaches_the_public_result_code() { + // Journaled path: the driver carries the durable ModelCallUnknownReason. + assert_eq!( + map_driver_outcome( + "run-unknown", + DriverOutcome::ModelCallRecoveryRequired { + call_id: "model-call-x".to_owned(), + reason: ModelCallUnknownReason::Timeout, + }, + ) + .unwrap(), + LocalCommandResult::RecoveryRequired { + run_id: "run-unknown".to_owned(), + reason: RecoveryReason::ModelCallUnknown(ModelCallUnknownReason::Timeout), + } + ); + // Un-journaled planner port failures map onto the same durable classes. + assert_eq!( + map_driver_outcome( + "run-unknown", + DriverOutcome::PlannerUnavailable(PlannerPortFailure::Unavailable), + ) + .unwrap(), + LocalCommandResult::RecoveryRequired { + run_id: "run-unknown".to_owned(), + reason: RecoveryReason::ModelCallUnknown( + ModelCallUnknownReason::TransportUnavailable + ), + } + ); + for (reason, code) in [ + ( + ModelCallUnknownReason::Timeout, + "model_call_unknown.timeout", + ), + ( + ModelCallUnknownReason::TransportUnavailable, + "model_call_unknown.transport_unavailable", + ), + ( + ModelCallUnknownReason::Interrupted, + "model_call_unknown.interrupted", + ), + ] { + assert_eq!(RecoveryReason::ModelCallUnknown(reason).code(), code); + } + assert_eq!( + RecoveryReason::EffectOutcomeUnknown.code(), + "effect_outcome_unknown" + ); + } + #[test] fn model_rejection_class_reaches_the_public_result_code() { // Journaled settlement path: the durable ModelCallRejectionReason must survive to the code. diff --git a/crates/xgeny-provider-openai/src/lib.rs b/crates/xgeny-provider-openai/src/lib.rs index 78e66f9..bd823a0 100644 --- a/crates/xgeny-provider-openai/src/lib.rs +++ b/crates/xgeny-provider-openai/src/lib.rs @@ -2252,6 +2252,41 @@ mod tests { ); } + #[test] + fn assistant_reasoning_fields_are_ignored_on_both_decode_paths() { + // Ollama returns `reasoning`, vLLM/llama.cpp return `reasoning_content` next to `content` + // for thinking models. Both are provider-side fields, not part of the strict document. + for field in ["reasoning", "reasoning_content"] { + let planner_body = serde_json::to_vec(&json!({ + "model": MODEL, + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": valid_plan(), field: "…thinking…"}, + "finish_reason": "stop" + }] + })) + .unwrap(); + assert!( + decode_chat_response(&planner_body, MODEL, 1 << 16, 8).is_ok(), + "planner path must ignore `{field}`" + ); + let probe_body = serde_json::to_vec(&json!({ + "model": MODEL, + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": COMPLETION_OK, field: "…thinking…"}, + "finish_reason": "stop" + }] + })) + .unwrap(); + assert_eq!( + decode_compatibility_response(&probe_body, MODEL, 8), + Ok(()), + "probe path must ignore `{field}`" + ); + } + } + #[test] fn compatibility_probe_requests_the_configured_output_budget() { let config = config("https://provider.example/v1") diff --git a/crates/xgeny-provider-openai/tests/http_contract.rs b/crates/xgeny-provider-openai/tests/http_contract.rs index 9b7927e..ef9a6d5 100644 --- a/crates/xgeny-provider-openai/tests/http_contract.rs +++ b/crates/xgeny-provider-openai/tests/http_contract.rs @@ -22,9 +22,9 @@ use xgeny_workgraph::{ AgentLoopBudget, AgentLoopState, AuthorizationBinding, AuthorizationUse, CompletionOutputRecord, EffectClass as WorkEffectClass, EffectIntent, EventRecord, InvocationBinding, ModelCallBudget, ModelCallLifecycleState, ModelCallRejectionReason, - ModelCallReservation, ModelCallSettlement, ReceiptPlacement, ReceiptProvenance, - ReconstructableMaterialReference, RunEvent, RunEventBody, RunState, SinkGuarantee, StepState, - StepStatus, TOOL_OUTPUT_PROFILE_V1, ToolOutputRecord, apply_record, + ModelCallReservation, ModelCallSettlement, ModelCallUnknownReason, ReceiptPlacement, + ReceiptProvenance, ReconstructableMaterialReference, RunEvent, RunEventBody, RunState, + SinkGuarantee, StepState, StepStatus, TOOL_OUTPUT_PROFILE_V1, ToolOutputRecord, apply_record, }; const AUTHORITY: &str = "local:test"; @@ -789,6 +789,82 @@ fn deterministic_provider_rejection_is_closed_without_raw_error_body() { assert!(!durable.contains(RAW_RESPONSE_SENTINEL)); } +/// A provider that accepts the request and then stalls longer than the planner budget. +fn spawn_stalling_server(stall: Duration) -> (String, thread::JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").expect("test listener should bind"); + let address = listener + .local_addr() + .expect("listener address should resolve"); + let handle = thread::spawn(move || { + let (mut stream, _) = listener.accept().expect("one request should connect"); + let _request = read_http_request(&mut stream); + thread::sleep(stall); + // Whatever we write now is too late; the client has already given up. + let _ = + stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"); + }); + (format!("http://{address}/v1"), handle) +} + +#[test] +fn slow_provider_becomes_a_durable_timeout_without_retry() { + // Measured in the wild: a 27B local model needs ~60s per planner call; when the configured + // budget is shorter the call must close as model_call_unknown(timeout), not as a rejection, + // and the client must not retry. + let (base_url, handle) = spawn_stalling_server(Duration::from_secs(2)); + let config = OpenAiPlannerConfig::new( + &base_url, + "xgeny.test.go50902", + "qwen3.8-27b", + "Qwen/Qwen3.8-27B-FP8", + ) + .expect("planner config should validate") + .with_max_output_tokens(512) + .expect("output limit should validate") + .with_timeout(Duration::from_secs(1)) + .expect("timeout should validate"); + let mut planner = OpenAiPlanner::new(config, None).expect("planner should build"); + let mut store = seed_store(); + let loop_runtime = configured_loop(&mut store, &mut planner); + let mut events = DeterministicEvents; + let mut materializer = EphemeralMaterializer; + let started = std::time::Instant::now(); + let tick = loop_runtime + .tick( + &mut store, + &mut events, + &FixedLease, + &synthetic_registry(), + &IdentityResolver::default(), + &mut planner, + &mut materializer, + ) + .expect("timeout should settle durably"); + assert!( + started.elapsed() < Duration::from_secs(2), + "client must give up at its own budget, not wait for the provider" + ); + // The first tick after a transport timeout reports the port failure; the reservation is + // durably marked Unknown(timeout) so a later tick or resume never replays the call. + assert!(matches!( + tick, + AgentLoopTick::PlannerUnavailable { + failure: PlannerPortFailure::Timeout, + .. + } + )); + let snapshot = store.load().unwrap().unwrap(); + let last = snapshot.records.last().expect("unknown event should exist"); + assert!(matches!( + &last.event.body, + RunEventBody::ModelCallBecameUnknown { + reason: ModelCallUnknownReason::Timeout, + .. + } + )); + handle.join().expect("server should finish"); +} + #[test] #[ignore = "requires an explicitly configured OpenAI-compatible model endpoint"] fn live_go50902_plan_smoke() { diff --git a/docs/getting-started.md b/docs/getting-started.md index 33eeb90..58d6e62 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -433,8 +433,9 @@ State 삭제는 Run 기록과 durable recovery 정보를 잃으므로 uninstall | `proposal_rejected.*` | 뒤의 class가 Core가 제안을 거부한 이유다. `capability_unavailable`/`capability_unsupported`는 허용하지 않은 capability 선택, `invocation_invalid`는 scope 밖 인자나 스키마 위반, `tool_call_budget_exhausted`는 예산 소진이다. Class는 Core 판정이며 model 출력 원문이 아니다. | | `model_rejected.*` | 뒤의 class는 journal의 model call settlement와 같은 값이다. `planner_invalid_response`는 provider가 strict JSON Schema를 지키지 않은 응답(문법 미지원·미적용), `provider_limit`은 출력 예산·요청 크기 초과, `provider_rejected`는 4xx 거부다. Class는 Core 판정이며 model 출력 원문이 아니다. | | `configuration_mismatch` | 원래 workspace, file/directory scope, executable와 model profile binding(inference timeout·출력 예산 포함)으로 resume한다. 자동 대체하지 말고 필요하면 새 Run을 시작한다. | -| `model_call_unknown`이 planner 호출마다 반복 | 프로필의 inference timeout이 model·hardware에 비해 짧다. 로컬 27B는 호출당 60초 안팎이 걸리므로 `--inference-timeout`을 올린다. | -| `model_call_unknown` 또는 `effect_outcome_unknown` | 불확정 작업을 자동 반복하지 않는다. `/status`와 `/resume`의 고정 진단을 확인하고 외부 상태를 별도로 검증한다. | +| `model_call_unknown.timeout`이 planner 호출마다 반복 | 프로필의 inference timeout이 model·hardware에 비해 짧다. 로컬 27B는 호출당 60초 안팎이 걸리므로 `--inference-timeout`을 올린다. | +| `model_call_unknown.transport_unavailable` / `.interrupted` | 요청이 전송됐을 수 있으나 결과를 못 받았다(연결 끊김) 또는 process가 응답 전에 끝났다. 자동 replay하지 않으므로 endpoint 상태를 확인한 뒤 `resume`한다. | +| `model_call_unknown.*` 또는 `effect_outcome_unknown` | 불확정 작업을 자동 반복하지 않는다. `/status`와 `/resume`의 고정 진단을 확인하고 외부 상태를 별도로 검증한다. | 지원 요청에는 `xgeny --version`, OS/architecture, 설치 채널, 종료 코드와 고정된 오류 코드만 우선 제공한다. API key, endpoint 전체 URL, prompt, model 원문 응답, source, process stdout/stderr, state DB와 Run ID는 공개