Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ jobs:
run: rustup show active-toolchain

- name: Test workspace
run: cargo test --workspace --locked
run: cargo test --workspace --locked --no-fail-fast

- name: Lint platform-specific filesystem adapter
run: cargo clippy --locked -p xgeny-adapter-filesystem --all-targets -- -D warnings
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ jobs:
run: cargo clippy --workspace --all-targets --locked -- -D warnings

- name: Test workspace
run: cargo test --workspace --locked
run: cargo test --workspace --locked --no-fail-fast

- name: Check third-party license notices
run: sh scripts/check-third-party-licenses.sh --check
Expand Down
115 changes: 97 additions & 18 deletions crates/xgeny-cli/src/composition.rs
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,9 @@ use xgeny_runtime::{
PlanningConstraint, ProposalRejection, RequiredRouteFeatures, RouteRequest,
};
use xgeny_workgraph::{
CompletionOutputRecord, ModelCallRejectionReason, ModelCallStatus, PlannedExecutionProfile,
ReconstructableMaterialReference, RunEvent, RunEventBody, RunState, StepStatus,
derive_frontier,
CompletionOutputRecord, ModelCallRejectionReason, ModelCallStatus, ModelCallUnknownReason,
PlannedExecutionProfile, ReconstructableMaterialReference, RunEvent, RunEventBody, RunState,
StepStatus, derive_frontier,
};

use crate::allow_file::{ALLOW_FILE_PROVIDER_ID, AllowFileCatalog};
Expand Down Expand Up @@ -508,15 +508,22 @@ const fn proposal_rejection_code(rejection: ProposalRejection) -> &'static str {

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RecoveryReason {
ModelCallUnknown,
/// A model call is durably Unknown. The class is the same value the journal records.
ModelCallUnknown(ModelCallUnknownReason),
EffectOutcomeUnknown,
}

impl RecoveryReason {
#[must_use]
pub const fn code(self) -> &'static str {
match self {
Self::ModelCallUnknown => "model_call_unknown",
Self::ModelCallUnknown(ModelCallUnknownReason::Timeout) => "model_call_unknown.timeout",
Self::ModelCallUnknown(ModelCallUnknownReason::TransportUnavailable) => {
"model_call_unknown.transport_unavailable"
}
Self::ModelCallUnknown(ModelCallUnknownReason::Interrupted) => {
"model_call_unknown.interrupted"
}
Self::EffectOutcomeUnknown => "effect_outcome_unknown",
}
}
Expand Down Expand Up @@ -880,19 +887,20 @@ where
summary: output.summary().to_owned(),
});
}
if has_unknown_model_call(&state) {
if let Some(reason) = unknown_model_call_reason(&state) {
return Ok(LocalCommandResult::RecoveryRequired {
run_id: state.run_id,
reason: RecoveryReason::ModelCallUnknown,
reason: RecoveryReason::ModelCallUnknown(reason),
});
}
if has_reserved_model_call(&state) {
drop(store);
let mut store = reopen_writable_verified(&layout, &manifest, &state)?;
mark_reserved_model_call_unknown(&mut store, &lease, &manifest)?;
// A reservation found without its outcome after a process boundary is Interrupted.
return Ok(LocalCommandResult::RecoveryRequired {
run_id: state.run_id,
reason: RecoveryReason::ModelCallUnknown,
reason: RecoveryReason::ModelCallUnknown(ModelCallUnknownReason::Interrupted),
});
}
if let Some(step_id) = executing_step_id(&state) {
Expand Down Expand Up @@ -1238,8 +1246,12 @@ fn classify_durable_boundary_after_driver_error(
.map_err(|_| PublicRunError::Integrity)?
.ok_or(PublicRunError::Integrity)?;
verify_manifest_state(manifest, &state)?;
let reason = if has_unknown_model_call(&state) || has_reserved_model_call(&state) {
Some(RecoveryReason::ModelCallUnknown)
let reason = if let Some(reason) = unknown_model_call_reason(&state) {
Some(RecoveryReason::ModelCallUnknown(reason))
} else if has_reserved_model_call(&state) {
Some(RecoveryReason::ModelCallUnknown(
ModelCallUnknownReason::Interrupted,
))
} else if executing_step_id(&state).is_some() || has_effect_uncertainty(&state) {
Some(RecoveryReason::EffectOutcomeUnknown)
} else {
Expand Down Expand Up @@ -1379,13 +1391,17 @@ fn verify_manifest_state(manifest: &RunManifest, state: &RunState) -> Result<(),
Ok(())
}

fn has_unknown_model_call(state: &RunState) -> bool {
/// The durable Unknown class of the active model call, if any.
fn unknown_model_call_reason(state: &RunState) -> Option<ModelCallUnknownReason> {
state
.agent_loop
.as_ref()
.and_then(|agent| agent.model_calls.as_ref())
.and_then(|lifecycle| lifecycle.active_call.as_ref())
.is_some_and(|call| matches!(call.status, ModelCallStatus::Unknown { .. }))
.and_then(|call| match call.status {
ModelCallStatus::Unknown { reason } => Some(reason),
ModelCallStatus::Reserved => None,
})
}

fn has_reserved_model_call(state: &RunState) -> bool {
Expand Down Expand Up @@ -1628,10 +1644,18 @@ fn map_layout_create(error: crate::run_layout::RunLayoutError) -> PublicRunError
/// durable `ModelCallRejectionReason` class so callers never see a bare `model_rejected`.
fn map_planner_unavailable(run_id: String, failure: PlannerPortFailure) -> LocalCommandResult {
let reason = match failure {
PlannerPortFailure::Timeout | PlannerPortFailure::Unavailable => {
PlannerPortFailure::Timeout => {
return LocalCommandResult::RecoveryRequired {
run_id,
reason: RecoveryReason::ModelCallUnknown,
reason: RecoveryReason::ModelCallUnknown(ModelCallUnknownReason::Timeout),
};
}
PlannerPortFailure::Unavailable => {
return LocalCommandResult::RecoveryRequired {
run_id,
reason: RecoveryReason::ModelCallUnknown(
ModelCallUnknownReason::TransportUnavailable,
),
};
}
PlannerPortFailure::InvalidResponse => ModelCallRejectionReason::PlannerInvalidResponse,
Expand Down Expand Up @@ -1717,10 +1741,12 @@ fn map_driver_outcome(
reason: RejectionReason::MaterialRejected,
},
DriverOutcome::PlannerUnavailable(failure) => map_planner_unavailable(run_id, failure),
DriverOutcome::ModelCallRecoveryRequired { .. } => LocalCommandResult::RecoveryRequired {
run_id,
reason: RecoveryReason::ModelCallUnknown,
},
DriverOutcome::ModelCallRecoveryRequired { reason, .. } => {
LocalCommandResult::RecoveryRequired {
run_id,
reason: RecoveryReason::ModelCallUnknown(reason),
}
}
DriverOutcome::ModelCallRejected(reason) => LocalCommandResult::Rejected {
run_id,
reason: RejectionReason::ModelRejected(reason),
Expand Down Expand Up @@ -2561,6 +2587,59 @@ mod tests {
);
}

#[test]
fn model_call_unknown_class_reaches_the_public_result_code() {
// Journaled path: the driver carries the durable ModelCallUnknownReason.
assert_eq!(
map_driver_outcome(
"run-unknown",
DriverOutcome::ModelCallRecoveryRequired {
call_id: "model-call-x".to_owned(),
reason: ModelCallUnknownReason::Timeout,
},
)
.unwrap(),
LocalCommandResult::RecoveryRequired {
run_id: "run-unknown".to_owned(),
reason: RecoveryReason::ModelCallUnknown(ModelCallUnknownReason::Timeout),
}
);
// Un-journaled planner port failures map onto the same durable classes.
assert_eq!(
map_driver_outcome(
"run-unknown",
DriverOutcome::PlannerUnavailable(PlannerPortFailure::Unavailable),
)
.unwrap(),
LocalCommandResult::RecoveryRequired {
run_id: "run-unknown".to_owned(),
reason: RecoveryReason::ModelCallUnknown(
ModelCallUnknownReason::TransportUnavailable
),
}
);
for (reason, code) in [
(
ModelCallUnknownReason::Timeout,
"model_call_unknown.timeout",
),
(
ModelCallUnknownReason::TransportUnavailable,
"model_call_unknown.transport_unavailable",
),
(
ModelCallUnknownReason::Interrupted,
"model_call_unknown.interrupted",
),
] {
assert_eq!(RecoveryReason::ModelCallUnknown(reason).code(), code);
}
assert_eq!(
RecoveryReason::EffectOutcomeUnknown.code(),
"effect_outcome_unknown"
);
}

#[test]
fn model_rejection_class_reaches_the_public_result_code() {
// Journaled settlement path: the durable ModelCallRejectionReason must survive to the code.
Expand Down
35 changes: 35 additions & 0 deletions crates/xgeny-provider-openai/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2308,6 +2308,41 @@ mod tests {
);
}

#[test]
fn assistant_reasoning_fields_are_ignored_on_both_decode_paths() {
// Ollama returns `reasoning`, vLLM/llama.cpp return `reasoning_content` next to `content`
// for thinking models. Both are provider-side fields, not part of the strict document.
for field in ["reasoning", "reasoning_content"] {
let planner_body = serde_json::to_vec(&json!({
"model": MODEL,
"choices": [{
"index": 0,
"message": {"role": "assistant", "content": valid_plan(), field: "…thinking…"},
"finish_reason": "stop"
}]
}))
.unwrap();
assert!(
decode_chat_response(&planner_body, MODEL, 1 << 16, 8).is_ok(),
"planner path must ignore `{field}`"
);
let probe_body = serde_json::to_vec(&json!({
"model": MODEL,
"choices": [{
"index": 0,
"message": {"role": "assistant", "content": COMPLETION_OK, field: "…thinking…"},
"finish_reason": "stop"
}]
}))
.unwrap();
assert_eq!(
decode_compatibility_response(&probe_body, MODEL, 8),
Ok(()),
"probe path must ignore `{field}`"
);
}
}

#[test]
fn compatibility_probe_requests_the_configured_output_budget() {
let config = config("https://provider.example/v1")
Expand Down
82 changes: 79 additions & 3 deletions crates/xgeny-provider-openai/tests/http_contract.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,9 @@ use xgeny_workgraph::{
AgentLoopBudget, AgentLoopState, AuthorizationBinding, AuthorizationUse,
CompletionOutputRecord, EffectClass as WorkEffectClass, EffectIntent, EventRecord,
InvocationBinding, ModelCallBudget, ModelCallLifecycleState, ModelCallRejectionReason,
ModelCallReservation, ModelCallSettlement, ReceiptPlacement, ReceiptProvenance,
ReconstructableMaterialReference, RunEvent, RunEventBody, RunState, SinkGuarantee, StepState,
StepStatus, TOOL_OUTPUT_PROFILE_V1, ToolOutputRecord, apply_record,
ModelCallReservation, ModelCallSettlement, ModelCallUnknownReason, ReceiptPlacement,
ReceiptProvenance, ReconstructableMaterialReference, RunEvent, RunEventBody, RunState,
SinkGuarantee, StepState, StepStatus, TOOL_OUTPUT_PROFILE_V1, ToolOutputRecord, apply_record,
};

const AUTHORITY: &str = "local:test";
Expand Down Expand Up @@ -804,6 +804,82 @@ fn deterministic_provider_rejection_is_closed_without_raw_error_body() {
assert!(!durable.contains(RAW_RESPONSE_SENTINEL));
}

/// A provider that accepts the request and then stalls longer than the planner budget.
fn spawn_stalling_server(stall: Duration) -> (String, thread::JoinHandle<()>) {
let listener = TcpListener::bind("127.0.0.1:0").expect("test listener should bind");
let address = listener
.local_addr()
.expect("listener address should resolve");
let handle = thread::spawn(move || {
let (mut stream, _) = listener.accept().expect("one request should connect");
let _request = read_http_request(&mut stream);
thread::sleep(stall);
// Whatever we write now is too late; the client has already given up.
let _ =
stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n");
});
(format!("http://{address}/v1"), handle)
}

#[test]
fn slow_provider_becomes_a_durable_timeout_without_retry() {
// Measured in the wild: a 27B local model needs ~60s per planner call; when the configured
// budget is shorter the call must close as model_call_unknown(timeout), not as a rejection,
// and the client must not retry.
let (base_url, handle) = spawn_stalling_server(Duration::from_secs(2));
let config = OpenAiPlannerConfig::new(
&base_url,
"xgeny.test.go50902",
"qwen3.8-27b",
"Qwen/Qwen3.8-27B-FP8",
)
.expect("planner config should validate")
.with_max_output_tokens(512)
.expect("output limit should validate")
.with_timeout(Duration::from_secs(1))
.expect("timeout should validate");
let mut planner = OpenAiPlanner::new(config, None).expect("planner should build");
let mut store = seed_store();
let loop_runtime = configured_loop(&mut store, &mut planner);
let mut events = DeterministicEvents;
let mut materializer = EphemeralMaterializer;
let started = std::time::Instant::now();
let tick = loop_runtime
.tick(
&mut store,
&mut events,
&FixedLease,
&synthetic_registry(),
&IdentityResolver::default(),
&mut planner,
&mut materializer,
)
.expect("timeout should settle durably");
assert!(
started.elapsed() < Duration::from_secs(2),
"client must give up at its own budget, not wait for the provider"
);
// The first tick after a transport timeout reports the port failure; the reservation is
// durably marked Unknown(timeout) so a later tick or resume never replays the call.
assert!(matches!(
tick,
AgentLoopTick::PlannerUnavailable {
failure: PlannerPortFailure::Timeout,
..
}
));
let snapshot = store.load().unwrap().unwrap();
let last = snapshot.records.last().expect("unknown event should exist");
assert!(matches!(
&last.event.body,
RunEventBody::ModelCallBecameUnknown {
reason: ModelCallUnknownReason::Timeout,
..
}
));
handle.join().expect("server should finish");
}

#[test]
#[ignore = "requires an explicitly configured OpenAI-compatible model endpoint"]
fn live_go50902_plan_smoke() {
Expand Down
5 changes: 3 additions & 2 deletions docs/getting-started.md
Original file line number Diff line number Diff line change
Expand Up @@ -439,8 +439,9 @@ profile 저장소를 격리하려면 `XGENY_CONFIG_HOME`을 따로 설정한다.
| `proposal_rejected.*` | 뒤의 class가 Core가 제안을 거부한 이유다. `capability_unavailable`/`capability_unsupported`는 허용하지 않은 capability 선택, `invocation_invalid`는 scope 밖 인자나 스키마 위반, `tool_call_budget_exhausted`는 예산 소진이다. Class는 Core 판정이며 model 출력 원문이 아니다. |
| `model_rejected.*` | 뒤의 class는 journal의 model call settlement와 같은 값이다. `planner_invalid_response`는 provider가 strict JSON Schema를 지키지 않은 응답(문법 미지원·미적용), `provider_limit`은 출력 예산·요청 크기 초과, `provider_rejected`는 4xx 거부다. Class는 Core 판정이며 model 출력 원문이 아니다. |
| `configuration_mismatch` | 원래 workspace, file/directory scope, executable와 model profile binding(inference timeout·출력 예산 포함)으로 resume한다. 자동 대체하지 말고 필요하면 새 Run을 시작한다. |
| `model_call_unknown`이 planner 호출마다 반복 | 프로필의 inference timeout이 model·hardware에 비해 짧다. 로컬 27B는 호출당 60초 안팎이 걸리므로 `--inference-timeout`을 올린다. |
| `model_call_unknown` 또는 `effect_outcome_unknown` | 불확정 작업을 자동 반복하지 않는다. `/status`와 `/resume`의 고정 진단을 확인하고 외부 상태를 별도로 검증한다. |
| `model_call_unknown.timeout`이 planner 호출마다 반복 | 프로필의 inference timeout이 model·hardware에 비해 짧다. 로컬 27B는 호출당 60초 안팎이 걸리므로 `--inference-timeout`을 올린다. |
| `model_call_unknown.transport_unavailable` / `.interrupted` | 요청이 전송됐을 수 있으나 결과를 못 받았다(연결 끊김) 또는 process가 응답 전에 끝났다. 자동 replay하지 않으므로 endpoint 상태를 확인한 뒤 `resume`한다. |
| `model_call_unknown.*` 또는 `effect_outcome_unknown` | 불확정 작업을 자동 반복하지 않는다. `/status`와 `/resume`의 고정 진단을 확인하고 외부 상태를 별도로 검증한다. |

지원 요청에는 `xgeny --version`, OS/architecture, 설치 채널, 종료 코드와 고정된 오류 코드만 우선 제공한다.
API key, endpoint 전체 URL, prompt, model 원문 응답, source, process stdout/stderr, state DB와 Run ID는 공개
Expand Down