From 537fd8c9604bd73ce38cfa67a906f58481f8e010 Mon Sep 17 00:00:00 2001 From: Michael Roy Date: Wed, 12 Aug 2026 12:17:16 -0700 Subject: [PATCH 01/19] feat: make model serving ports collision-safe Signed-off-by: Michael Roy --- apps/rocm/src/main.rs | 265 ++-- apps/rocm/src/serve_cmd.rs | 13 +- apps/rocm/src/serve_port.rs | 478 ++++++ apps/rocm/src/service_launch.rs | 827 +++++++++- apps/rocmd/src/watchers.rs | 212 ++- crates/rocm-core/src/lib.rs | 389 ++++- crates/rocm-dash-tui/src/app/mod.rs | 4 +- crates/rocm-dash-tui/src/ui/serve_wizard.rs | 1573 +++++++++++++++---- crates/rocm-dash-tui/src/ui/tabs/serving.rs | 30 +- 9 files changed, 3283 insertions(+), 508 deletions(-) create mode 100644 apps/rocm/src/serve_port.rs diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index f1c489ce9..739ad12f5 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -32,6 +32,7 @@ mod runtime_install_finalize; mod runtime_services; mod runtimes_cmd; mod serve_cmd; +mod serve_port; mod serve_summary; mod service_hints; mod service_launch; @@ -502,9 +503,16 @@ rocm serve qwen --verbose --device gpu_required")] /// Host address to bind. #[arg(long, default_value = DEFAULT_LOCAL_HOST)] host: String, - /// TCP port to bind. - #[arg(long, default_value_t = rocm_core::DEFAULT_LOCAL_PORT)] - port: u16, + /// TCP port to bind. Omit it to choose an available port automatically, + /// starting at 11435, on the default 127.0.0.1 host. Any other --host + /// requires an explicit --port: proactive collision detection is claimed + /// only for local 127.0.0.1 serving. + // `allow_hyphen_values` so a negative value such as `-1` reaches + // `parse_serve_port` and gets the domain error naming the valid range, + // instead of clap rejecting it as an unknown flag. Mirrors the same + // choice made for `--gpu-memory-utilization` below. + #[arg(long, value_parser = serve_port::parse_serve_port, allow_hyphen_values = true)] + port: Option, /// Attach to the server in this terminal and stream its logs (Ctrl-D to /// detach and leave it running, Ctrl-C to stop). Same as --verbose. #[arg(long, conflicts_with = "managed")] @@ -8177,30 +8185,29 @@ pub(crate) fn managed_service_is_live(record: &ManagedServiceRecord) -> bool { } /// Idempotency guard for managed launches: returns an already-live managed -/// service for this engine+model, if any. The caller compares its effective -/// launch recipe before deciding whether reuse is safe. +/// service for this engine+model, if any, from an already-refreshed live set. +/// The caller compares its effective launch recipe before deciding whether reuse +/// is safe. /// /// Keyed on `(engine, canonical_model_id)` — NOT `service_id`. `generate_service_id` /// embeds `unix_time_millis()`, so every launch mints a unique id; matching on it -/// would never catch a duplicate. `load_managed_services` refreshes liveness, so +/// would never catch a duplicate. [`live_managed_services`] refreshes liveness, so /// stale manifests (dead PIDs) demote to "stopped" and are skipped, letting a /// genuine relaunch proceed. Records are sorted newest-first, so `find` returns /// the newest live match. Prevents a second `serve --managed` for the same /// engine+model from spawning a duplicate process once the TUI job-bridge guard /// has cleared. -fn existing_live_managed_service( - paths: &AppPaths, +/// +/// Takes the snapshot rather than the paths so the allocation transaction loads +/// it once and uses it for both this check and the port reservations: duplicate +/// detection and candidate selection can never observe different states. +pub(crate) fn existing_live_managed_service_in<'a>( + live: &'a [ManagedServiceRecord], engine: &str, canonical_model_id: &str, -) -> Option { - load_managed_services(paths) - .ok()? - .into_iter() - .find(|record| { - record.engine == engine - && record.canonical_model_id == canonical_model_id - && managed_service_is_live(record) - }) +) -> Option<&'a ManagedServiceRecord> { + live.iter() + .find(|record| record.engine == engine && record.canonical_model_id == canonical_model_id) } const SERVICE_LIVENESS_CHECK_TIMEOUT: Duration = Duration::from_millis(750); @@ -12565,6 +12572,21 @@ pub(crate) mod tests { .collect() } + fn serve_arg_help(arg_id: &str) -> String { + let mut cli = Cli::command(); + let serve = cli + .find_subcommand_mut("serve") + .expect("serve subcommand exists"); + let arg = serve + .get_arguments() + .find(|arg| arg.get_id().as_str() == arg_id) + .unwrap_or_else(|| panic!("serve has no `{arg_id}` argument")); + arg.get_long_help() + .map(ToString::to_string) + .or_else(|| arg.get_help().map(ToString::to_string)) + .unwrap_or_default() + } + // Both `--engine` and `--device` restrict their input to a fixed set via a // clap `value_parser`/`ValueEnum`, so invalid input is rejected as a usage // error (exit code 2) with the accepted choices listed. Values are advertised @@ -12772,6 +12794,76 @@ pub(crate) mod tests { } } + #[test] + fn serve_without_port_flag_requests_automatic_selection() { + let cli = parse_serve(&[]).expect("bare serve parses"); + match cli.command { + Some(Command::Serve { port, host, .. }) => { + assert_eq!(port, None, "omitted --port must stay unresolved"); + assert_eq!(host, DEFAULT_LOCAL_HOST); + assert!(serve_port::PortRequest::from_flag(port).is_auto()); + } + other => panic!("expected Serve, got {other:?}"), + } + } + + #[test] + fn serve_with_explicit_port_keeps_it_concrete() { + let cli = parse_serve(&["--port", "8000"]).expect("explicit port parses"); + match cli.command { + Some(Command::Serve { port, .. }) => { + assert_eq!(port, Some(8000)); + assert_eq!( + serve_port::PortRequest::from_flag(port), + serve_port::PortRequest::Explicit(8000) + ); + } + other => panic!("expected Serve, got {other:?}"), + } + } + + #[test] + fn serve_rejects_a_zero_or_out_of_range_port() { + for bad in ["0", "65536", "-1", "auto"] { + let error = parse_serve(&["--port", bad]).expect_err("must be rejected"); + assert!( + error.to_string().contains("between 1 and 65535"), + "unexpected error for --port {bad}: {error}" + ); + } + } + + #[test] + fn hidden_engine_serve_command_keeps_a_concrete_default_port() { + // Only the user-facing verb gained automatic selection; the direct engine + // entry point still requires (and defaults to) a concrete port. + let cli = Cli::try_parse_from([ + "rocm", + "__engine-serve-http", + "lemonade", + "svc-1", + "qwen", + "--state-path", + "state.json", + ]) + .expect("hidden engine command parses"); + match cli.command { + Some(Command::EngineServeHttp { port, host, .. }) => { + assert_eq!(port, rocm_core::DEFAULT_LOCAL_PORT); + assert_eq!(host, DEFAULT_LOCAL_HOST); + } + other => panic!("expected EngineServeHttp, got {other:?}"), + } + } + + #[test] + fn serve_port_help_documents_omission_and_the_custom_host_rule() { + let help = serve_arg_help("port"); + assert!(help.contains("11435"), "{help}"); + assert!(help.contains("automatically"), "{help}"); + assert!(help.contains("127.0.0.1"), "{help}"); + } + #[test] fn serve_verbose_conflicts_with_managed() { // `--verbose` streams logs in the foreground; a backgrounded managed @@ -16613,145 +16705,6 @@ pub(crate) mod tests { Ok(()) } - #[test] - fn duplicate_managed_launch_detected_across_distinct_service_ids() -> Result<()> { - // `generate_service_id` embeds a timestamp, so a second launch for the - // same engine+model has a DIFFERENT service_id. The guard must still - // detect the live service by (engine, canonical_model_id), and return - // the newest live match. `starting` skips the endpoint probe; the - // current process id is a guaranteed-live PID. - let (root, paths) = test_paths("dup-managed-distinct-ids"); - paths.ensure()?; - - // Older, dead manifest for the same engine+model (distinct service_id). - let mut dead = ManagedServiceRecord::new( - &paths, - "lemonade-qwen-1000", - "lemonade", - "qwen", - "qwen-canonical", - "127.0.0.1", - 11500, - "managed", - 999_999_999, - None, - None, - None, - ); - dead.status = "ready".to_owned(); - dead.engine_pid = Some(999_999_999); - dead.created_at_unix_ms = 1000; - dead.write()?; - - // Newer, live manifest for the same engine+model (distinct service_id). - let mut live = ManagedServiceRecord::new( - &paths, - "lemonade-qwen-2000", - "lemonade", - "qwen", - "qwen-canonical", - "127.0.0.1", - 11501, - "managed", - std::process::id(), - None, - None, - None, - ); - live.status = "starting".to_owned(); - live.engine_pid = Some(std::process::id()); - live.created_at_unix_ms = 2000; - live.write()?; - - let found = existing_live_managed_service(&paths, "lemonade", "qwen-canonical"); - let _ = fs::remove_dir_all(root); - - let found = found.expect("a live managed service should be detected by engine+model"); - assert_eq!( - found.service_id, "lemonade-qwen-2000", - "should return the newest live match" - ); - assert!(managed_service_is_live(&found)); - Ok(()) - } - - #[test] - fn dead_managed_service_allows_relaunch() -> Result<()> { - // A stale manifest with dead PIDs must NOT block a relaunch: liveness - // refresh demotes it to "stopped", so the guard returns None. - let (root, paths) = test_paths("dup-managed-dead"); - paths.ensure()?; - let mut record = ManagedServiceRecord::new( - &paths, - "lemonade-qwen-3000", - "lemonade", - "qwen", - "qwen-canonical", - "127.0.0.1", - 11502, - "managed", - 999_999_999, - None, - None, - None, - ); - record.status = "ready".to_owned(); - record.engine_pid = Some(999_999_999); - record.write()?; - - let found = existing_live_managed_service(&paths, "lemonade", "qwen-canonical"); - let _ = fs::remove_dir_all(root); - - assert!( - found.is_none(), - "a dead managed service must not block relaunch" - ); - Ok(()) - } - - #[test] - fn live_service_for_other_model_does_not_block() -> Result<()> { - // A live service for a DIFFERENT model must not match — the guard keys - // on the model, not just the engine. - let (root, paths) = test_paths("dup-managed-other-model"); - paths.ensure()?; - let mut record = ManagedServiceRecord::new( - &paths, - "lemonade-other-1", - "lemonade", - "other", - "other-canonical", - "127.0.0.1", - 11503, - "managed", - std::process::id(), - None, - None, - None, - ); - record.status = "starting".to_owned(); - record.engine_pid = Some(std::process::id()); - record.write()?; - - let found = existing_live_managed_service(&paths, "lemonade", "qwen-canonical"); - let _ = fs::remove_dir_all(root); - - assert!( - found.is_none(), - "a live service for a different model must not match" - ); - Ok(()) - } - - #[test] - fn missing_manifest_allows_launch() { - // No services dir / manifests → nothing to detect, launch proceeds. - let (root, paths) = test_paths("dup-managed-missing"); - let found = existing_live_managed_service(&paths, "lemonade", "qwen-canonical"); - let _ = fs::remove_dir_all(root); - assert!(found.is_none()); - } - #[test] fn service_actions_require_yes_and_render_sandbox_result() { let (_root, paths) = test_paths("services-action-approval"); diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index 275115f2a..c1fb124ba 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -38,6 +38,7 @@ use rocm_engine_protocol::{ use crate::DevicePolicyArg; use crate::endpoint_keys; use crate::engines_cmd::{engine_manages_own_runtime, ensure_self_managed_engine_ready}; +use crate::serve_port; use crate::serve_summary; use crate::service_launch::{ drop_orphaned_endpoint_key_on_already_running, ensure_background_helper_running_quiet, @@ -501,7 +502,7 @@ pub(crate) struct ServeArgs { pub(crate) runtime_id: Option, pub(crate) env_id: Option, pub(crate) host: String, - pub(crate) port: u16, + pub(crate) port: Option, pub(crate) foreground: bool, pub(crate) managed: bool, pub(crate) verbose: bool, @@ -539,6 +540,7 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { max_tokens, api_key, } = args; + let port_request = serve_port::PortRequest::from_flag(port); let _ = managed; // background is now the default; --managed is accepted as an explicit synonym. validate_bind_host(&host, allow_public_bind)?; // Loopback stays credential-free; a public bind must be authenticated. Resolve @@ -780,7 +782,10 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { println!(" engine: {selected_engine}"); println!("{}", serve_engine_selection_line(&serve_engine)); println!(" host: {host}"); - println!(" port: {port}"); + match port { + Some(port) => println!(" port: {port}"), + None => println!(" {}", serve_port::AUTOMATIC_PORT_DISCLOSURE), + } if let Some(runtime_id) = resolved_selection.runtime_id.as_deref() { println!(" runtime_id: {runtime_id}"); } @@ -863,7 +868,7 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { &model, &resolve, &host, - port, + port_request, &resolve.device_policy, &gpu_indices, managed_runtime_id.as_deref(), @@ -949,7 +954,7 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { &model, &resolve, &host, - port, + port_request, &gpu_indices, resolved_selection.runtime_id.as_deref(), resolved_selection.env_id.as_deref(), diff --git a/apps/rocm/src/serve_port.rs b/apps/rocm/src/serve_port.rs new file mode 100644 index 000000000..e99b47334 --- /dev/null +++ b/apps/rocm/src/serve_port.rs @@ -0,0 +1,478 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Central serve-port policy for the user-facing `rocm serve` verb. +//! +//! `--port` is an explicit-or-automatic *request*, not a concrete default. This +//! module turns that request into one concrete, leased port before any manifest, +//! log, endpoint key, or child process exists: +//! +//! - **Automatic** (`--port` omitted) is accepted only on the canonical default +//! loopback host `127.0.0.1`, and scans [`AUTO_PORT_FIRST`]..=[`AUTO_PORT_LAST`] +//! for a candidate that is neither reserved by a live managed service nor +//! occupied on the OS. +//! - **Explicit on canonical loopback** goes through the same reservation and +//! OS preflight, so a collision fails immediately with a clear message instead +//! of surfacing as an engine bind error minutes into startup. +//! - **Explicit on a custom host** (`localhost`, `::1`, `0.0.0.0`, any other +//! IPv4/IPv6 address) is passed through untouched: bind success or failure +//! stays engine-owned, because an IPv4-loopback probe proves nothing about +//! those addresses. Automatic is refused there rather than guessing. +//! +//! Everything downstream of this module — engine requests, service records, +//! readiness, supervision, recovery — keeps its concrete `u16` contract. + +use anyhow::{Result, bail}; +use rocm_core::{DEFAULT_LOCAL_HOST, DEFAULT_LOCAL_PORT, LoopbackPortLease}; +use std::io; + +/// First automatic candidate. Also the legacy well-known local endpoint, so an +/// otherwise-idle machine keeps serving on the port users already know. +pub(crate) const AUTO_PORT_FIRST: u16 = DEFAULT_LOCAL_PORT; +/// Number of candidates *after* the first one. The scanned range is inclusive on +/// both ends: `11435..=11535`, 101 candidates. +pub(crate) const AUTO_PORT_SPAN: u16 = 100; +/// Last automatic candidate. +pub(crate) const AUTO_PORT_LAST: u16 = AUTO_PORT_FIRST + AUTO_PORT_SPAN; + +/// The one phrase used wherever a port must be described *before* it is +/// resolved. The CLI plan output and the Dash approval card share it verbatim so +/// neither promises a concrete port it has not yet acquired. +pub(crate) const AUTOMATIC_PORT_DISCLOSURE: &str = "Port: automatic; endpoint shown after launch"; + +/// What the user asked for on the command line. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum PortRequest { + /// `--port` omitted: choose an available local port. + Auto, + /// `--port N`: use exactly this port. + Explicit(u16), +} + +impl PortRequest { + #[must_use] + pub(crate) const fn from_flag(port: Option) -> Self { + match port { + Some(port) => Self::Explicit(port), + None => Self::Auto, + } + } + + #[must_use] + pub(crate) const fn is_auto(self) -> bool { + matches!(self, Self::Auto) + } + + /// The concrete port, when the user named one. + #[must_use] + pub(crate) const fn explicit(self) -> Option { + match self { + Self::Auto => None, + Self::Explicit(port) => Some(port), + } + } +} + +/// A port a live managed service still owns. Carried with its service id so a +/// collision names the thing actually holding the port. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct ReservedPort { + pub(crate) port: u16, + pub(crate) service_id: String, + pub(crate) status: String, +} + +/// clap value parser for every user-facing `--port`. +/// +/// Rejects `0` explicitly: the OS reads it as "any ephemeral port", which would +/// silently produce an endpoint nobody asked for. +/// +/// # Errors +/// Returns a user-facing message for anything outside 1–65535. +pub(crate) fn parse_serve_port(value: &str) -> Result { + match value.trim().parse::() { + Ok(0) | Err(_) => Err(format!( + "port must be a number between 1 and 65535, got '{value}'" + )), + Ok(port) => Ok(port), + } +} + +/// Reject an automatic request on a host this policy cannot preflight, before +/// any engine resolution or side effect happens. +/// +/// # Errors +/// Fails when `--port` was omitted for anything but the canonical loopback host. +pub(crate) fn validate_port_request(host: &str, request: PortRequest) -> Result<()> { + if request.is_auto() && !rocm_core::is_canonical_loopback_host(host) { + bail!( + "`rocm serve --host {host}` needs an explicit `--port <1-65535>`. Automatic port \ + selection is supported only on the default host {DEFAULT_LOCAL_HOST}, where ROCm \ + can prove a port is free before launching; on any other address the engine owns \ + the bind." + ); + } + Ok(()) +} + +/// Production bind probe: take a real `127.0.0.1:` lease. +/// +/// # Errors +/// Propagates the OS bind error verbatim so the caller can distinguish +/// "occupied" from "not permitted"/"not available". +pub(crate) fn loopback_bind_probe(port: u16) -> io::Result { + rocm_core::lease_loopback_port(port) +} + +/// Turn a [`PortRequest`] into one concrete, leased port. +/// +/// Callers must hold the shared managed-service allocation lock across this call +/// *and* the record publication that follows it, so a concurrent launch cannot +/// pick the same candidate between the probe and the spawn. +/// +/// `probe` is injected so tests can drive deterministic OS outcomes; production +/// passes [`loopback_bind_probe`]. +/// +/// # Errors +/// Fails when an explicit port is reserved or occupied, when the automatic range +/// is exhausted, or when the OS reports anything other than "address in use" — +/// a permission or address-availability error must never be papered over by +/// quietly choosing a different port. +pub(crate) fn resolve_serve_port( + host: &str, + request: PortRequest, + reserved: &[ReservedPort], + probe: &mut dyn FnMut(u16) -> io::Result, +) -> Result { + if !rocm_core::is_canonical_loopback_host(host) { + // Custom host: an explicit port passes through untouched so bind success + // or failure stays engine-owned. Automatic is refused here as well as in + // `serve()`, because this is the shared choke point every caller reaches + // — and it is refused by returning an error, never by panicking. + let Some(port) = request.explicit() else { + validate_port_request(host, request)?; + bail!( + "`--host {host}` needs an explicit `--port <1-65535>`; automatic port \ + selection is supported only on {DEFAULT_LOCAL_HOST}." + ); + }; + return Ok(LoopbackPortLease::engine_owned(port)); + } + + match request { + PortRequest::Explicit(port) => { + if let Some(owner) = reserved.iter().find(|entry| entry.port == port) { + bail!( + "{DEFAULT_LOCAL_HOST}:{port} is already reserved by managed service {} \ + ({}). Stop it with `rocm services stop {} --yes`, or pass a different \ + `--port`.", + owner.service_id, + owner.status, + owner.service_id + ); + } + probe(port).map_err(|error| explicit_bind_error(port, &error)) + } + PortRequest::Auto => { + for port in AUTO_PORT_FIRST..=AUTO_PORT_LAST { + if reserved.iter().any(|entry| entry.port == port) { + continue; + } + match probe(port) { + Ok(lease) => return Ok(lease), + // Occupied is the only outcome that advances the scan. + Err(error) if error.kind() == io::ErrorKind::AddrInUse => {} + Err(error) => bail!( + "failed to reserve {DEFAULT_LOCAL_HOST}:{port} while choosing an \ + automatic port: {error}" + ), + } + } + bail!( + "no free TCP port for {DEFAULT_LOCAL_HOST} in {AUTO_PORT_FIRST}–{AUTO_PORT_LAST}; \ + every candidate is reserved by a live managed service or already in use. Stop an \ + unused server with `rocm services stop --yes`, or pass an explicit \ + `--port`." + ); + } + } +} + +fn explicit_bind_error(port: u16, error: &io::Error) -> anyhow::Error { + if error.kind() == io::ErrorKind::AddrInUse { + return anyhow::anyhow!( + "{DEFAULT_LOCAL_HOST}:{port} is already in use by another process. Choose a free \ + `--port`, stop the process holding it, or omit `--port` to let ROCm pick an \ + available port starting at {AUTO_PORT_FIRST}." + ); + } + anyhow::anyhow!("failed to reserve {DEFAULT_LOCAL_HOST}:{port}: {error}") +} + +#[cfg(test)] +mod tests { + use super::*; + use std::net::TcpListener; + + fn reserved(port: u16, service_id: &str, status: &str) -> ReservedPort { + ReservedPort { + port, + service_id: service_id.to_owned(), + status: status.to_owned(), + } + } + + /// A probe driven by a table of scripted per-port failures. Any port absent + /// from the table binds successfully. + fn scripted( + outcomes: Vec<(u16, io::ErrorKind)>, + ) -> impl FnMut(u16) -> io::Result { + move |port| match outcomes.iter().find(|(candidate, _)| *candidate == port) { + Some((_, kind)) => Err(io::Error::new(*kind, "scripted")), + None => Ok(LoopbackPortLease::engine_owned(port)), + } + } + + #[test] + fn port_request_distinguishes_omitted_from_explicit() { + assert_eq!(PortRequest::from_flag(None), PortRequest::Auto); + assert!(PortRequest::from_flag(None).is_auto()); + assert_eq!(PortRequest::from_flag(None).explicit(), None); + assert_eq!( + PortRequest::from_flag(Some(8000)), + PortRequest::Explicit(8000) + ); + assert!(!PortRequest::from_flag(Some(8000)).is_auto()); + assert_eq!(PortRequest::from_flag(Some(8000)).explicit(), Some(8000)); + } + + #[test] + fn port_parser_rejects_zero_and_out_of_range_values() { + assert_eq!(parse_serve_port("8000"), Ok(8000)); + assert_eq!(parse_serve_port(" 11435 "), Ok(11_435)); + assert_eq!(parse_serve_port("65535"), Ok(65_535)); + for bad in ["0", "65536", "-1", "eleven", ""] { + let error = parse_serve_port(bad).expect_err("must be rejected"); + assert!( + error.contains("between 1 and 65535"), + "unexpected message for {bad:?}: {error}" + ); + } + } + + #[test] + fn automatic_range_is_exactly_the_documented_window() { + assert_eq!(AUTO_PORT_FIRST, 11_435); + assert_eq!(AUTO_PORT_LAST, 11_535); + assert_eq!((AUTO_PORT_FIRST..=AUTO_PORT_LAST).count(), 101); + } + + #[test] + fn automatic_is_refused_on_every_custom_host() { + validate_port_request(DEFAULT_LOCAL_HOST, PortRequest::Auto).expect("canonical host is ok"); + for host in ["localhost", "::1", "0.0.0.0", "127.0.0.2", "10.0.0.5"] { + let error = validate_port_request(host, PortRequest::Auto) + .expect_err("automatic must be refused on a custom host"); + let message = format!("{error:#}"); + assert!(message.contains("explicit `--port"), "{message}"); + assert!(message.contains(host), "{message}"); + // An explicit port is always accepted there. + validate_port_request(host, PortRequest::Explicit(8000)) + .expect("explicit port is valid on a custom host"); + } + } + + #[test] + fn custom_host_explicit_port_bypasses_the_loopback_preflight() { + let mut probe = |_port: u16| -> io::Result { + panic!("custom hosts must not be preflighted against IPv4 loopback") + }; + // Even a port a live loopback service reserves is passed straight + // through: the engine owns bind on that address. + let lease = resolve_serve_port( + "0.0.0.0", + PortRequest::Explicit(11_435), + &[reserved(11_435, "svc-a", "running")], + &mut probe, + ) + .expect("custom host explicit port is engine-owned"); + assert_eq!(lease.port(), 11_435); + assert!(!lease.is_held()); + } + + #[test] + fn resolving_auto_on_a_custom_host_errors_instead_of_panicking() { + // The choke point must refuse Auto on its own, even when a caller skipped + // the earlier `validate_port_request` gate — as a Result, never a panic. + let mut probe = |_port: u16| -> io::Result { + panic!("a custom host must never be preflighted") + }; + for host in ["localhost", "::1", "0.0.0.0", "10.0.0.5"] { + let error = resolve_serve_port(host, PortRequest::Auto, &[], &mut probe) + .expect_err("automatic selection is not supported on a custom host"); + let message = format!("{error:#}"); + assert!(message.contains("explicit `--port"), "{message}"); + assert!(message.contains(host), "{message}"); + } + } + + #[test] + fn explicit_loopback_port_outside_the_auto_range_is_retained() { + let mut probe = scripted(vec![]); + let lease = resolve_serve_port( + DEFAULT_LOCAL_HOST, + PortRequest::Explicit(8000), + &[], + &mut probe, + ) + .expect("a free explicit port is kept"); + assert_eq!(lease.port(), 8000); + } + + #[test] + fn explicit_loopback_port_that_is_occupied_is_rejected() { + let mut probe = scripted(vec![(8000, io::ErrorKind::AddrInUse)]); + let error = resolve_serve_port( + DEFAULT_LOCAL_HOST, + PortRequest::Explicit(8000), + &[], + &mut probe, + ) + .expect_err("an occupied explicit port must fail"); + let message = format!("{error:#}"); + assert!(message.contains("127.0.0.1:8000"), "{message}"); + assert!(message.contains("already in use"), "{message}"); + } + + #[test] + fn explicit_loopback_port_reserved_by_a_live_service_names_that_service() { + let mut probe = |_port: u16| -> io::Result { + panic!("a reserved port must be rejected before probing") + }; + let error = resolve_serve_port( + DEFAULT_LOCAL_HOST, + PortRequest::Explicit(11_435), + &[reserved(11_435, "vllm-qwen-1", "starting")], + &mut probe, + ) + .expect_err("a reserved explicit port must fail"); + let message = format!("{error:#}"); + assert!(message.contains("vllm-qwen-1"), "{message}"); + assert!(message.contains("starting"), "{message}"); + } + + #[test] + fn automatic_takes_the_first_candidate_when_it_is_free() { + let mut probe = scripted(vec![]); + let lease = resolve_serve_port(DEFAULT_LOCAL_HOST, PortRequest::Auto, &[], &mut probe) + .expect("first candidate is free"); + assert_eq!(lease.port(), AUTO_PORT_FIRST); + } + + #[test] + fn automatic_skips_occupied_candidates_until_one_binds() { + let mut probe = scripted(vec![ + (11_435, io::ErrorKind::AddrInUse), + (11_436, io::ErrorKind::AddrInUse), + (11_437, io::ErrorKind::AddrInUse), + ]); + let lease = resolve_serve_port(DEFAULT_LOCAL_HOST, PortRequest::Auto, &[], &mut probe) + .expect("scan advances past occupied candidates"); + assert_eq!(lease.port(), 11_438); + } + + #[test] + fn automatic_skips_ports_reserved_by_live_records_without_probing_them() { + let mut probed: Vec = Vec::new(); + let lease = { + let mut probe = |port: u16| -> io::Result { + probed.push(port); + Ok(LoopbackPortLease::engine_owned(port)) + }; + resolve_serve_port( + DEFAULT_LOCAL_HOST, + PortRequest::Auto, + &[ + reserved(11_435, "svc-ready", "ready"), + reserved(11_436, "svc-recovering", "recovering"), + ], + &mut probe, + ) + .expect("scan skips reserved candidates") + }; + assert_eq!(lease.port(), 11_437); + assert_eq!(probed, vec![11_437]); + } + + #[test] + fn automatic_stops_immediately_on_a_non_collision_bind_error() { + for kind in [ + io::ErrorKind::PermissionDenied, + io::ErrorKind::AddrNotAvailable, + ] { + let mut probe = scripted(vec![(11_435, kind)]); + let error = resolve_serve_port(DEFAULT_LOCAL_HOST, PortRequest::Auto, &[], &mut probe) + .expect_err("only AddrInUse may advance the scan"); + let message = format!("{error:#}"); + assert!(message.contains("127.0.0.1:11435"), "{message}"); + } + } + + #[test] + fn explicit_non_collision_bind_error_reports_host_port_and_os_detail() { + let mut probe = |_port: u16| -> io::Result { + Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "permission denied", + )) + }; + let error = resolve_serve_port( + DEFAULT_LOCAL_HOST, + PortRequest::Explicit(80), + &[], + &mut probe, + ) + .expect_err("a privileged port must fail"); + let message = format!("{error:#}"); + assert!(message.contains("127.0.0.1:80"), "{message}"); + assert!(message.contains("permission denied"), "{message}"); + } + + #[test] + fn automatic_exhaustion_reports_the_exact_scanned_range() { + let mut probe = |_port: u16| -> io::Result { + Err(io::Error::new(io::ErrorKind::AddrInUse, "occupied")) + }; + let error = resolve_serve_port(DEFAULT_LOCAL_HOST, PortRequest::Auto, &[], &mut probe) + .expect_err("an exhausted range must fail"); + let message = format!("{error:#}"); + assert!(message.contains("11435–11535"), "{message}"); + } + + #[test] + fn automatic_selects_against_real_loopback_listeners() { + // Real sockets, not a script: hold the first two candidates with live + // listeners and prove the production probe walks past them. If either is + // already owned by something outside this test the scenario cannot be set + // up deterministically, so skip rather than assert on a moving target. + let (Ok(first), Ok(second)) = ( + TcpListener::bind(("127.0.0.1", AUTO_PORT_FIRST)), + TcpListener::bind(("127.0.0.1", AUTO_PORT_FIRST + 1)), + ) else { + return; + }; + let mut probe = loopback_bind_probe; + let lease = resolve_serve_port(DEFAULT_LOCAL_HOST, PortRequest::Auto, &[], &mut probe) + .expect("a candidate above the held ones is free"); + assert!( + lease.port() >= AUTO_PORT_FIRST + 2, + "selected {} while {AUTO_PORT_FIRST} and {} were held", + lease.port(), + AUTO_PORT_FIRST + 1 + ); + assert!(lease.is_held(), "the production probe must hold the socket"); + // Guards outlive every assertion above. + drop((first, second)); + } +} diff --git a/apps/rocm/src/service_launch.rs b/apps/rocm/src/service_launch.rs index 4e71cf58b..c1786f583 100644 --- a/apps/rocm/src/service_launch.rs +++ b/apps/rocm/src/service_launch.rs @@ -47,12 +47,13 @@ use rocm_engine_protocol::{DevicePolicy, EngineRecipeHint, ResolveModelResponse} use crate::endpoint_keys; use crate::engines_cmd::env_root_for_service; +use crate::serve_port; use crate::service_hints; use crate::{ SandboxToolArg, builtin_engine_serve_http_args, device_policy_name, - existing_live_managed_service, managed_service_launcher_path, record_cli_audit_event, - render_service_action_result, run_internal_sandbox_tool, status_for_readiness, - wait_for_service_http_ready_with_progress, + existing_live_managed_service_in, load_managed_services, managed_service_is_live, + managed_service_launcher_path, record_cli_audit_event, render_service_action_result, + run_internal_sandbox_tool, status_for_readiness, wait_for_service_http_ready_with_progress, }; #[cfg(windows)] use crate::{app_path_env_var_refs, app_path_env_var_values}; @@ -307,11 +308,132 @@ enum ManagedSpawn { }, } +/// Outcome of the locked allocation step. +/// +/// `Debug` so tests can assert on `Result::expect_err`. +#[derive(Debug)] +enum ManagedPortDecision { + /// An equivalent managed service is already live; nothing may be spawned. + /// Boxed to keep the variants a similar size (clippy::large_enum_variant). + AlreadyLive(Box), + /// The concrete port this launch owns until it hands it to the child. + Lease(rocm_core::LoopbackPortLease), +} + +/// Live managed services, with liveness refreshed by [`load_managed_services`] +/// (dead PIDs demote to "stopped" and therefore neither satisfy the duplicate +/// check nor reserve a port). +/// +/// A read failure degrades to "nothing is live": the OS preflight still guards +/// the port, and refusing to serve because the services directory could not be +/// listed would be a worse failure than launching. +fn live_managed_services(paths: &AppPaths) -> Vec { + load_managed_services(paths) + .unwrap_or_default() + .into_iter() + .filter(managed_service_is_live) + .collect() +} + +/// Ports that live managed services still own. +/// +/// Conservative by design: a record reserves its port regardless of the host it +/// recorded, because a `0.0.0.0` (or `localhost`) listener also blocks the +/// loopback address a new local server would bind. Over-reserving costs one +/// candidate in a 101-port scan; under-reserving costs a collision. +fn reserved_service_ports(live: &[ManagedServiceRecord]) -> Vec { + live.iter() + .map(|record| serve_port::ReservedPort { + port: record.port, + service_id: record.service_id.clone(), + status: record.status.clone(), + }) + .collect() +} + +/// The decision half of the launch transaction: refresh liveness, apply the +/// `(engine, canonical_model_id)` idempotency guard, then lease a concrete port +/// against both live records and the real OS. +/// +/// Callers MUST hold the shared allocation lock across this call *and* the +/// record publication that follows, or two launches can pick the same candidate. +/// `probe` is injected so tests can drive deterministic bind outcomes without a +/// GPU or a spawn. +fn resolve_managed_port_in_transaction( + paths: &AppPaths, + engine: &str, + canonical_model_id: &str, + host: &str, + request: serve_port::PortRequest, + probe: &mut dyn FnMut(u16) -> std::io::Result, +) -> Result { + let live = live_managed_services(paths); + // Idempotency first: an already-live equivalent service means no port is + // needed at all, so it must be answered before any candidate is leased. + if let Some(existing) = existing_live_managed_service_in(&live, engine, canonical_model_id) { + return Ok(ManagedPortDecision::AlreadyLive(Box::new(existing.clone()))); + } + let reserved = reserved_service_ports(&live); + serve_port::resolve_serve_port(host, request, &reserved, probe).map(ManagedPortDecision::Lease) +} + +/// Grace given to a child that must be terminated because its record could not +/// be published. Short: the child is milliseconds old and has nothing to flush. +const MANAGED_LAUNCH_CLEANUP_GRACE: Duration = Duration::from_secs(2); + +/// Undo a failed managed launch attempt before the allocation lock is released, +/// so no failed transaction leaves a live reservation or a stray endpoint key. +/// +/// Order matters: a spawned child is terminated and confirmed dead *first*, so +/// nothing is still holding the port or writing to the log when the artifacts — +/// including the record that reserves the port for other launches — are removed. +/// Every step is best-effort and idempotent; cleanup must never mask the +/// original failure with a second one. +fn abandon_failed_managed_launch( + paths: &AppPaths, + record: &ManagedServiceRecord, + child_pid: Option, +) { + if let Some(pid) = child_pid.filter(|pid| *pid != 0) { + let outcome = rocm_core::terminate_verified( + &rocm_core::ProcessIdentity::capture(pid), + rocm_core::KillScope::Tree, + MANAGED_LAUNCH_CLEANUP_GRACE, + true, + ); + record_cli_audit_event( + paths, + "service", + "managed_service_launch_cleanup", + "warn", + format!( + "terminated engine child pid={pid} after a failed launch publication outcome={}", + outcome.as_str() + ), + Some(&record.service_id), + ); + } + for path in [ + record.manifest_path.as_path(), + record.log_path.as_path(), + record.engine_state_path.as_path(), + ] { + let _ = fs::remove_file(path); + } + endpoint_keys::clear_endpoint_api_key(paths, &record.service_id); +} + /// Spawn the detached engine child shared by the managed (background) and /// attached (`--verbose`/`--foreground`) serve paths. Returns before the HTTP /// readiness wait; callers decide whether to block on readiness /// ([`start_managed_service`]) or start tailing the log immediately /// ([`run_attached_service`]). +/// +/// The whole body is one serialized transaction under the shared allocation +/// lock: duplicate detection, live-reservation refresh, candidate leasing, +/// manifest/log creation, argv preparation, the bounded immediate-exit check, +/// and record publication. Readiness deliberately runs after the lock is +/// released. #[allow(clippy::too_many_arguments)] fn spawn_managed_engine_child( paths: &AppPaths, @@ -320,7 +442,7 @@ fn spawn_managed_engine_child( requested_model: &str, resolve: &ResolveModelResponse, host: &str, - port: u16, + port_request: serve_port::PortRequest, device_policy: &DevicePolicy, gpu_indices: &[u32], runtime_id: Option<&str>, @@ -331,6 +453,12 @@ fn spawn_managed_engine_child( paths.ensure()?; fs::create_dir_all(paths.services_dir())?; + // Concurrent `rocm serve` launches and `rocmd` recovery all contend here, so + // no two of them can choose the same port or publish competing records for + // one engine+model. Bounded: a stuck peer produces an actionable error + // rather than a hang. Released when this guard drops, and by process death. + let _allocation = rocm_core::lock_service_allocation(paths)?; + // Idempotency guard: if a managed service for this engine+model is already // alive, surface it and spawn nothing. A second `serve --managed` (e.g. the // chat assistant re-issuing the same request) is treated as satisfied, not @@ -341,63 +469,72 @@ fn spawn_managed_engine_child( .map(serde_json::to_string) .transpose() .context("failed to encode engine recipe hint")?; - if let Some(existing) = - existing_live_managed_service(paths, engine, &resolve.canonical_model_id) - { - if existing.engine_recipe_json != requested_recipe_json { - bail!( - "managed service `{}` is already running for engine `{engine}` and model `{}` with different serve options (recipe hint, tool-call parser, or generation defaults); stop it and run `rocm serve` again to apply the requested options", - existing.service_id, - resolve.canonical_model_id - ); - } - // Reuse cannot satisfy a demand for auth the running server never got. - // The engine reads its key once, at launch, from the environment this - // function builds below — so a server started without one keeps serving - // anonymously no matter what is written afterwards. Upgrading the record - // here would be worse than doing nothing: the record would claim auth - // that the live process does not enforce, and - // `ensure_public_service_has_endpoint_key` would pass on the strength of - // a key file nothing reads. - // - // This is what `rocm remote serve` relies on. It publishes a loopback - // port onto the tailnet and prints "the API key above is what stops - // anyone else calling it". Reusing an unauthenticated service silently - // would make that sentence false about an endpoint the whole tailnet can - // reach. Refusing is the only answer that fails closed, and it is the - // same shape as the recipe mismatch above. - if require_api_key && !existing.requires_api_key { - bail!( - "managed service `{}` is already running for engine `{engine}` and model `{}` \ - without authentication, and a running server cannot be given a key it did not \ - start with; stop it with `{}` and run the command again to \ - serve it with `--require-api-key`", - existing.service_id, - resolve.canonical_model_id, - service_hints::service_stop_hint(&existing.service_id) + let lease = match resolve_managed_port_in_transaction( + paths, + engine, + &resolve.canonical_model_id, + host, + port_request, + &mut serve_port::loopback_bind_probe, + )? { + ManagedPortDecision::AlreadyLive(existing) => { + let existing = *existing; + if existing.engine_recipe_json != requested_recipe_json { + bail!( + "managed service `{}` is already running for engine `{engine}` and model `{}` with different serve options (recipe hint, tool-call parser, or generation defaults); stop it and run `rocm serve` again to apply the requested options", + existing.service_id, + resolve.canonical_model_id + ); + } + // Reuse cannot satisfy a demand for auth the running server never got. + // The engine reads its key once, at launch, from the environment this + // function builds below — so a server started without one keeps serving + // anonymously no matter what is written afterwards. Upgrading the record + // here would be worse than doing nothing: the record would claim auth + // that the live process does not enforce, and + // `ensure_public_service_has_endpoint_key` would pass on the strength of + // a key file nothing reads. + // + // This is what `rocm remote serve` relies on. It publishes a loopback + // port onto the tailnet and prints "the API key above is what stops + // anyone else calling it". Reusing an unauthenticated service silently + // would make that sentence false about an endpoint the whole tailnet can + // reach. Refusing is the only answer that fails closed, and it is the + // same shape as the recipe mismatch above. + if require_api_key && !existing.requires_api_key { + bail!( + "managed service `{}` is already running for engine `{engine}` and model `{}` \ + without authentication, and a running server cannot be given a key it did not \ + start with; stop it with `{}` and run the command again to \ + serve it with `--require-api-key`", + existing.service_id, + resolve.canonical_model_id, + service_hints::service_stop_hint(&existing.service_id) + ); + } + record_cli_audit_event( + paths, + "service", + "managed_service_launch_skipped", + "info", + format!( + "skipped duplicate managed launch engine={engine} model={} existing_service_id={} status={}", + resolve.canonical_model_id, existing.service_id, existing.status + ), + Some(&existing.service_id), ); + return Ok(ManagedSpawn::AlreadyRunning(ManagedLaunchReport { + service_id: existing.service_id, + endpoint_url: existing.endpoint_url, + status: existing.status, + already_running: true, + child_pid: None, + log_path: None, + manifest_path: None, + })); } - record_cli_audit_event( - paths, - "service", - "managed_service_launch_skipped", - "info", - format!( - "skipped duplicate managed launch engine={engine} model={} existing_service_id={} status={}", - resolve.canonical_model_id, existing.service_id, existing.status - ), - Some(&existing.service_id), - ); - return Ok(ManagedSpawn::AlreadyRunning(ManagedLaunchReport { - service_id: existing.service_id, - endpoint_url: existing.endpoint_url, - status: existing.status, - already_running: true, - child_pid: None, - log_path: None, - manifest_path: None, - })); - } + ManagedPortDecision::Lease(lease) => lease, + }; let mut record = ManagedServiceRecord::new( paths, @@ -406,7 +543,7 @@ fn spawn_managed_engine_child( requested_model, resolve.canonical_model_id.clone(), host, - port, + lease.port(), "managed", 0, runtime_id.map(str::to_owned), @@ -428,8 +565,61 @@ fn spawn_managed_engine_child( // This field means "the user demanded auth on a bind that would not otherwise // require it". A public bind needs no such record; its address still says so. record.requires_api_key = require_api_key; + // Publishing the record inside the lock is what reserves the leased port for + // every other transaction. From here on, any failure must remove it again. record.write()?; + match launch_managed_engine_child_locked( + paths, + &mut record, + lease, + engine, + service_id, + resolve, + host, + device_policy, + gpu_indices, + runtime_id, + env_id, + engine_recipe, + ) { + Ok(child_pid) => Ok(ManagedSpawn::Spawned { + record: Box::new(record), + child_pid, + }), + Err(error) => { + // Pre-spawn failures reach here with nothing running; a post-spawn + // publication failure already terminated its child, and the removals + // are idempotent. + abandon_failed_managed_launch(paths, &record, None); + Err(error) + } + } +} + +/// The side-effecting half of the launch transaction: create the log, prepare +/// concrete argv, hand the leased port to the child, spawn it, reject a child +/// that has already exited, and publish the running record. +/// +/// Split out of [`spawn_managed_engine_child`] so that *every* failure below the +/// record's first write is funnelled through one cleanup path before the +/// allocation lock is released. Returns the child pid. +#[allow(clippy::too_many_arguments)] +fn launch_managed_engine_child_locked( + paths: &AppPaths, + record: &mut ManagedServiceRecord, + lease: rocm_core::LoopbackPortLease, + engine: &str, + service_id: &str, + resolve: &ResolveModelResponse, + host: &str, + device_policy: &DevicePolicy, + gpu_indices: &[u32], + runtime_id: Option<&str>, + env_id: Option<&str>, + engine_recipe: Option<&EngineRecipeHint>, +) -> Result { + let port = record.port; if let Some(parent) = record.engine_state_path.parent() { fs::create_dir_all(parent) .with_context(|| format!("failed to create {}", parent.display()))?; @@ -481,6 +671,12 @@ fn spawn_managed_engine_child( endpoint_key_file.is_some(), record.requires_api_key, )?; + // Hand the port over: close the probe socket immediately before the spawn, + // still under the allocation lock, so no other ROCm transaction can take it + // in the meantime. On a custom host the lease never held a socket and this + // is a no-op — the engine owns that bind. + debug_assert_eq!(lease.port(), port); + lease.release(); #[cfg(windows)] let child_pid = { let env_values = app_path_env_var_values(paths, engine_envs_root.as_deref()); @@ -525,15 +721,20 @@ fn spawn_managed_engine_child( // alive: a stop pairs every recorded PID with its own token, and the two // PIDs diverge as soon as `rocmd` restarts the service — at which point an // engine PID recorded without one could only be signalled blind. + // The child is both roles until the engine state file names a separate + // server process. Each role gets its own token, captured while the child is + // alive: a stop pairs every recorded PID with its own token, and the two + // PIDs diverge as soon as `rocmd` restarts the service — at which point an + // engine PID recorded without one could only be signalled blind. record.record_supervisor_identity(child_pid); record.record_engine_identity(child_pid); record.status = "running".to_owned(); - record.write()?; + if let Err(error) = record.write() { + abandon_failed_managed_launch(paths, record, Some(child_pid)); + return Err(error.context("failed to publish the managed service record")); + } - Ok(ManagedSpawn::Spawned { - record: Box::new(record), - child_pid, - }) + Ok(child_pid) } #[allow(clippy::too_many_arguments)] @@ -543,7 +744,7 @@ pub(crate) fn start_managed_service( requested_model: &str, resolve: &ResolveModelResponse, host: &str, - port: u16, + port_request: serve_port::PortRequest, device_policy: &DevicePolicy, gpu_indices: &[u32], runtime_id: Option<&str>, @@ -562,7 +763,7 @@ pub(crate) fn start_managed_service( requested_model, resolve, host, - port, + port_request, device_policy, gpu_indices, runtime_id, @@ -573,6 +774,9 @@ pub(crate) fn start_managed_service( ManagedSpawn::AlreadyRunning(report) => return Ok(report), ManagedSpawn::Spawned { record, child_pid } => (*record, child_pid), }; + // The transaction resolved the request to one concrete port; everything + // below — readiness, the endpoint URL, the audit line — uses that. + let port = record.port; // The claiming service record is now persisted, so the selected GPU is // visible to any concurrent auto-selection. Release the launch lock before // the readiness wait below, which can block for many seconds — holding it @@ -870,7 +1074,7 @@ pub(crate) fn run_attached_service( requested_model: &str, resolve: &ResolveModelResponse, host: &str, - port: u16, + port_request: serve_port::PortRequest, gpu_indices: &[u32], runtime_id: Option<&str>, env_id: Option<&str>, @@ -887,7 +1091,7 @@ pub(crate) fn run_attached_service( requested_model, resolve, host, - port, + port_request, &resolve.device_policy, gpu_indices, runtime_id, @@ -900,7 +1104,9 @@ pub(crate) fn run_attached_service( // lock before streaming logs, which blocks for the whole attached session. drop(launch_lock); - let (service_id, log_path, child_pid) = match spawn { + // Attached mode is a managed record too, so it shares the transaction's + // resolved concrete port. + let (service_id, log_path, port, child_pid) = match spawn { // A server for this engine+model is already live. Don't fight it for the // port — point the user at the existing one instead of tailing a log we // did not start. @@ -917,9 +1123,12 @@ pub(crate) fn run_attached_service( drop_orphaned_endpoint_key_on_already_running(&paths, service_id, endpoint_api_key); return Ok(()); } - ManagedSpawn::Spawned { record, child_pid } => { - (service_id.to_owned(), record.log_path.clone(), child_pid) - } + ManagedSpawn::Spawned { record, child_pid } => ( + service_id.to_owned(), + record.log_path.clone(), + record.port, + child_pid, + ), }; // The child is a managed service that outlives this session once detached, so @@ -1244,7 +1453,7 @@ mod tests { "qwen", &resolve, "127.0.0.1", - 11511, + serve_port::PortRequest::Explicit(11511), &resolve.device_policy, &[], None, @@ -1327,7 +1536,7 @@ mod tests { "qwen", &resolve, "127.0.0.1", - 11520, + serve_port::PortRequest::Explicit(11520), &resolve.device_policy, &[], None, @@ -1413,7 +1622,7 @@ mod tests { // Loopback on purpose: the public-bind branch must not be what // refuses this, or the test would pass with the guard disabled. "127.0.0.1", - 11512, + serve_port::PortRequest::Explicit(11512), &resolve.device_policy, &[], None, @@ -1753,4 +1962,470 @@ mod tests { assert_eq!(read_autostart_claim(&path), None); let _ = fs::remove_dir_all(&root); } + + #[test] + fn duplicate_managed_launch_detected_across_distinct_service_ids() -> Result<()> { + // `generate_service_id` embeds a timestamp, so a second launch for the + // same engine+model has a DIFFERENT service_id. The guard must still + // detect the live service by (engine, canonical_model_id), and return + // the newest live match. `starting` skips the endpoint probe; the + // current process id is a guaranteed-live PID. + let (root, paths) = test_paths("dup-managed-distinct-ids"); + paths.ensure()?; + + // Older, dead manifest for the same engine+model (distinct service_id). + let mut dead = ManagedServiceRecord::new( + &paths, + "lemonade-qwen-1000", + "lemonade", + "qwen", + "qwen-canonical", + "127.0.0.1", + 11500, + "managed", + 999_999_999, + None, + None, + None, + ); + dead.status = "ready".to_owned(); + dead.engine_pid = Some(999_999_999); + dead.created_at_unix_ms = 1000; + dead.write()?; + + // Newer, live manifest for the same engine+model (distinct service_id). + let mut live = ManagedServiceRecord::new( + &paths, + "lemonade-qwen-2000", + "lemonade", + "qwen", + "qwen-canonical", + "127.0.0.1", + 11501, + "managed", + std::process::id(), + None, + None, + None, + ); + live.status = "starting".to_owned(); + live.engine_pid = Some(std::process::id()); + live.created_at_unix_ms = 2000; + live.write()?; + + let live_services = live_managed_services(&paths); + let found = + existing_live_managed_service_in(&live_services, "lemonade", "qwen-canonical").cloned(); + let _ = fs::remove_dir_all(root); + + let found = found.expect("a live managed service should be detected by engine+model"); + assert_eq!( + found.service_id, "lemonade-qwen-2000", + "should return the newest live match" + ); + assert!(managed_service_is_live(&found)); + Ok(()) + } + + #[test] + fn dead_managed_service_allows_relaunch() -> Result<()> { + // A stale manifest with dead PIDs must NOT block a relaunch: liveness + // refresh demotes it to "stopped", so the guard returns None. + let (root, paths) = test_paths("dup-managed-dead"); + paths.ensure()?; + let mut record = ManagedServiceRecord::new( + &paths, + "lemonade-qwen-3000", + "lemonade", + "qwen", + "qwen-canonical", + "127.0.0.1", + 11502, + "managed", + 999_999_999, + None, + None, + None, + ); + record.status = "ready".to_owned(); + record.engine_pid = Some(999_999_999); + record.write()?; + + let live = live_managed_services(&paths); + let found = existing_live_managed_service_in(&live, "lemonade", "qwen-canonical"); + let _ = fs::remove_dir_all(root); + + assert!( + found.is_none(), + "a dead managed service must not block relaunch" + ); + Ok(()) + } + + #[test] + fn live_service_for_other_model_does_not_block() -> Result<()> { + // A live service for a DIFFERENT model must not match — the guard keys + // on the model, not just the engine. + let (root, paths) = test_paths("dup-managed-other-model"); + paths.ensure()?; + let mut record = ManagedServiceRecord::new( + &paths, + "lemonade-other-1", + "lemonade", + "other", + "other-canonical", + "127.0.0.1", + 11503, + "managed", + std::process::id(), + None, + None, + None, + ); + record.status = "starting".to_owned(); + record.engine_pid = Some(std::process::id()); + record.write()?; + + let live = live_managed_services(&paths); + let found = existing_live_managed_service_in(&live, "lemonade", "qwen-canonical"); + let _ = fs::remove_dir_all(root); + + assert!( + found.is_none(), + "a live service for a different model must not match" + ); + Ok(()) + } + + #[test] + fn missing_manifest_allows_launch() { + // No services dir / manifests → nothing to detect, launch proceeds. + let (root, paths) = test_paths("dup-managed-missing"); + let live = live_managed_services(&paths); + let found = existing_live_managed_service_in(&live, "lemonade", "qwen-canonical"); + let _ = fs::remove_dir_all(root); + assert!(found.is_none()); + } + + fn write_live_service_record( + paths: &AppPaths, + service_id: &str, + canonical_model_id: &str, + port: u16, + status: &str, + ) -> Result { + let mut record = ManagedServiceRecord::new( + paths, + service_id, + "lemonade", + canonical_model_id, + canonical_model_id.to_owned(), + rocm_core::DEFAULT_LOCAL_HOST, + port, + "managed", + std::process::id(), + None, + None, + None, + ); + record.status = status.to_owned(); + record.engine_pid = Some(std::process::id()); + record.write()?; + Ok(record) + } + + fn never_probe(_port: u16) -> std::io::Result { + panic!("no port may be probed on this path") + } + + fn always_free(port: u16) -> std::io::Result { + Ok(rocm_core::LoopbackPortLease::engine_owned(port)) + } + + #[test] + fn allocation_transaction_answers_an_equivalent_live_service_before_leasing() -> Result<()> { + let (root, paths) = test_paths("alloc-txn-already-live"); + paths.ensure()?; + let existing = + write_live_service_record(&paths, "lemonade-qwen-9000", "qwen-canonical", 11501, "starting")?; + + let decision = resolve_managed_port_in_transaction( + &paths, + "lemonade", + "qwen-canonical", + "127.0.0.1", + serve_port::PortRequest::Auto, + &mut never_probe, + )?; + let _ = fs::remove_dir_all(root); + + match decision { + ManagedPortDecision::AlreadyLive(record) => { + assert_eq!(record.service_id, existing.service_id); + assert_eq!(record.port, existing.port); + } + ManagedPortDecision::Lease(_) => { + panic!("an equivalent live service must be answered, not leased a new port") + } + } + Ok(()) + } + + #[test] + fn allocation_transaction_skips_ports_reserved_by_other_live_services() -> Result<()> { + let (root, paths) = test_paths("alloc-txn-skips-reserved"); + paths.ensure()?; + write_live_service_record(&paths, "lemonade-a", "model-a", 11_435, "running")?; + write_live_service_record(&paths, "lemonade-b", "model-b", 11_436, "recovering")?; + + let decision = resolve_managed_port_in_transaction( + &paths, + "lemonade", + "model-c", + "127.0.0.1", + serve_port::PortRequest::Auto, + &mut always_free, + )?; + let _ = fs::remove_dir_all(root); + + match decision { + ManagedPortDecision::Lease(lease) => assert_eq!(lease.port(), 11_437), + ManagedPortDecision::AlreadyLive(record) => { + panic!("model-c has no live service, expected a lease, got {record:?}") + } + } + Ok(()) + } + + #[test] + fn allocation_transaction_ignores_stopped_records() -> Result<()> { + let (root, paths) = test_paths("alloc-txn-ignores-stopped"); + paths.ensure()?; + let mut stale = + write_live_service_record(&paths, "lemonade-stale", "model-stale", 11_435, "running")?; + // A dead pid demotes the record to stopped when liveness is refreshed, so + // it must not reserve its port. + stale.engine_pid = Some(999_999_999); + stale.supervisor_pid = 999_999_999; + stale.write()?; + + let decision = resolve_managed_port_in_transaction( + &paths, + "lemonade", + "model-fresh", + "127.0.0.1", + serve_port::PortRequest::Auto, + &mut always_free, + )?; + let _ = fs::remove_dir_all(root); + + match decision { + ManagedPortDecision::Lease(lease) => assert_eq!(lease.port(), 11_435), + ManagedPortDecision::AlreadyLive(record) => { + panic!("a stopped record must not block a fresh launch, got {record:?}") + } + } + Ok(()) + } + + #[test] + fn allocation_transaction_rejects_an_explicit_port_a_recovering_service_holds() -> Result<()> { + let (root, paths) = test_paths("alloc-txn-rejects-explicit-collision"); + paths.ensure()?; + let existing = write_live_service_record( + &paths, + "lemonade-recovering", + "model-recovering", + 11_435, + "recovering", + )?; + + let result = resolve_managed_port_in_transaction( + &paths, + "lemonade", + "model-other", + "127.0.0.1", + serve_port::PortRequest::Explicit(11_435), + &mut never_probe, + ); + let _ = fs::remove_dir_all(root); + + let Err(error) = result else { + panic!("an explicit port a recovering service holds must be refused") + }; + let message = error.to_string(); + assert!( + message.contains(&existing.service_id), + "the refusal must name the service holding the port: {message}" + ); + assert!( + message.contains("recovering"), + "the refusal must say why the port is unavailable: {message}" + ); + Ok(()) + } + + #[test] + fn concurrent_auto_transactions_publish_distinct_ports() -> Result<()> { + let (root, paths) = test_paths("alloc-txn-concurrent-distinct"); + paths.ensure()?; + let paths = std::sync::Arc::new(paths); + + let spawn_one = |model: &'static str| { + let paths = std::sync::Arc::clone(&paths); + thread::spawn(move || -> Result { + let _allocation = rocm_core::lock_service_allocation(&paths)?; + let lease = match resolve_managed_port_in_transaction( + &paths, + "lemonade", + model, + "127.0.0.1", + serve_port::PortRequest::Auto, + &mut serve_port::loopback_bind_probe, + )? { + ManagedPortDecision::Lease(lease) => lease, + ManagedPortDecision::AlreadyLive(record) => { + panic!("fresh model must not already be live: {record:?}") + } + }; + let port = lease.port(); + write_live_service_record(&paths, model, model, port, "running")?; + lease.release(); + Ok(port) + }) + }; + + let first = spawn_one("model-one"); + let second = spawn_one("model-two"); + let first_port = first.join().expect("thread one panicked")?; + let second_port = second.join().expect("thread two panicked")?; + let _ = fs::remove_dir_all(root); + + assert_ne!(first_port, second_port); + for port in [first_port, second_port] { + assert!( + (serve_port::AUTO_PORT_FIRST..=serve_port::AUTO_PORT_LAST).contains(&port), + "leased port {port} must fall within the auto-select range" + ); + } + Ok(()) + } + + #[test] + fn concurrent_equivalent_transactions_converge_on_one_record() -> Result<()> { + let (root, paths) = test_paths("alloc-txn-concurrent-converge"); + paths.ensure()?; + let paths = std::sync::Arc::new(paths); + + let race_once = || { + let paths = std::sync::Arc::clone(&paths); + thread::spawn(move || -> Result { + let _allocation = rocm_core::lock_service_allocation(&paths)?; + match resolve_managed_port_in_transaction( + &paths, + "lemonade", + "shared-model", + "127.0.0.1", + serve_port::PortRequest::Auto, + &mut serve_port::loopback_bind_probe, + )? { + ManagedPortDecision::Lease(lease) => { + let port = lease.port(); + write_live_service_record( + &paths, + "lemonade-shared", + "shared-model", + port, + "running", + )?; + lease.release(); + Ok(true) + } + ManagedPortDecision::AlreadyLive(_) => Ok(false), + } + }) + }; + + let first = race_once(); + let second = race_once(); + let first_launched = first.join().expect("thread one panicked")?; + let second_launched = second.join().expect("thread two panicked")?; + + let live_count = live_managed_services(&paths).len(); + let _ = fs::remove_dir_all(root); + + assert_ne!( + first_launched, second_launched, + "exactly one racing transaction must launch" + ); + assert_eq!(live_count, 1, "only one record may end up live"); + Ok(()) + } + + #[test] + fn abandoned_launch_leaves_no_record_log_or_endpoint_key() -> Result<()> { + let (root, paths) = test_paths("abandon-launch-cleanup"); + paths.ensure()?; + let record = write_live_service_record(&paths, "lemonade-abandon", "abandon-model", 11_490, "running")?; + fs::create_dir_all( + record + .engine_state_path + .parent() + .expect("engine state parent"), + )?; + fs::write(&record.engine_state_path, "{}")?; + endpoint_keys::store_endpoint_api_key(&paths, &record.service_id, "secret-key")?; + + abandon_failed_managed_launch(&paths, &record, None); + + assert!(!record.manifest_path.exists()); + assert!(!record.log_path.exists()); + assert!(!record.engine_state_path.exists()); + assert_eq!(endpoint_keys::endpoint_api_key(&paths, &record.service_id), None); + assert!(live_managed_services(&paths).is_empty()); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + #[ignore = "spawned as a child process by abandoned_launch_terminates_the_spawned_child"] + fn managed_launch_cleanup_sleeper_child() { + if std::env::var_os("ROCM_TEST_SLEEPER_CHILD").is_some() { + thread::sleep(Duration::from_mins(2)); + } + } + + #[test] + fn abandoned_launch_terminates_the_spawned_child() -> Result<()> { + let (root, paths) = test_paths("abandon-launch-terminates-child"); + paths.ensure()?; + let record = + write_live_service_record(&paths, "lemonade-abandon-child", "abandon-child-model", 11_491, "running")?; + + let current_exe = std::env::current_exe().expect("current test exe"); + let mut child = ProcessCommand::new(current_exe) + .arg("--exact") + .arg("service_launch::tests::managed_launch_cleanup_sleeper_child") + .arg("--ignored") + .arg("--test-threads=1") + .env("ROCM_TEST_SLEEPER_CHILD", "1") + .spawn() + .expect("spawn sleeper child"); + let child_pid = child.id(); + + abandon_failed_managed_launch(&paths, &record, Some(child_pid)); + + let wait_result = child.wait(); + let _ = fs::remove_dir_all(root); + + match wait_result { + Ok(status) => assert!( + !status.success(), + "the sleeper child must be terminated, not exit cleanly" + ), + Err(error) => panic!("failed to wait for sleeper child: {error}"), + } + assert!(!record.manifest_path.exists()); + Ok(()) + } } diff --git a/apps/rocmd/src/watchers.rs b/apps/rocmd/src/watchers.rs index 79efcb269..b674682d6 100644 --- a/apps/rocmd/src/watchers.rs +++ b/apps/rocmd/src/watchers.rs @@ -1279,7 +1279,25 @@ fn handle_server_recover_event_with_record( Some(record.service_id.clone()), ); } - restart_managed_service(paths, &mut *record)?; + // A contended allocation lock or an occupied recorded port is a + // recoverable, per-service condition — report it and move on rather + // than letting one service take the whole 30s watcher tick (and the + // daemon) down. + if let Err(error) = restart_managed_service(paths, &mut *record) { + return crate::persistence::record_event( + paths, + state, + "server-recover", + "error", + "restart_managed_service_failed", + &format!( + "failed to recover managed service {} on {}:{} after \ + {recovery_reason_display}: {error:#}", + record.service_id, record.host, record.port + ), + Some(record.service_id.clone()), + ); + } crate::persistence::record_event( paths, state, @@ -1451,6 +1469,26 @@ pub(crate) fn restart_managed_service( paths: &AppPaths, record: &mut ManagedServiceRecord, ) -> Result<()> { + let _allocation = rocm_core::lock_service_allocation(paths)?; + + // On the canonical loopback host, prove the recorded port is genuinely free + // before publishing `recovering` or spawning anything: a supervisor that + // cannot bind would replace a truthful record with a lie. Custom hosts keep + // the existing engine-owned bind behavior — the record transition is still + // serialized, but the bind is not preflighted. + let lease = if rocm_core::is_canonical_loopback_host(&record.host) { + Some( + rocm_core::lease_loopback_port(record.port).with_context(|| { + format!( + "cannot recover managed service {} on {}:{}: its recorded port is in use", + record.service_id, record.host, record.port + ) + })?, + ) + } else { + None + }; + let rocmd_binary = std::env::current_exe().context("failed to resolve current rocmd executable path")?; let log_file = fs::OpenOptions::new() @@ -1491,6 +1529,12 @@ pub(crate) fn restart_managed_service( record.stop_requested_unix_ms = None; record.write()?; + // Hand the port to the supervisor: close the probe socket immediately before + // the spawn, still holding the allocation lock. + if let Some(lease) = lease { + lease.release(); + } + let mut child = detached_rocmd_command(&rocmd_binary) .args(recovery_supervise_args(record)) .stdin(Stdio::null()) @@ -1499,7 +1543,12 @@ pub(crate) fn restart_managed_service( .spawn() .context("failed to spawn recovery supervisor")?; - record_spawned_supervisor(paths, record, child.id())?; + if let Err(error) = record_spawned_supervisor(paths, record, child.id()) { + // The supervisor is alive but unrecorded: it would hold the port while + // being invisible to every stop/recovery path. + terminate_recovery_supervisor(&mut child); + return Err(error); + } thread::sleep(Duration::from_millis(200)); if let Some(status) = child @@ -1523,6 +1572,26 @@ pub(crate) fn restart_managed_service( /// Record the supervisor [`restart_managed_service`] just spawned, as `pid`. /// +/// Grace given to a recovery supervisor that must be terminated because its +/// record could not be published. Short: it is milliseconds old. +const RECOVERY_CLEANUP_GRACE: Duration = Duration::from_secs(2); + +/// Terminate and reap a recovery supervisor that could not be recorded. +/// +/// Tree scope, forced: the supervisor may already have started an engine child, +/// and leaving either alive would keep the recorded port occupied by a process +/// no stop or recovery path can find. Reaping afterwards keeps the daemon free +/// of zombies. +fn terminate_recovery_supervisor(child: &mut std::process::Child) { + let _outcome = rocm_core::terminate_verified( + &rocm_core::ProcessIdentity::capture(child.id()), + rocm_core::KillScope::Tree, + RECOVERY_CLEANUP_GRACE, + true, + ); + let _ = child.wait(); +} + /// Between the restart's first write — which clears the recorded PIDs, since /// no supervisor exists yet — and this one, the record names no process at /// all. A stop that starts in that window persists its marker first and then @@ -3389,6 +3458,145 @@ mod tests { Ok(()) } + /// Recovery re-claims the concrete port already written in the record, so + /// it must serialize on the same cross-process allocation lock a fresh + /// `rocm serve` launch takes, and must refuse a recorded port something + /// else already holds rather than publish a `recovering` record that + /// lies. + #[test] + fn recovery_waits_for_the_allocation_lock_and_refuses_an_occupied_port() -> Result<()> { + use std::net::TcpListener; + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + + let (root, paths) = temp_app_paths("recovery-port-collision"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + // Something else already owns the port this service recorded. + let occupant = TcpListener::bind(("127.0.0.1", 0))?; + let port = occupant.local_addr()?.port(); + + let mut record = identity_probe_record(&paths, "svc-recovery-port-collision", port); + record.status = "failed".to_owned(); + record.write()?; + + let guard = rocm_core::lock_service_allocation(&paths)?; + let finished = Arc::new(AtomicBool::new(false)); + let handle = { + let paths = paths.clone(); + let finished = Arc::clone(&finished); + let mut record = record.clone(); + thread::spawn(move || { + let result = restart_managed_service(&paths, &mut record); + finished.store(true, Ordering::SeqCst); + result + }) + }; + + thread::sleep(Duration::from_millis(250)); + assert!( + !finished.load(Ordering::SeqCst), + "recovery must serialize on the shared allocation lock" + ); + drop(guard); + + let error = handle + .join() + .expect("recovery thread") + .expect_err("an occupied recorded port must fail recovery"); + let message = format!("{error:#}"); + assert!(message.contains("recorded port is in use"), "{message}"); + + // Nothing was spawned and the prior truthful record is untouched: it must + // not have been rewritten as `recovering`. + let on_disk = crate::persistence::load_managed_services(&paths)? + .into_iter() + .find(|candidate| candidate.service_id == "svc-recovery-port-collision") + .expect("record still present"); + fs::remove_dir_all(root).ok(); + drop(occupant); + + assert_eq!(on_disk.status, "failed"); + assert_eq!(on_disk.restart_count, record.restart_count); + Ok(()) + } + + /// A write failure before anything is spawned must leave the prior record + /// intact: there is nothing to roll back but the in-memory transition. + #[test] + fn recovery_publication_failure_leaves_the_prior_record_intact() -> Result<()> { + use std::net::TcpListener; + + let (root, paths) = temp_app_paths("recovery-publication-failure"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + // A free recorded port, so the lease succeeds and the failure under test + // is the record publication itself. + let probe = TcpListener::bind(("127.0.0.1", 0))?; + let port = probe.local_addr()?.port(); + drop(probe); + + let mut record = identity_probe_record(&paths, "svc-recovery-publication-failure", port); + record.status = "failed".to_owned(); + record.write()?; + + // Make the manifest unwritable by replacing it with a directory. The + // `recovering` transition can no longer be published. + let manifest_path = paths.service_manifest_path(&record.service_id); + fs::remove_file(&manifest_path)?; + fs::create_dir_all(&manifest_path)?; + + let error = restart_managed_service(&paths, &mut record) + .expect_err("an unpublishable record must fail recovery"); + let message = format!("{error:#}"); + assert!(message.contains("failed to write service manifest"), "{message}"); + + // The caller's record must not claim a recovery that never happened. + assert_eq!(record.status, "recovering"); + fs::remove_dir_all(&manifest_path).ok(); + fs::remove_dir_all(root).ok(); + Ok(()) + } + + #[test] + #[ignore = "spawned as a child process by terminate_recovery_supervisor_kills_and_reaps_the_child"] + fn recovery_cleanup_sleeper_child() { + if std::env::var_os("ROCMD_TEST_SLEEPER_CHILD").is_none() { + return; + } + thread::sleep(Duration::from_mins(2)); + } + + #[test] + fn terminate_recovery_supervisor_kills_and_reaps_the_child() { + // A real long-lived process stands in for a supervisor that was spawned + // but whose record could not be published. + let mut child = ProcessCommand::new(std::env::current_exe().expect("test binary")) + .arg("watchers::tests::recovery_cleanup_sleeper_child") + .arg("--exact") + .arg("--ignored") + .env("ROCMD_TEST_SLEEPER_CHILD", "1") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn sleeper child"); + let child_pid = child.id(); + + terminate_recovery_supervisor(&mut child); + + // The helper reaps, so the pid must be gone rather than a zombie. + assert!( + !rocm_core::process_is_running(child_pid), + "an unrecordable supervisor must be terminated and reaped" + ); + // The helper already reaped it; wait again so the child is provably + // handled on every path in this function (and reaping is idempotent). + assert!(child.wait().is_ok()); + } + /// A stop that lands while a restart waits to see whether its supervisor /// survived startup must outlive the restart's `failed` write: a `failed` /// record without the marker is exactly what recovery restarts. diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index ec4163dc2..6ee996355 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -6,8 +6,10 @@ use anyhow::{Context, Result, bail}; use serde::{Deserialize, Serialize}; use std::ffi::OsString; use std::fs; +use std::net::TcpListener; use std::path::{Path, PathBuf}; -use std::time::{SystemTime, UNIX_EPOCH}; +use std::thread; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; #[cfg(all(test, unix))] mod archive_props; @@ -520,12 +522,197 @@ pub(crate) fn write_atomic(path: &Path, bytes: &[u8], tmp_ext: &str) -> Result<( Ok(()) } +// --------------------------------------------------------------------------- +// Managed-service port leasing and the shared allocation lock. +// +// Both `rocm serve` and `rocmd` recovery contend for the same local TCP ports +// and the same service records, so the primitives that arbitrate them live here +// rather than in either binary. +// --------------------------------------------------------------------------- + +/// True only for the exact canonical default loopback host. +/// +/// Proactive OS collision detection (leasing a real socket before launch) is +/// claimed only for this one host. `localhost`, `::1`, `0.0.0.0`, and other +/// IPv4 addresses are "custom hosts": they require an explicit port and keep the +/// existing engine-owned bind/error behavior, because an IPv4-loopback probe +/// would not prove anything about the address the engine will actually bind. +#[must_use] +pub fn is_canonical_loopback_host(host: &str) -> bool { + host.trim() == DEFAULT_LOCAL_HOST +} + +/// A real, held `127.0.0.1:` listener proving the port is bindable now. +/// +/// The lease is the collision check: it is acquired before any manifest, log, or +/// child process exists, and [`LoopbackPortLease::release`] drops the socket +/// immediately before the engine child is spawned so the child can bind it. The +/// window between release and the child's bind is unavoidable without engine +/// socket-passing; it is far narrower than the previous "bind and hope" launch. +#[derive(Debug)] +pub struct LoopbackPortLease { + port: u16, + listener: Option, +} + +impl LoopbackPortLease { + #[must_use] + pub const fn port(&self) -> u16 { + self.port + } + + /// True while this lease still holds the socket open. + #[must_use] + pub const fn is_held(&self) -> bool { + self.listener.is_some() + } + + /// Close the socket and hand the port to the caller. Call this immediately + /// before spawning the process that will bind it. + pub fn release(mut self) -> u16 { + self.listener = None; + self.port + } + + /// A lease over a port this process never probed, for custom hosts where the + /// engine — not the CLI — owns bind success or failure. + #[must_use] + pub const fn engine_owned(port: u16) -> Self { + Self { + port, + listener: None, + } + } +} + +/// Try to take an exclusive IPv4-loopback lease on `port`. +/// +/// Errors are returned verbatim so callers can distinguish "occupied" +/// ([`std::io::ErrorKind::AddrInUse`], the only condition that should advance an +/// automatic scan) from a privileged or unavailable address, which must fail +/// immediately rather than silently choosing a different port. +pub fn lease_loopback_port(port: u16) -> std::io::Result { + let listener = TcpListener::bind(std::net::SocketAddrV4::new( + std::net::Ipv4Addr::LOCALHOST, + port, + ))?; + Ok(LoopbackPortLease { + port, + listener: Some(listener), + }) +} + +/// File name of the shared allocation lock, inside [`AppPaths::services_dir`]. +/// +/// Not a `.json` file, so `load_managed_services` never mistakes it for a +/// service manifest. +pub const SERVICE_ALLOCATION_LOCK_FILE: &str = "allocation.lock"; +/// Total time a launch or recovery waits for the allocation lock before failing +/// with an actionable error instead of hanging. +pub const SERVICE_ALLOCATION_LOCK_TIMEOUT: Duration = Duration::from_secs(5); +const SERVICE_ALLOCATION_LOCK_POLL: Duration = Duration::from_millis(25); + +#[must_use] +pub fn service_allocation_lock_path(paths: &AppPaths) -> PathBuf { + paths.services_dir().join(SERVICE_ALLOCATION_LOCK_FILE) +} + +/// Cross-process guard serializing the managed-service allocation transaction: +/// duplicate detection, live-reservation refresh, port leasing, child start +/// confirmation, and record publication. +/// +/// The lock is an OS advisory file lock, so it is released both by dropping this +/// guard and by process death — a crashed launcher never wedges the next one. +/// Full HTTP/model readiness deliberately runs *outside* the guard: it takes up +/// to 45 seconds and holds no allocation decision. +#[derive(Debug)] +pub struct ServiceAllocationLock { + file: Option, + path: PathBuf, +} + +impl ServiceAllocationLock { + #[must_use] + pub fn path(&self) -> &Path { + &self.path + } +} + +impl Drop for ServiceAllocationLock { + fn drop(&mut self) { + if let Some(file) = self.file.take() { + let _ = file.unlock(); + } + } +} + +/// Acquire the shared allocation lock for `paths`, waiting at most +/// [`SERVICE_ALLOCATION_LOCK_TIMEOUT`]. +/// +/// # Errors +/// Fails when the lock file cannot be created/locked, or when another process +/// held it for the whole timeout. +pub fn lock_service_allocation(paths: &AppPaths) -> Result { + lock_service_allocation_at( + &service_allocation_lock_path(paths), + SERVICE_ALLOCATION_LOCK_TIMEOUT, + ) +} + +/// [`lock_service_allocation`] against an explicit path and timeout. Exposed for +/// tests that need a private lock file or a short deadline. +/// +/// # Errors +/// See [`lock_service_allocation`]. +pub fn lock_service_allocation_at(path: &Path, timeout: Duration) -> Result { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent) + .with_context(|| format!("failed to create {}", parent.display()))?; + } + let file = fs::OpenOptions::new() + .create(true) + .read(true) + .write(true) + .truncate(false) + .open(path) + .with_context(|| format!("failed to open {}", path.display()))?; + let deadline = Instant::now() + timeout; + loop { + match file.try_lock() { + Ok(()) => { + return Ok(ServiceAllocationLock { + file: Some(file), + path: path.to_path_buf(), + }); + } + Err(std::fs::TryLockError::WouldBlock) => {} + Err(std::fs::TryLockError::Error(error)) => { + return Err( + anyhow::Error::new(error).context(format!("failed to lock {}", path.display())) + ); + } + } + let now = Instant::now(); + if now >= deadline { + bail!( + "timed out after {timeout:?} waiting for the managed-service allocation lock at \ + {path}; another `rocm serve` launch or `rocmd` recovery is mid-transaction. \ + Retry in a moment, and check `rocm services` for a stuck launch.", + path = path.display() + ); + } + thread::sleep(SERVICE_ALLOCATION_LOCK_POLL.min(deadline - now)); + } +} + #[cfg(test)] mod tests { use super::*; + use crate::test_support::workspace_test_artifact_dir; use std::fs; use std::path::Path; + use std::process::{Command, Stdio}; #[test] fn service_id_accepts_generated_and_plain_ids() { @@ -788,4 +975,204 @@ mod tests { ); Ok(()) } + + // -- managed-service allocation lock and loopback leasing --------------- + + const LOCK_CHILD_PATH_ENV: &str = "ROCM_CORE_TEST_LOCK_PATH"; + const LOCK_CHILD_READY_ENV: &str = "ROCM_CORE_TEST_LOCK_READY"; + const HANDOFF_PORT_ENV: &str = "ROCM_CORE_TEST_HANDOFF_PORT"; + + fn lock_test_dir(label: &str) -> PathBuf { + let dir = workspace_test_artifact_dir().join(format!( + "{label}-{}-{}", + std::process::id(), + unix_time_millis() + )); + fs::create_dir_all(&dir).expect("create lock test dir"); + dir + } + + /// Re-run this test binary, executing exactly one `#[ignore]`d helper test in + /// a genuinely separate process. Cross-process locking and socket handoff + /// cannot be proven inside one process. + fn spawn_helper_test(name: &str, envs: &[(&str, String)]) -> std::process::Child { + let exe = std::env::current_exe().expect("test binary path"); + let mut command = Command::new(exe); + command + .arg(name) + .arg("--exact") + .arg("--ignored") + .arg("--test-threads") + .arg("1") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + for (key, value) in envs { + command.env(key, value); + } + command.spawn().expect("spawn helper test process") + } + + #[test] + fn canonical_loopback_host_is_only_the_exact_default_address() { + assert!(is_canonical_loopback_host(DEFAULT_LOCAL_HOST)); + assert!(is_canonical_loopback_host(" 127.0.0.1 ")); + // Everything else is a "custom host": explicit port, engine-owned bind. + for host in [ + "localhost", + "::1", + "0.0.0.0", + "127.0.0.2", + "192.168.1.4", + "", + ] { + assert!( + !is_canonical_loopback_host(host), + "{host} must not be treated as the canonical loopback host" + ); + } + } + + #[test] + fn allocation_lock_path_lives_in_services_dir_and_is_not_a_manifest() { + let paths = AppPaths { + config_dir: PathBuf::from("/tmp/cfg"), + data_dir: PathBuf::from("/tmp/data"), + cache_dir: PathBuf::from("/tmp/cache"), + }; + let path = service_allocation_lock_path(&paths); + assert_eq!(path.parent(), Some(paths.services_dir().as_path())); + // `load_managed_services` only reads `*.json`, so the lock must not be one. + assert_ne!( + path.extension().and_then(|value| value.to_str()), + Some("json") + ); + } + + #[test] + fn allocation_lock_excludes_independent_handles_and_fails_closed_on_timeout() { + let path = lock_test_dir("core-alloc-lock-exclusive").join("allocation.lock"); + let held = lock_service_allocation_at(&path, Duration::from_secs(5)).expect("first holder"); + assert_eq!(held.path(), path.as_path()); + + // A second, independent file handle must contend on the OS lock rather + // than sail through, and must give up with an actionable error. + let started = Instant::now(); + let error = lock_service_allocation_at(&path, Duration::from_millis(200)) + .expect_err("a held lock must not be acquired twice"); + assert!( + started.elapsed() >= Duration::from_millis(200), + "waiter returned before its deadline" + ); + let message = format!("{error:#}"); + assert!(message.contains("timed out"), "{message}"); + assert!(message.contains("allocation lock"), "{message}"); + + drop(held); + lock_service_allocation_at(&path, Duration::from_secs(5)) + .expect("lock is free once the guard is dropped"); + } + + #[test] + #[ignore = "spawned as a child process by allocation_lock_is_released_when_the_holder_process_exits"] + fn allocation_lock_child_holder() { + let Ok(path) = std::env::var(LOCK_CHILD_PATH_ENV) else { + return; + }; + let ready = std::env::var(LOCK_CHILD_READY_ENV).expect("ready-file path"); + let guard = lock_service_allocation_at(Path::new(&path), Duration::from_secs(10)) + .expect("child acquires the shared lock"); + fs::write(&ready, "held").expect("signal that the child holds the lock"); + thread::sleep(Duration::from_millis(300)); + // Leak the guard so the lock is released by process exit alone — that is + // the property a crashed launcher depends on. + std::mem::forget(guard); + } + + #[test] + fn allocation_lock_is_released_when_the_holder_process_exits() { + let dir = lock_test_dir("core-alloc-lock-crossproc"); + let path = dir.join("allocation.lock"); + let ready = dir.join("child-ready"); + let mut child = spawn_helper_test( + "tests::allocation_lock_child_holder", + &[ + (LOCK_CHILD_PATH_ENV, path.display().to_string()), + (LOCK_CHILD_READY_ENV, ready.display().to_string()), + ], + ); + + let deadline = Instant::now() + Duration::from_mins(1); + while !ready.exists() { + assert!( + Instant::now() < deadline, + "child never reported holding the lock" + ); + thread::sleep(Duration::from_millis(25)); + } + + let busy = lock_service_allocation_at(&path, Duration::from_millis(50)) + .expect_err("the lock must be exclusive across processes"); + assert!(format!("{busy:#}").contains("allocation lock")); + + let status = child.wait().expect("child holder exits"); + assert!(status.success(), "child holder failed: {status}"); + lock_service_allocation_at(&path, Duration::from_secs(10)) + .expect("the OS releases the lock when the holding process exits"); + } + + #[test] + fn loopback_lease_blocks_a_second_bind_until_released() { + let ephemeral = TcpListener::bind(("127.0.0.1", 0)).expect("ephemeral bind"); + let port = ephemeral.local_addr().expect("local addr").port(); + drop(ephemeral); + + let lease = lease_loopback_port(port).expect("free port leases"); + assert_eq!(lease.port(), port); + assert!(lease.is_held()); + let conflict = lease_loopback_port(port).expect_err("a held port must not lease twice"); + assert_eq!(conflict.kind(), std::io::ErrorKind::AddrInUse); + + assert_eq!(lease.release(), port); + lease_loopback_port(port).expect("the port is bindable again after release"); + } + + #[test] + fn engine_owned_lease_holds_no_socket() { + // Custom hosts get a port carrier, never a loopback probe. + let lease = LoopbackPortLease::engine_owned(8000); + assert_eq!(lease.port(), 8000); + assert!(!lease.is_held()); + } + + #[test] + #[ignore = "spawned as a child process by released_loopback_lease_is_immediately_bindable_by_another_process"] + fn loopback_handoff_child() { + let Ok(port) = std::env::var(HANDOFF_PORT_ENV) else { + return; + }; + let port: u16 = port.parse().expect("handoff port"); + let lease = lease_loopback_port(port).expect("child binds the released port"); + assert_eq!(lease.port(), port); + } + + #[test] + fn released_loopback_lease_is_immediately_bindable_by_another_process() { + let ephemeral = TcpListener::bind(("127.0.0.1", 0)).expect("ephemeral bind"); + let port = ephemeral.local_addr().expect("local addr").port(); + drop(ephemeral); + let lease = lease_loopback_port(port).expect("parent leases the port"); + assert_eq!(lease.release(), port); + + let status = spawn_helper_test( + "tests::loopback_handoff_child", + &[(HANDOFF_PORT_ENV, port.to_string())], + ) + .wait() + .expect("handoff child exits"); + assert!( + status.success(), + "a separate process must bind the port immediately after the lease is released" + ); + } } diff --git a/crates/rocm-dash-tui/src/app/mod.rs b/crates/rocm-dash-tui/src/app/mod.rs index cbd113566..28687dd1b 100644 --- a/crates/rocm-dash-tui/src/app/mod.rs +++ b/crates/rocm-dash-tui/src/app/mod.rs @@ -1826,10 +1826,12 @@ mod tests { field: _, model: _, engine_idx: _, - device_idx: _, host: _, + port_mode: _, port: _, managed: _, + advanced_expanded: _, + editor: _, browser, picker, approval, diff --git a/crates/rocm-dash-tui/src/ui/serve_wizard.rs b/crates/rocm-dash-tui/src/ui/serve_wizard.rs index daff4b3f6..a5d77043c 100644 --- a/crates/rocm-dash-tui/src/ui/serve_wizard.rs +++ b/crates/rocm-dash-tui/src/ui/serve_wizard.rs @@ -5,11 +5,18 @@ //! Serve wizard overlay (Phase 3 Wave 1). //! //! The headline operational screen rebuilt on the Wave-0 primitives: a compact -//! form that builds a `rocm serve … --managed` invocation and runs it **through -//! the approval gate and the job-bridge** — never inline, never with a legacy -//! `std::thread::spawn` + `try_recv`. A served model launched here surfaces in -//! the services manager and the dashboard's live `gen_tps` (the D7 wire is -//! already in place via `rocm serve --managed`). +//! **model-first** form that builds a `rocm serve …` invocation and runs it +//! **through the approval gate and the job-bridge** — never inline, never with +//! a legacy `std::thread::spawn` + `try_recv`. A served model launched here +//! surfaces in the services manager and the dashboard's live `gen_tps`. +//! +//! Progressive disclosure: the default form is Model → `Advanced settings` → +//! Launch. Everything else (engine, device policy, host, port, mode) is +//! automatic and lives behind the inline `Advanced settings` row — expanded in +//! place, never as a second modal. Because the defaults are genuinely +//! automatic, the default invocation is exactly `rocm serve MODEL`: the CLI +//! stays the single resolution authority for engine choice, the GPU-required +//! policy, the loopback host, the port, and managed mode. //! //! The model field can be typed directly (a recipe name, alias, or path) or //! filled from the reusable Wave-0 [`FolderBrowser`] for a local model path @@ -36,28 +43,65 @@ use crate::ui::model_picker::{ModelPicker, ModelRecipeSummary, PickerOutcome, dr use crate::ui::panel::{self, BoxRole}; use crate::ui::theme::Theme; -/// Engine inventory — names mirror `apps/rocm` `engine_inventory()`. Kept -/// TUI-local (a stable, small list) so this layer needs no `rocm-core` dep. -pub const ENGINES: &[&str] = &["lemonade", "vllm"]; - -/// Device-policy choices. Index 0 omits `--device` entirely (engine default); -/// the rest mirror `rocm-core`'s validated `gpu_required|gpu_preferred|cpu_only`. -pub const DEVICES: &[&str] = &[ - "(engine default)", - "gpu_required", - "gpu_preferred", - "cpu_only", -]; +/// Engine inventory — index 0 is the automatic choice. +/// +/// Automatic emits no `--engine` and lets the CLI resolve one; the rest mirror +/// `apps/rocm` `engine_inventory()`. Kept TUI-local (a stable, small list) so +/// this layer needs no `rocm-core` dep. +pub const ENGINES: &[&str] = &["automatic", "lemonade", "vllm"]; + +/// Index into [`ENGINES`] that means "let the CLI decide". +pub const ENGINE_AUTO: usize = 0; + +/// The device policy this wizard offers: read-only, GPU-required, automatic. +/// +/// ROCm never falls back to CPU, so there is nothing here to choose — omitting +/// `--device` already resolves to `gpu_required` in the CLI. +pub const DEVICE_POLICY: &str = "GPU required (automatic)"; /// Mirrors `rocm-core::DEFAULT_LOCAL_HOST` / `DEFAULT_LOCAL_PORT` (TUI-local to -/// avoid the dep; the CLI re-applies its own defaults if these are cleared). +/// avoid the dep). The default host is omitted from argv; the default port text +/// is only the starting point for a *Custom* port. const DEFAULT_HOST: &str = "127.0.0.1"; const DEFAULT_PORT: &str = "11435"; -/// The form fields, in vertical order. +/// The shared phrase for an automatic port. +/// +/// The wizard cannot truthfully promise a concrete port before the CLI leases +/// one, so the approval card says exactly this and the launch output prints the +/// resolved endpoint. +pub const AUTO_PORT_NOTE: &str = "Port: automatic; endpoint shown after launch"; + +/// The exact cross-field guidance for a custom host left on an automatic port. +/// +/// Automatic selection is only supported on the canonical loopback host. +pub const CUSTOM_HOST_NEEDS_PORT: &str = "Custom hosts require a custom port; set Port to Custom."; + +/// Loopback hosts the CLI accepts without `--allow-public-bind`. +/// +/// Dash has no public-bind confirmation or endpoint-key UI, so a public bind +/// must be requested deliberately from the CLI instead. +pub const LOOPBACK_HOSTS: &[&str] = &["127.0.0.1", "localhost", "::1", "[::1]"]; + +/// The guidance shown when a public / non-loopback host is typed into Dash. +pub const PUBLIC_HOST_NEEDS_CLI: &str = "Dash serves on loopback only; for a public bind run \ + `rocm serve --host … --port … --allow-public-bind` from the CLI."; + +/// Whether `host` is one of the loopback spellings Dash may bind. +#[must_use] +pub fn is_loopback_host(host: &str) -> bool { + LOOPBACK_HOSTS + .iter() + .any(|known| known.eq_ignore_ascii_case(host)) +} + +/// The form fields. Not every field is visible at once — see +/// [`ServeWizardState::visible_fields`]. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Field { Model, + /// Inline progressive-disclosure row (`Advanced settings ▸/▾`). + Advanced, Engine, Device, Host, @@ -66,9 +110,10 @@ pub enum Field { Launch, } -/// Field order; `state.field` indexes this. +/// Full (expanded) field order. pub const FIELDS: &[Field] = &[ Field::Model, + Field::Advanced, Field::Engine, Field::Device, Field::Host, @@ -77,13 +122,98 @@ pub const FIELDS: &[Field] = &[ Field::Launch, ]; +/// Collapsed (default) field order — model-first, then disclosure, then launch. +pub const BASIC_FIELDS: &[Field] = &[Field::Model, Field::Advanced, Field::Launch]; + +/// Port policy: automatic (the CLI leases a free local port) or an explicit +/// custom port typed by the user. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum PortMode { + #[default] + Auto, + Custom, +} + +/// Inline single-line text editor state for the Host / Custom-port rows. +/// +/// On entry the existing value is *selected*: the first printable character +/// replaces it wholesale (the common "retype it" case) while Left/Right, +/// Backspace, and further typing fall back to ordinary cursor editing. +#[derive(Debug, Clone)] +pub struct InlineEditor { + pub field: Field, + /// Value to restore on Escape. + pub original: String, + pub value: String, + /// Cursor position, in characters, in `0..=value.chars().count()`. + pub cursor: usize, + /// Whether the initial value is still fully selected. + pub selected: bool, +} + +impl InlineEditor { + fn new(field: Field, value: &str) -> Self { + Self { + field, + original: value.to_string(), + value: value.to_string(), + cursor: value.chars().count(), + selected: true, + } + } + + fn len(&self) -> usize { + self.value.chars().count() + } + + fn byte_at(&self, cursor: usize) -> usize { + self.value + .char_indices() + .nth(cursor) + .map_or(self.value.len(), |(b, _)| b) + } + + fn insert(&mut self, c: char) { + if self.selected { + self.value.clear(); + self.cursor = 0; + self.selected = false; + } + let at = self.byte_at(self.cursor); + self.value.insert(at, c); + self.cursor += 1; + } + + fn backspace(&mut self) { + if self.selected { + self.value.clear(); + self.cursor = 0; + self.selected = false; + return; + } + if self.cursor == 0 { + return; + } + let from = self.byte_at(self.cursor - 1); + let to = self.byte_at(self.cursor); + self.value.replace_range(from..to, ""); + self.cursor -= 1; + } + + fn move_cursor(&mut self, delta: isize) { + self.selected = false; + let max = self.len().cast_signed(); + self.cursor = (self.cursor.cast_signed() + delta).clamp(0, max) as usize; + } +} + /// An approved-but-not-yet-launched serve invocation. #[derive(Debug, Clone)] pub struct PendingServe { /// Resolved `rocm` binary path (captured at approval time so a later /// `current_exe()` failure can't silently drop an approved launch). pub cmd: String, - /// The argv after the binary (`["serve", model, "--engine", …]`). + /// The argv after the binary (`["serve", model, …]`). pub args: Vec, pub request: ApprovalRequest, pub choice: ApprovalChoice, @@ -92,13 +222,19 @@ pub struct PendingServe { /// Overlay state. `None` on `AppState` means the wizard is closed. #[derive(Debug, Clone)] pub struct ServeWizardState { + /// Index into [`ServeWizardState::visible_fields`]. pub field: usize, pub model: String, pub engine_idx: usize, - pub device_idx: usize, pub host: String, + pub port_mode: PortMode, + /// The custom port text (only meaningful when `port_mode` is `Custom`). pub port: String, pub managed: bool, + /// Whether the inline `Advanced settings` rows are showing. + pub advanced_expanded: bool, + /// Inline text editor; `Some` while editing Host or the custom port. + pub editor: Option, /// Local-path picker (Wave-0 primitive); `Some` while browsing. pub browser: Option, /// Model-recipe picker sub-step; `Some` while choosing a recipe. @@ -116,11 +252,13 @@ impl Default for ServeWizardState { Self { field: 0, model: String::new(), - engine_idx: 0, - device_idx: 0, + engine_idx: ENGINE_AUTO, host: DEFAULT_HOST.to_string(), + port_mode: PortMode::Auto, port: DEFAULT_PORT.to_string(), managed: true, + advanced_expanded: false, + editor: None, browser: None, picker: None, approval: None, @@ -131,98 +269,173 @@ impl Default for ServeWizardState { } impl ServeWizardState { - fn current_field(&self) -> Field { - FIELDS[self.field.min(FIELDS.len() - 1)] + /// The rows the user can actually see and reach right now. + #[must_use] + pub const fn visible_fields(&self) -> &'static [Field] { + if self.advanced_expanded { + FIELDS + } else { + BASIC_FIELDS + } + } + + /// The focused row (clamped — collapsing can never strand focus). + #[must_use] + pub fn current_field(&self) -> Field { + let vis = self.visible_fields(); + vis[self.field.min(vis.len() - 1)] } fn move_field(&mut self, delta: isize) { - let max = FIELDS.len().cast_signed() - 1; - self.field = (self.field.cast_signed() + delta).clamp(0, max) as usize; + let vis = self.visible_fields(); + let cur = self.field.min(vis.len() - 1); + let max = vis.len().cast_signed() - 1; + self.field = (cur.cast_signed() + delta).clamp(0, max) as usize; + } + + /// Focus the given field, expanding Advanced when it is hidden. + fn focus(&mut self, field: Field) { + if !self.visible_fields().contains(&field) { + self.advanced_expanded = true; + } + if let Some(idx) = self.visible_fields().iter().position(|f| *f == field) { + self.field = idx; + } } fn cycle(&mut self, delta: isize) { match self.current_field() { + // Advanced is a choice row too: ← collapses, → expands. Only + // reachable while Advanced owns focus, and Advanced sits at the + // same index in both orders, so focus is preserved either way. + Field::Advanced => self.advanced_expanded = delta > 0, Field::Engine => self.engine_idx = cycle_idx(self.engine_idx, ENGINES.len(), delta), - Field::Device => self.device_idx = cycle_idx(self.device_idx, DEVICES.len(), delta), + Field::Port => { + self.port_mode = match self.port_mode { + PortMode::Auto => PortMode::Custom, + PortMode::Custom => PortMode::Auto, + }; + } Field::Mode => self.managed = !self.managed, _ => {} } } + /// Direct typing only applies to the Model row; Host and the custom port + /// are edited through the explicit inline editor. fn type_char(&mut self, c: char) { - match self.current_field() { - Field::Model => self.model.push(c), - Field::Host => self.host.push(c), - // Port accepts digits only — never builds an unparseable `--port`. - Field::Port if c.is_ascii_digit() => self.port.push(c), - _ => {} + if self.current_field() == Field::Model { + self.model.push(c); } } fn backspace(&mut self) { - match self.current_field() { - Field::Model => { - self.model.pop(); - } - Field::Host => { - self.host.pop(); - } - Field::Port => { - self.port.pop(); - } - _ => {} + if self.current_field() == Field::Model { + self.model.pop(); } } - /// Build the `rocm` argv for the current form, or an error message. - fn build_args(&self) -> Result, String> { - // Trimmed, and deliberately so: `self.model` is typed as often as it is - // browser-filled, and three other places already derive from the - // trimmed form — the approval card's title, the job id, and the - // "already running" guard. Staging an untrimmed value here would let - // two launches differing only by trailing whitespace collide on one job - // id while actually running different commands. `host` and `port` below - // follow the same rule. - let model = self.model.trim(); - if model.is_empty() { - return Err("model is required".to_string()); + /// Whether any advanced setting deviates from the automatic defaults. + #[must_use] + pub fn is_customized(&self) -> bool { + self.engine_idx != ENGINE_AUTO + || self.host.trim() != DEFAULT_HOST + || self.port_mode != PortMode::Auto + || !self.managed + } + + /// The `Advanced settings` summary text — the one place a *collapsed* form + /// still tells the truth about overrides, including a broken hidden one. + #[must_use] + pub fn advanced_summary(&self) -> &'static str { + if !self.is_customized() { + return "Automatic"; + } + if self.port_mode == PortMode::Custom && parse_port(&self.port).is_none() { + "Customized · port needs attention" + } else { + "Customized" } + } + + /// Validate the whole form. On success returns the concrete port to emit + /// (`None` for automatic); on failure the offending field plus a plain fix. + fn validate(&self) -> Result, (Field, String)> { + if self.model.trim().is_empty() { + return Err((Field::Model, "model is required".to_string())); + } + let host = self.host.trim(); + if host.is_empty() { + return Err(( + Field::Host, + format!("host is required; type an address or restore {DEFAULT_HOST}"), + )); + } + // Automatic port selection is only supported on the canonical loopback + // host — anything else must name its own port. Never invent one. + if host != DEFAULT_HOST && self.port_mode == PortMode::Auto { + return Err((Field::Port, CUSTOM_HOST_NEEDS_PORT.to_string())); + } + // Dash has no `--allow-public-bind` confirmation and no endpoint-key + // UI, so it must never stage a public bind — even with an explicit + // port. That deliberate step belongs to the CLI. + if !is_loopback_host(host) { + return Err((Field::Host, PUBLIC_HOST_NEEDS_CLI.to_string())); + } + if self.port_mode == PortMode::Custom { + let Some(port) = parse_port(&self.port) else { + let shown = self.port.trim(); + return Err(( + Field::Port, + format!("port `{shown}` is not a valid 1–65535 value"), + )); + }; + return Ok(Some(port)); + } + Ok(None) + } + + /// Build the `rocm` argv for the current form, or the offending field and + /// an error message. Only explicit overrides are emitted: the automatic + /// form is exactly `serve MODEL`. + fn build_args(&self) -> Result, (Field, String)> { + let port = self.validate()?; + let model = self.model.trim(); let mut args = vec!["serve".to_string(), model.to_string()]; - args.push("--engine".to_string()); - args.push(ENGINES[self.engine_idx.min(ENGINES.len() - 1)].to_string()); - // Index 0 = engine default → omit --device. - if self.device_idx > 0 { - args.push("--device".to_string()); - args.push(DEVICES[self.device_idx.min(DEVICES.len() - 1)].to_string()); + // Automatic engine → the CLI resolves it. + if self.engine_idx != ENGINE_AUTO { + args.push("--engine".to_string()); + args.push(ENGINES[self.engine_idx.min(ENGINES.len() - 1)].to_string()); } + // Device policy is automatic and GPU-required; omitting `--device` + // already means `gpu_required` in the CLI. let host = self.host.trim(); - if !host.is_empty() { + if host != DEFAULT_HOST { args.push("--host".to_string()); args.push(host.to_string()); } - let port = self.port.trim(); - if !port.is_empty() { - // u16 accepts 0, but 0 is not a bindable listen port — reject it so - // the error surfaces in the form, not as a downstream bind failure. - match port.parse::() { - Ok(p) if p > 0 => { - args.push("--port".to_string()); - args.push(p.to_string()); - } - _ => return Err(format!("port `{port}` is not a valid 1–65535 value")), - } + if let Some(port) = port { + args.push("--port".to_string()); + args.push(port.to_string()); } - // Managed (default) hands supervision to the daemon → it shows up in the - // services manager + dashboard gen_tps. Foreground runs in this job. - if self.managed { - args.push("--managed".to_string()); - } else { + // Managed (the default) hands supervision to the daemon → it shows up + // in the services manager + dashboard gen_tps, and needs no flag. + if !self.managed { args.push("--foreground".to_string()); } Ok(args) } } +/// Parse a custom port: digits only, 1–65535. `0` parses as `u16` but is not a +/// bindable listen port, so it is rejected here rather than downstream. +fn parse_port(raw: &str) -> Option { + match raw.trim().parse::() { + Ok(p) if p > 0 => Some(p), + _ => None, + } +} + const fn cycle_idx(cur: usize, len: usize, delta: isize) -> usize { if len == 0 { return 0; @@ -249,14 +462,10 @@ pub fn on_key( if let Some(picker) = w.picker.as_mut() { match picker.on_key(key.code, recipes) { PickerOutcome::Chosen(summary) => { + // Only the model is filled. A recipe's preferred engine is NOT + // forced onto the form: the CLI stays the resolution authority + // and the wizard keeps emitting an automatic engine. w.model = summary.id; - // Pre-select the recipe's preferred engine when it is one this - // wizard lists; otherwise leave the engine choice untouched. - if let Some(eng) = summary.preferred_engine - && let Some(idx) = ENGINES.iter().position(|e| *e == eng) - { - w.engine_idx = idx; - } w.picker = None; } PickerOutcome::Cancelled => w.picker = None, @@ -278,7 +487,7 @@ pub fn on_key( return Vec::new(); } - // 2) Approval modal has focus. + // 3) Approval modal has focus. if let Some(pending) = w.approval.as_mut() { let (choice, verdict) = approval_key(key.code, pending.choice); pending.choice = choice; @@ -294,7 +503,7 @@ pub fn on_key( return Vec::new(); } - // 3) A launch job is showing in the console. + // 4) A launch job is showing in the console. if let Some(job_id) = w.active_job.clone() { match on_console_key(&job_id, jobs, key) { ConsoleOutcome::Cancelled(fx) => return fx, @@ -305,7 +514,15 @@ pub fn on_key( return Vec::new(); } - // 4) Form editing. + // 5) Inline text editor (Host / custom Port) owns the keyboard. It is not a + // modal: the form stays painted underneath, and form navigation keys + // cannot reach hidden rows from here. + if w.editor.is_some() { + editor_key(w, key.code); + return Vec::new(); + } + + // 6) Form editing. match key.code { KeyCode::Esc => *wizard = None, KeyCode::Up => w.move_field(-1), @@ -313,15 +530,24 @@ pub fn on_key( KeyCode::Left => w.cycle(-1), KeyCode::Right => w.cycle(1), KeyCode::Char(' ') if w.current_field() == Field::Mode => w.cycle(1), + KeyCode::Char(' ') if w.current_field() == Field::Advanced => { + w.advanced_expanded = !w.advanced_expanded; + } // Tab on the Model field opens the local-path picker (Wave-0 primitive). KeyCode::Tab if w.current_field() == Field::Model => { let start = std::env::current_dir().unwrap_or_else(|_| std::path::PathBuf::from("/")); w.browser = Some(FolderBrowser::new("Pick a local model path", start)); } - KeyCode::Enter => { - if w.current_field() == Field::Launch { - request_launch(w); - } else if w.current_field() == Field::Model && !recipes.is_empty() { + KeyCode::Enter => match w.current_field() { + Field::Launch => request_launch(w), + // Expand/collapse only ever happens while Advanced owns focus, so + // focus lands back on Advanced in both directions. + Field::Advanced => w.advanced_expanded = !w.advanced_expanded, + Field::Host => w.editor = Some(InlineEditor::new(Field::Host, &w.host)), + Field::Port if w.port_mode == PortMode::Custom => { + w.editor = Some(InlineEditor::new(Field::Port, &w.port)); + } + Field::Model if !recipes.is_empty() => { // On the Model field, Enter opens the recipe picker (the // model_picker sub-step); free-text typing + Tab-browse remain. // Seed the filter with anything already typed so the picker @@ -330,10 +556,9 @@ pub fn on_key( query: w.model.trim().to_string(), selected: 0, }); - } else { - w.move_field(1); } - } + _ => w.move_field(1), + }, KeyCode::Backspace => w.backspace(), KeyCode::Char(c) => w.type_char(c), _ => {} @@ -341,25 +566,114 @@ pub fn on_key( Vec::new() } +/// Drive the inline editor. Enter accepts, Escape restores, and nothing here +/// can open another overlay. +fn editor_key(w: &mut ServeWizardState, code: KeyCode) { + if w.editor.is_none() { + return; + } + match code { + KeyCode::Esc => { + // Restore the original — an abandoned edit never mutates the form. + if let Some(ed) = w.editor.take() { + match ed.field { + Field::Host => w.host = ed.original, + Field::Port => w.port = ed.original, + _ => {} + } + } + refresh_message(w); + } + KeyCode::Left | KeyCode::Right => { + let delta = if code == KeyCode::Left { -1 } else { 1 }; + if let Some(ed) = w.editor.as_mut() { + ed.move_cursor(delta); + } + } + KeyCode::Backspace => { + if let Some(ed) = w.editor.as_mut() { + ed.backspace(); + } + } + KeyCode::Enter => { + let Some((field, value)) = w.editor.as_ref().map(|e| (e.field, e.value.clone())) else { + return; + }; + match field { + Field::Host => { + let trimmed = value.trim(); + if trimmed.is_empty() { + // Keep the editor open with a plain fix rather than + // silently accepting an unusable host. + w.message = Some(format!( + "host cannot be empty; type an address or press Esc to restore {DEFAULT_HOST}" + )); + return; + } + w.host = trimmed.to_string(); + } + Field::Port => { + if value.trim().is_empty() { + // Mirror the Host rule: an empty custom port is not a + // value, so keep the editor open with a plain fix. + w.message = Some( + "port cannot be empty; type a 1–65535 value or press Esc to cancel" + .to_string(), + ); + return; + } + // Digits-only is enforced on input; range validation happens + // at review time so an in-progress value stays visible. + w.port = value; + } + _ => {} + } + refresh_message(w); + w.editor = None; + } + KeyCode::Char(c) => { + if let Some(ed) = w.editor.as_mut() { + if ed.field == Field::Port && !c.is_ascii_digit() { + return; + } + ed.insert(c); + } + } + _ => {} + } +} + +/// Re-evaluate an *outstanding* validation complaint against the current form: +/// a genuine fix clears it, a still-broken form re-states the reason. Silent +/// when nothing has been complained about yet, so ordinary edits stay quiet. +fn refresh_message(w: &mut ServeWizardState) { + if w.message.is_some() { + w.message = w.validate().err().map(|(_, msg)| msg); + } +} + /// Validate the form and stage an approval (no job runs until approved). fn request_launch(w: &mut ServeWizardState) { match w.build_args() { Ok(args) => { let cmd = resolve_exe(); let cmdline = format!("{} {}", exe_label(&cmd), display_args(&args)); - let request = ApprovalRequest::new( - format!("serve “{}”", w.model.trim()), - vec![ - cmdline, - String::new(), - "This launches a local model server through the ROCm CLI.".to_string(), - if w.managed { - "Managed: it will appear in the services manager and dashboard.".to_string() - } else { - "Foreground: it runs in this job console until stopped.".to_string() - }, - ], - ); + let mut body = vec![ + cmdline, + String::new(), + "This launches a local model server through the ROCm CLI.".to_string(), + ]; + // Only claim an automatic port once the combination is valid — the + // concrete endpoint is printed by the CLI after it leases one. + if w.port_mode == PortMode::Auto { + body.push(AUTO_PORT_NOTE.to_string()); + } + body.push(if w.managed { + "Managed: it will appear in the services manager and dashboard.".to_string() + } else { + "Foreground: it runs in this job console until stopped.".to_string() + }); + let request = ApprovalRequest::new(format!("serve “{}”", w.model.trim()), body); w.message = None; w.approval = Some(PendingServe { cmd, @@ -368,7 +682,12 @@ fn request_launch(w: &mut ServeWizardState) { choice: ApprovalChoice::default(), }); } - Err(msg) => w.message = Some(msg), + Err((field, msg)) => { + // Move focus to the problem — expanding Advanced when the offending + // row is hidden — so the fix is always visible, never guessed at. + w.focus(field); + w.message = Some(msg); + } } } @@ -435,10 +754,12 @@ pub fn draw_serve_wizard( .split(inner); let has_recipes = !recipes.is_empty(); - let lines: Vec = FIELDS + let vis = w.visible_fields(); + let focused = w.field.min(vis.len() - 1); + let lines: Vec = vis .iter() .enumerate() - .map(|(i, field)| field_line(*field, i == w.field, w, has_recipes, theme)) + .map(|(i, field)| field_line(*field, i == focused, w, has_recipes, theme)) .collect(); f.render_widget(Paragraph::new(lines), rows[0]); @@ -451,14 +772,9 @@ pub fn draw_serve_wizard( rows[1], ); - let hint = if recipes.is_empty() { - "↑↓ field · ←→ change · Tab browse (model) · Enter next/launch · Esc close" - } else { - "↑↓ field · ←→ change · Enter pick (model)/next/launch · Tab browse · Esc close" - }; f.render_widget( Paragraph::new(Line::from(Span::styled( - hint, + footer_hint(w, has_recipes), Style::default().fg(theme.muted), ))), rows[2], @@ -476,6 +792,41 @@ pub fn draw_serve_wizard( } } +/// Footer help, derived from focus and edit state. Every hint names the row's +/// role in words (`choice`, `editable`, `read only`) so meaning never depends +/// on colour or glyphs alone. +fn footer_hint(w: &ServeWizardState, has_recipes: bool) -> &'static str { + if let Some(ed) = &w.editor { + return match ed.field { + Field::Port => { + "editing port · ←→ cursor · Backspace delete · Enter accept · Esc cancel" + } + _ => "editing host · ←→ cursor · Backspace delete · Enter accept · Esc cancel", + }; + } + match w.current_field() { + Field::Model if has_recipes => { + "editable · Enter pick a recipe · type a name · Tab browse a path · Esc close" + } + Field::Model => "editable · type a name or path · Tab browse a path · Esc close", + Field::Advanced if w.advanced_expanded => { + "Enter or ← collapse advanced settings · defaults stay automatic" + } + Field::Advanced => "Enter or → expand advanced settings · defaults stay automatic", + Field::Engine => "choice · ←→ pick an engine · automatic lets the CLI decide", + Field::Device => "read only · GPU required · ROCm never falls back to CPU", + Field::Host => "editable · Enter to edit the host · loopback only in Dash", + Field::Port if w.port_mode == PortMode::Custom => { + "choice · ←→ back to automatic · Enter to edit the port" + } + Field::Port => "choice · ←→ switch to custom · automatic picks a free local port", + Field::Mode => "choice · ←→ managed or foreground", + Field::Launch => "Enter to review the exact command · nothing runs before you approve", + } +} + +/// Render one row. Choice values carry chevrons, editable values brackets, and +/// the automatic device policy is plain read-only text. fn field_line<'a>( field: Field, selected: bool, @@ -483,43 +834,6 @@ fn field_line<'a>( has_recipes: bool, theme: &Theme, ) -> Line<'a> { - let model_placeholder = if has_recipes { - "(Enter to pick a recipe · type a name · Tab to browse)" - } else { - "(type a name / path, or Tab to browse)" - }; - let (label, value): (&str, String) = match field { - Field::Model => ( - "Model", - crate::ui::format::display_or_placeholder(&w.model, model_placeholder), - ), - Field::Engine => ( - "Engine", - ENGINES[w.engine_idx.min(ENGINES.len() - 1)].to_string(), - ), - Field::Device => ( - "Device", - DEVICES[w.device_idx.min(DEVICES.len() - 1)].to_string(), - ), - Field::Host => ( - "Host", - crate::ui::format::display_or_placeholder(&w.host, "(engine default)"), - ), - Field::Port => ( - "Port", - crate::ui::format::display_or_placeholder(&w.port, "(engine default)"), - ), - Field::Mode => ( - "Mode", - if w.managed { - "managed".to_string() - } else { - "foreground".to_string() - }, - ), - Field::Launch => ("", String::new()), - }; - if field == Field::Launch { let style = if selected { Style::default() @@ -545,13 +859,126 @@ fn field_line<'a>( } else { Style::default().fg(theme.fg) }; - Line::from(vec![ + + if field == Field::Advanced { + let chevron = if w.advanced_expanded { "▾" } else { "▸" }; + return Line::from(vec![ + Span::styled(marker, label_style), + Span::styled("Advanced settings ", label_style), + Span::styled(chevron, label_style), + Span::styled(format!(" {}", w.advanced_summary()), value_style), + ]); + } + + let model_placeholder = if has_recipes { + "(Enter to pick a recipe · type a name · Tab to browse)" + } else { + "(type a name / path, or Tab to browse)" + }; + + let label = match field { + Field::Model => "Model", + Field::Engine => "Engine", + Field::Device => "Device", + Field::Host => "Host", + Field::Port => "Port", + Field::Mode => "Mode", + Field::Advanced | Field::Launch => unreachable!(), + }; + + let mut spans = vec![ Span::styled(marker, label_style), Span::styled(format!("{label:<8}"), label_style), - Span::styled(value, value_style), - ]) + ]; + + // The editor, when open, replaces its own row's value with a live caret. + let editing = w.editor.as_ref().filter(|e| e.field == field); + + match field { + Field::Model => spans.push(Span::styled( + bracketed(&w.model, model_placeholder), + value_style, + )), + Field::Engine => spans.push(Span::styled( + chevroned(ENGINES[w.engine_idx.min(ENGINES.len() - 1)]), + value_style, + )), + // Read-only: no chevrons, no brackets — nothing to change here. + Field::Device => spans.push(Span::styled(DEVICE_POLICY, value_style)), + Field::Host => { + if let Some(ed) = editing { + spans.extend(editor_spans(ed, value_style, theme)); + } else { + spans.push(Span::styled(bracketed(&w.host, "(unset)"), value_style)); + } + } + Field::Port => { + let mode = match w.port_mode { + PortMode::Auto => "automatic", + PortMode::Custom => "custom", + }; + spans.push(Span::styled(chevroned(mode), value_style)); + if w.port_mode == PortMode::Custom { + spans.push(Span::styled(" ", value_style)); + if let Some(ed) = editing { + spans.extend(editor_spans(ed, value_style, theme)); + } else { + spans.push(Span::styled(bracketed(&w.port, "(unset)"), value_style)); + } + } + } + Field::Mode => spans.push(Span::styled( + chevroned(if w.managed { "managed" } else { "foreground" }), + value_style, + )), + Field::Advanced | Field::Launch => unreachable!(), + } + + Line::from(spans) } +/// Editable value with a visible caret (and a reversed run while the initial +/// value is still selected), rendered so the text stays contiguous. +fn editor_spans<'a>(ed: &'a InlineEditor, base: Style, theme: &Theme) -> Vec> { + let caret = Style::default() + .bg(theme.accent) + .fg(theme.bg) + .add_modifier(Modifier::BOLD); + let mut spans = vec![Span::styled("[", base)]; + if ed.selected && !ed.value.is_empty() { + spans.push(Span::styled(ed.value.as_str(), caret)); + } else { + let chars: Vec = ed.value.chars().collect(); + let cut = ed.cursor.min(chars.len()); + let before: String = chars[..cut].iter().collect(); + if !before.is_empty() { + spans.push(Span::styled(before, base)); + } + if cut < chars.len() { + spans.push(Span::styled(chars[cut].to_string(), caret)); + let after: String = chars[cut + 1..].iter().collect(); + if !after.is_empty() { + spans.push(Span::styled(after, base)); + } + } else { + spans.push(Span::styled(" ", caret)); + } + } + spans.push(Span::styled("]", base)); + spans +} + +fn bracketed(v: &str, placeholder: &'static str) -> String { + if v.is_empty() { + placeholder.to_string() + } else { + format!("[{v}]") + } +} + +fn chevroned(v: &str) -> String { + format!("‹ {v} ›") +} #[cfg(test)] mod tests { use super::*; @@ -565,6 +992,17 @@ mod tests { s.chars().map(|c| key(KeyCode::Char(c))).collect() } + fn feed(wiz: &mut Option, jobs: &mut State, keys: &[KeyCode]) { + for c in keys { + on_key(wiz, jobs, &[], key(*c)); + } + } + + /// Put focus on a row, expanding Advanced when that row is hidden. + fn focus_field(w: &mut ServeWizardState, field: Field) { + w.focus(field); + } + #[test] fn cycle_idx_wraps_both_directions() { assert_eq!(cycle_idx(0, 3, -1), 2); @@ -572,57 +1010,78 @@ mod tests { assert_eq!(cycle_idx(0, 0, 1), 0); } + // ---------------------------------------------------------------- defaults + #[test] - fn default_form_targets_managed_lemonade() { + fn default_form_is_fully_automatic_and_collapsed() { let w = ServeWizardState::default(); - assert!(w.managed); - assert_eq!(ENGINES[w.engine_idx], "lemonade"); - assert_eq!(w.device_idx, 0); // engine default → no --device + assert!(!w.advanced_expanded); + assert_eq!(w.visible_fields(), BASIC_FIELDS); + assert_eq!(w.engine_idx, ENGINE_AUTO); + assert_eq!(ENGINES[w.engine_idx], "automatic"); assert_eq!(w.host, "127.0.0.1"); - assert_eq!(w.port, "11435"); + assert_eq!(w.port_mode, PortMode::Auto); + assert!(w.managed); + assert_eq!(w.advanced_summary(), "Automatic"); + assert!(!w.is_customized()); } + // ------------------------------------------------------------------- argv + #[test] fn build_args_requires_a_model() { let w = ServeWizardState::default(); - assert_eq!(w.build_args().unwrap_err(), "model is required"); + let (field, msg) = w.build_args().unwrap_err(); + assert_eq!(field, Field::Model); + assert_eq!(msg, "model is required"); } #[test] - fn build_args_emits_managed_serve_with_defaults() { + fn default_argv_is_bare_serve_model() { + let w = ServeWizardState { + model: "qwen".into(), + ..Default::default() + }; + assert_eq!(w.build_args().unwrap(), vec!["serve", "qwen"]); + } + + #[test] + fn automatic_defaults_omit_engine_device_host_port_and_managed() { let w = ServeWizardState { model: "qwen".into(), ..Default::default() }; let args = w.build_args().unwrap(); - assert_eq!( - args, - vec![ - "serve", - "qwen", - "--engine", - "lemonade", - "--host", - "127.0.0.1", - "--port", - "11435", - "--managed", - ] - ); + for flag in ["--engine", "--device", "--host", "--port", "--managed"] { + assert!(!args.contains(&flag.to_string()), "{flag} must be omitted"); + } } #[test] - fn build_args_includes_device_when_not_default_and_foreground() { + fn explicit_overrides_emit_only_their_own_flags() { let w = ServeWizardState { model: "glm".into(), - device_idx: 1, // gpu_required + engine_idx: 2, // vllm + host: "localhost".into(), + port_mode: PortMode::Custom, + port: "8000".into(), managed: false, ..Default::default() }; - let args = w.build_args().unwrap(); - assert!(args.windows(2).any(|p| p == ["--device", "gpu_required"])); - assert!(args.contains(&"--foreground".to_string())); - assert!(!args.contains(&"--managed".to_string())); + assert_eq!( + w.build_args().unwrap(), + vec![ + "serve", + "glm", + "--engine", + "vllm", + "--host", + "localhost", + "--port", + "8000", + "--foreground", + ] + ); } #[test] @@ -641,59 +1100,558 @@ mod tests { } #[test] - fn build_args_rejects_bad_port() { + fn custom_port_on_default_host_emits_port_only() { let w = ServeWizardState { model: "m".into(), - port: "99999".into(), + port_mode: PortMode::Custom, + port: "11500".into(), ..Default::default() }; - assert!(w.build_args().unwrap_err().contains("port")); + assert_eq!( + w.build_args().unwrap(), + vec!["serve", "m", "--port", "11500"] + ); + } + + #[test] + fn build_args_rejects_out_of_range_and_zero_ports() { + for bad in ["99999", "0", "", " "] { + let w = ServeWizardState { + model: "m".into(), + port_mode: PortMode::Custom, + port: bad.into(), + ..Default::default() + }; + let (field, msg) = w.build_args().unwrap_err(); + assert_eq!(field, Field::Port, "{bad:?}"); + assert!(msg.contains("1–65535"), "{bad:?}: {msg}"); + } } #[test] - fn build_args_rejects_port_zero() { - // u16 parses 0, but it is not a bindable listen port — must be rejected - // in the form, matching the "1–65535" message. + fn empty_host_is_rejected_with_a_plain_fix() { let w = ServeWizardState { model: "m".into(), - port: "0".into(), + host: " ".into(), ..Default::default() }; - assert!(w.build_args().unwrap_err().contains("port")); + let (field, msg) = w.build_args().unwrap_err(); + assert_eq!(field, Field::Host); + assert!(msg.contains("host is required"), "{msg}"); } #[test] - fn port_field_accepts_digits_only() { - let mut w = ServeWizardState::default(); - w.port.clear(); - w.field = FIELDS.iter().position(|f| *f == Field::Port).unwrap(); - for k in typed("80a0") { - // route through type_char via the field - if let KeyCode::Char(c) = k.code { - w.type_char(c); - } + fn noncanonical_host_with_auto_port_is_blocked_with_the_contract_message() { + for host in ["localhost", "::1", "0.0.0.0", "192.168.1.5"] { + let w = ServeWizardState { + model: "m".into(), + host: host.into(), + port_mode: PortMode::Auto, + ..Default::default() + }; + let (field, msg) = w.build_args().unwrap_err(); + assert_eq!(field, Field::Port, "{host}"); + assert_eq!(msg, CUSTOM_HOST_NEEDS_PORT, "{host}"); + } + } + + #[test] + fn loopback_host_with_custom_port_is_allowed() { + for host in ["localhost", "::1", "[::1]", "LocalHost"] { + let w = ServeWizardState { + model: "m".into(), + host: host.into(), + port_mode: PortMode::Custom, + port: "9000".into(), + ..Default::default() + }; + assert_eq!( + w.build_args().unwrap(), + vec!["serve", "m", "--host", host, "--port", "9000"], + "{host}" + ); + } + } + + #[test] + fn public_host_never_reaches_approval_even_with_a_custom_port() { + // Dash has no --allow-public-bind confirmation and no endpoint-key UI, + // so a public bind must be requested from the CLI instead. + for host in ["0.0.0.0", "192.168.1.5", "::", "example.invalid"] { + let w = ServeWizardState { + model: "m".into(), + host: host.into(), + port_mode: PortMode::Custom, + port: "8000".into(), + ..Default::default() + }; + let (field, msg) = w.build_args().unwrap_err(); + assert_eq!(field, Field::Host, "{host}"); + assert_eq!(msg, PUBLIC_HOST_NEEDS_CLI, "{host}"); } - assert_eq!(w.port, "800"); } + #[test] + fn public_host_launch_stages_no_approval() { + let mut wiz = Some(ServeWizardState { + model: "m".into(), + host: "0.0.0.0".into(), + port_mode: PortMode::Custom, + port: "8000".into(), + ..Default::default() + }); + let mut jobs = State::default(); + wiz.as_mut().unwrap().focus(Field::Launch); + let fx = on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Enter)); + assert!(fx.is_empty()); + let w = wiz.as_ref().unwrap(); + assert!(w.approval.is_none(), "a public bind never reaches approval"); + assert!(w.advanced_expanded, "the offending row is revealed"); + assert_eq!(w.current_field(), Field::Host); + assert_eq!(w.message.as_deref(), Some(PUBLIC_HOST_NEEDS_CLI)); + assert!(jobs.jobs.is_empty()); + } + + #[test] + fn is_loopback_host_matches_the_cli_accepted_spellings() { + for ok in ["127.0.0.1", "localhost", "::1", "[::1]", "LOCALHOST"] { + assert!(is_loopback_host(ok), "{ok}"); + } + for bad in ["0.0.0.0", "127.0.0.2", "::", "192.168.1.5", ""] { + assert!(!is_loopback_host(bad), "{bad}"); + } + } + + // -------------------------------------------------------- disclosure/focus + + #[test] + fn advanced_expands_and_collapses_only_while_focused_and_keeps_focus() { + let mut wiz = Some(ServeWizardState::default()); + let mut jobs = State::default(); + // Model has focus: Right must not expand anything. + feed(&mut wiz, &mut jobs, &[KeyCode::Right]); + assert!(!wiz.as_ref().unwrap().advanced_expanded); + + // Down → Advanced, Enter expands, focus stays on Advanced. + feed(&mut wiz, &mut jobs, &[KeyCode::Down, KeyCode::Enter]); + { + let w = wiz.as_ref().unwrap(); + assert!(w.advanced_expanded); + assert_eq!(w.visible_fields(), FIELDS); + assert_eq!(w.current_field(), Field::Advanced); + } + // Enter collapses again, still on Advanced. + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + { + let w = wiz.as_ref().unwrap(); + assert!(!w.advanced_expanded); + assert_eq!(w.current_field(), Field::Advanced); + } + // ← collapses / → expands from the same row. + feed(&mut wiz, &mut jobs, &[KeyCode::Right]); + assert!(wiz.as_ref().unwrap().advanced_expanded); + feed(&mut wiz, &mut jobs, &[KeyCode::Left]); + assert!(!wiz.as_ref().unwrap().advanced_expanded); + assert_eq!(wiz.as_ref().unwrap().current_field(), Field::Advanced); + } + + #[test] + fn expanded_focus_order_is_model_advanced_engine_device_host_port_mode_launch() { + let mut w = ServeWizardState { + advanced_expanded: true, + ..Default::default() + }; + let mut seen = Vec::new(); + for _ in 0..FIELDS.len() { + seen.push(w.current_field()); + w.move_field(1); + } + assert_eq!(seen, FIELDS.to_vec()); + // Clamped at the end. + assert_eq!(w.current_field(), Field::Launch); + } + + #[test] + fn collapsing_preserves_overrides_and_reports_customized() { + let mut w = ServeWizardState { + advanced_expanded: true, + engine_idx: 1, + port_mode: PortMode::Custom, + port: "8000".into(), + ..Default::default() + }; + w.advanced_expanded = false; + assert_eq!(w.engine_idx, 1); + assert_eq!(w.port, "8000"); + assert_eq!(w.advanced_summary(), "Customized"); + } + + #[test] + fn invalid_hidden_custom_port_says_it_needs_attention() { + let w = ServeWizardState { + model: "m".into(), + port_mode: PortMode::Custom, + port: "abc".into(), + ..Default::default() + }; + assert!(!w.advanced_expanded); + assert_eq!(w.advanced_summary(), "Customized · port needs attention"); + assert!(w.build_args().is_err(), "launch stays blocked"); + } + + #[test] + fn out_of_range_hidden_port_also_needs_attention() { + let w = ServeWizardState { + port_mode: PortMode::Custom, + port: "70000".into(), + ..Default::default() + }; + assert_eq!(w.advanced_summary(), "Customized · port needs attention"); + } + + #[test] + fn invalid_combination_expands_advanced_and_focuses_the_problem() { + let mut wiz = Some(ServeWizardState { + model: "m".into(), + host: "localhost".into(), + ..Default::default() + }); + let mut jobs = State::default(); + wiz.as_mut().unwrap().field = 2; // Launch (collapsed order) + let fx = on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Enter)); + assert!(fx.is_empty(), "nothing may run"); + let w = wiz.as_ref().unwrap(); + assert!(w.approval.is_none(), "invalid combos never reach approval"); + assert!(w.advanced_expanded, "the hidden problem is revealed"); + assert_eq!(w.current_field(), Field::Port); + assert_eq!(w.message.as_deref(), Some(CUSTOM_HOST_NEEDS_PORT)); + assert_eq!(w.host, "localhost", "both values preserved"); + assert_eq!(w.port_mode, PortMode::Auto); + } + + // --------------------------------------------------------------- controls + + #[test] + fn left_right_cycles_engine_including_automatic() { + let mut wiz = Some(ServeWizardState { + advanced_expanded: true, + ..Default::default() + }); + let mut jobs = State::default(); + focus_field(wiz.as_mut().unwrap(), Field::Engine); + feed(&mut wiz, &mut jobs, &[KeyCode::Right]); + assert_eq!(ENGINES[wiz.as_ref().unwrap().engine_idx], "lemonade"); + feed(&mut wiz, &mut jobs, &[KeyCode::Right]); + assert_eq!(ENGINES[wiz.as_ref().unwrap().engine_idx], "vllm"); + feed(&mut wiz, &mut jobs, &[KeyCode::Right]); + assert_eq!(wiz.as_ref().unwrap().engine_idx, ENGINE_AUTO, "wraps"); + } + + #[test] + fn left_right_toggles_port_mode_only_when_focused() { + let mut wiz = Some(ServeWizardState { + advanced_expanded: true, + ..Default::default() + }); + let mut jobs = State::default(); + focus_field(wiz.as_mut().unwrap(), Field::Device); + feed(&mut wiz, &mut jobs, &[KeyCode::Right]); + assert_eq!( + wiz.as_ref().unwrap().port_mode, + PortMode::Auto, + "Device is read-only and cannot change the port" + ); + focus_field(wiz.as_mut().unwrap(), Field::Port); + feed(&mut wiz, &mut jobs, &[KeyCode::Right]); + assert_eq!(wiz.as_ref().unwrap().port_mode, PortMode::Custom); + feed(&mut wiz, &mut jobs, &[KeyCode::Left]); + assert_eq!(wiz.as_ref().unwrap().port_mode, PortMode::Auto); + } + + #[test] + fn mode_toggles_between_managed_and_foreground() { + let mut wiz = Some(ServeWizardState { + advanced_expanded: true, + ..Default::default() + }); + let mut jobs = State::default(); + focus_field(wiz.as_mut().unwrap(), Field::Mode); + feed(&mut wiz, &mut jobs, &[KeyCode::Char(' ')]); + assert!(!wiz.as_ref().unwrap().managed); + feed(&mut wiz, &mut jobs, &[KeyCode::Left]); + assert!(wiz.as_ref().unwrap().managed); + } + + #[test] + fn typing_only_edits_the_model_row() { + let mut wiz = Some(ServeWizardState { + advanced_expanded: true, + ..Default::default() + }); + let mut jobs = State::default(); + focus_field(wiz.as_mut().unwrap(), Field::Host); + for k in typed("zzz") { + on_key(&mut wiz, &mut jobs, &[], k); + } + assert_eq!( + wiz.as_ref().unwrap().host, + "127.0.0.1", + "host only changes through the inline editor" + ); + focus_field(wiz.as_mut().unwrap(), Field::Model); + for k in typed("qwen") { + on_key(&mut wiz, &mut jobs, &[], k); + } + assert_eq!(wiz.as_ref().unwrap().model, "qwen"); + feed(&mut wiz, &mut jobs, &[KeyCode::Backspace]); + assert_eq!(wiz.as_ref().unwrap().model, "qwe"); + } + + // ----------------------------------------------------------- inline editor + + #[test] + fn host_editor_selects_on_entry_and_first_char_replaces() { + let mut wiz = Some(ServeWizardState { + advanced_expanded: true, + ..Default::default() + }); + let mut jobs = State::default(); + focus_field(wiz.as_mut().unwrap(), Field::Host); + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + { + let ed = wiz.as_ref().unwrap().editor.as_ref().unwrap(); + assert_eq!(ed.field, Field::Host); + assert!(ed.selected); + assert_eq!(ed.value, "127.0.0.1"); + } + for k in typed("0.0.0.0") { + on_key(&mut wiz, &mut jobs, &[], k); + } + assert_eq!( + wiz.as_ref().unwrap().editor.as_ref().unwrap().value, + "0.0.0.0" + ); + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + let w = wiz.as_ref().unwrap(); + assert!(w.editor.is_none()); + assert_eq!(w.host, "0.0.0.0"); + } + + #[test] + fn editor_cursor_insert_and_backspace() { + let mut wiz = Some(ServeWizardState { + advanced_expanded: true, + host: "abc".into(), + ..Default::default() + }); + let mut jobs = State::default(); + focus_field(wiz.as_mut().unwrap(), Field::Host); + feed(&mut wiz, &mut jobs, &[KeyCode::Enter, KeyCode::Left]); + // Left cleared the selection and moved the cursor to before 'c'. + { + let ed = wiz.as_ref().unwrap().editor.as_ref().unwrap(); + assert!(!ed.selected); + assert_eq!(ed.cursor, 2); + } + on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Char('X'))); + assert_eq!(wiz.as_ref().unwrap().editor.as_ref().unwrap().value, "abXc"); + feed(&mut wiz, &mut jobs, &[KeyCode::Backspace]); + assert_eq!(wiz.as_ref().unwrap().editor.as_ref().unwrap().value, "abc"); + feed(&mut wiz, &mut jobs, &[KeyCode::Right, KeyCode::Right]); + assert_eq!(wiz.as_ref().unwrap().editor.as_ref().unwrap().cursor, 3); + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + assert_eq!(wiz.as_ref().unwrap().host, "abc"); + } + + #[test] + fn editor_escape_restores_the_original_value() { + let mut wiz = Some(ServeWizardState { + advanced_expanded: true, + ..Default::default() + }); + let mut jobs = State::default(); + focus_field(wiz.as_mut().unwrap(), Field::Host); + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + for k in typed("nonsense") { + on_key(&mut wiz, &mut jobs, &[], k); + } + feed(&mut wiz, &mut jobs, &[KeyCode::Esc]); + let w = wiz.as_ref().unwrap(); + assert!(w.editor.is_none(), "Esc closes the editor"); + assert!(wiz.is_some(), "Esc in the editor never closes the overlay"); + assert_eq!(w.host, "127.0.0.1", "original restored"); + } + + #[test] + fn host_editor_trims_and_rejects_blank() { + let mut wiz = Some(ServeWizardState { + advanced_expanded: true, + ..Default::default() + }); + let mut jobs = State::default(); + focus_field(wiz.as_mut().unwrap(), Field::Host); + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + for k in typed(" ") { + on_key(&mut wiz, &mut jobs, &[], k); + } + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + { + let w = wiz.as_ref().unwrap(); + assert!(w.editor.is_some(), "blank host keeps the editor open"); + assert_eq!(w.host, "127.0.0.1", "form untouched"); + assert!( + w.message + .as_deref() + .unwrap() + .contains("host cannot be empty") + ); + } + for k in typed(" 10.0.0.7 ") { + on_key(&mut wiz, &mut jobs, &[], k); + } + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + let w = wiz.as_ref().unwrap(); + assert!(w.editor.is_none()); + assert_eq!(w.host, "10.0.0.7", "trimmed on accept"); + } + + #[test] + fn port_editor_accepts_digits_only() { + let mut wiz = Some(ServeWizardState { + advanced_expanded: true, + port_mode: PortMode::Custom, + ..Default::default() + }); + let mut jobs = State::default(); + focus_field(wiz.as_mut().unwrap(), Field::Port); + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + for k in typed("80a0!") { + on_key(&mut wiz, &mut jobs, &[], k); + } + assert_eq!(wiz.as_ref().unwrap().editor.as_ref().unwrap().value, "800"); + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + assert_eq!(wiz.as_ref().unwrap().port, "800"); + } + + #[test] + fn port_editor_rejects_an_empty_value_like_host_does() { + let mut wiz = Some(ServeWizardState { + advanced_expanded: true, + model: "m".into(), + port_mode: PortMode::Custom, + ..Default::default() + }); + let mut jobs = State::default(); + focus_field(wiz.as_mut().unwrap(), Field::Port); + // Enter selects "11435"; Backspace clears the whole selection. + feed(&mut wiz, &mut jobs, &[KeyCode::Enter, KeyCode::Backspace]); + assert_eq!(wiz.as_ref().unwrap().editor.as_ref().unwrap().value, ""); + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + { + let w = wiz.as_ref().unwrap(); + assert!(w.editor.is_some(), "empty port keeps the editor open"); + assert_eq!(w.port, "11435", "form untouched"); + let msg = w.message.as_deref().unwrap(); + assert!(msg.contains("port cannot be empty"), "{msg}"); + assert!(msg.contains("1–65535"), "{msg}"); + } + // Typing a real value accepts normally and clears the complaint. + for k in typed("9001") { + on_key(&mut wiz, &mut jobs, &[], k); + } + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + let w = wiz.as_ref().unwrap(); + assert!(w.editor.is_none()); + assert_eq!(w.port, "9001"); + assert!(w.message.is_none(), "a fixed form stops complaining"); + } + + #[test] + fn escaping_an_editor_recomputes_an_outstanding_complaint() { + // A blank-host complaint must not survive the Esc that restores a + // perfectly good host… + let mut wiz = Some(ServeWizardState { + advanced_expanded: true, + model: "m".into(), + ..Default::default() + }); + let mut jobs = State::default(); + focus_field(wiz.as_mut().unwrap(), Field::Host); + feed( + &mut wiz, + &mut jobs, + &[KeyCode::Enter, KeyCode::Backspace, KeyCode::Enter], + ); + assert!(wiz.as_ref().unwrap().message.is_some()); + feed(&mut wiz, &mut jobs, &[KeyCode::Esc]); + let w = wiz.as_ref().unwrap(); + assert_eq!(w.host, "127.0.0.1"); + assert!(w.message.is_none(), "stale complaint cleared by the fix"); + + // …but a still-broken form keeps saying why. + let mut wiz = Some(ServeWizardState { + advanced_expanded: true, + model: "m".into(), + host: "localhost".into(), + ..Default::default() + }); + wiz.as_mut().unwrap().message = Some(CUSTOM_HOST_NEEDS_PORT.to_string()); + focus_field(wiz.as_mut().unwrap(), Field::Host); + feed(&mut wiz, &mut jobs, &[KeyCode::Enter, KeyCode::Esc]); + let w = wiz.as_ref().unwrap(); + assert_eq!(w.host, "localhost"); + assert_eq!(w.message.as_deref(), Some(CUSTOM_HOST_NEEDS_PORT)); + } + + #[test] + fn enter_on_auto_port_advances_instead_of_editing() { + let mut wiz = Some(ServeWizardState { + advanced_expanded: true, + ..Default::default() + }); + let mut jobs = State::default(); + focus_field(wiz.as_mut().unwrap(), Field::Port); + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + let w = wiz.as_ref().unwrap(); + assert!(w.editor.is_none()); + assert_eq!(w.current_field(), Field::Mode); + } + + #[test] + fn editor_navigation_keys_cannot_move_form_focus() { + let mut wiz = Some(ServeWizardState { + advanced_expanded: true, + ..Default::default() + }); + let mut jobs = State::default(); + focus_field(wiz.as_mut().unwrap(), Field::Host); + let before = wiz.as_ref().unwrap().field; + feed( + &mut wiz, + &mut jobs, + &[KeyCode::Enter, KeyCode::Up, KeyCode::Down], + ); + let w = wiz.as_ref().unwrap(); + assert!(w.editor.is_some()); + assert_eq!(w.field, before, "form navigation is inert while editing"); + } + + // -------------------------------------------------------------- approval + #[test] fn launch_requires_approval_then_spawns_job() { let mut wiz = Some(ServeWizardState::default()); let mut jobs = State::default(); - // Fill the model by typing on the Model field (field 0 by default). for k in typed("qwen") { on_key(&mut wiz, &mut jobs, &[], k); } assert_eq!(wiz.as_ref().unwrap().model, "qwen"); - // Jump to Launch and press Enter → approval staged, NO job yet. - let launch_idx = FIELDS.iter().position(|f| *f == Field::Launch).unwrap(); - wiz.as_mut().unwrap().field = launch_idx; + wiz.as_mut().unwrap().focus(Field::Launch); let fx = on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Enter)); assert!(fx.is_empty(), "launch must not run before approval"); assert!(wiz.as_ref().unwrap().approval.is_some()); assert!(jobs.jobs.is_empty()); - // Approve → exactly one SpawnJob, job registered, console active. let fx = on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Char('y'))); assert_eq!(fx.len(), 1); assert!(matches!(fx[0], SideEffect::SpawnJob { .. })); @@ -703,17 +1661,49 @@ mod tests { assert_eq!(jobs.jobs.len(), 1); } + #[test] + fn approval_shows_exact_argv_and_the_automatic_port_phrase() { + let mut w = ServeWizardState { + model: "qwen".into(), + ..Default::default() + }; + request_launch(&mut w); + let pending = w.approval.as_ref().unwrap(); + assert_eq!(pending.args, vec!["serve", "qwen"]); + let body = pending.request.body.join("\n"); + assert!(body.contains("serve qwen"), "{body}"); + assert!(body.contains(AUTO_PORT_NOTE), "{body}"); + } + + #[test] + fn approval_omits_the_automatic_phrase_for_a_custom_port() { + let mut w = ServeWizardState { + model: "qwen".into(), + port_mode: PortMode::Custom, + port: "8000".into(), + ..Default::default() + }; + request_launch(&mut w); + let pending = w.approval.as_ref().unwrap(); + assert!(pending.args.windows(2).any(|p| p == ["--port", "8000"])); + assert!(!pending.request.body.join("\n").contains(AUTO_PORT_NOTE)); + } + #[test] fn empty_model_launch_sets_message_not_approval() { let mut wiz = Some(ServeWizardState::default()); let mut jobs = State::default(); - let launch_idx = FIELDS.iter().position(|f| *f == Field::Launch).unwrap(); - wiz.as_mut().unwrap().field = launch_idx; + wiz.as_mut().unwrap().focus(Field::Launch); let fx = on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Enter)); assert!(fx.is_empty()); let w = wiz.as_ref().unwrap(); assert!(w.approval.is_none()); assert_eq!(w.message.as_deref(), Some("model is required")); + assert_eq!( + w.current_field(), + Field::Model, + "focus lands on the problem" + ); } #[test] @@ -721,7 +1711,7 @@ mod tests { let mut wiz = Some(ServeWizardState::default()); let mut jobs = State::default(); wiz.as_mut().unwrap().model = "m".into(); - wiz.as_mut().unwrap().field = FIELDS.iter().position(|f| *f == Field::Launch).unwrap(); + wiz.as_mut().unwrap().focus(Field::Launch); on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Enter)); let fx = on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Char('n'))); assert!(fx.is_empty()); @@ -742,7 +1732,7 @@ mod tests { let mut wiz = Some(ServeWizardState::default()); let mut jobs = State::default(); wiz.as_mut().unwrap().model = "m".into(); - wiz.as_mut().unwrap().field = FIELDS.iter().position(|f| *f == Field::Launch).unwrap(); + wiz.as_mut().unwrap().focus(Field::Launch); on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Enter)); on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Char('y'))); assert!(wiz.as_ref().unwrap().active_job.is_some()); @@ -757,7 +1747,7 @@ mod tests { let mut wiz = Some(ServeWizardState::default()); let mut jobs = State::default(); wiz.as_mut().unwrap().model = "m".into(); - wiz.as_mut().unwrap().field = FIELDS.iter().position(|f| *f == Field::Launch).unwrap(); + wiz.as_mut().unwrap().focus(Field::Launch); on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Enter)); on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Char('y'))); assert!(wiz.as_ref().unwrap().active_job.is_some()); @@ -768,7 +1758,7 @@ mod tests { let mut wiz = Some(ServeWizardState::default()); let mut jobs = State::default(); wiz.as_mut().unwrap().model = "m".into(); - wiz.as_mut().unwrap().field = FIELDS.iter().position(|f| *f == Field::Launch).unwrap(); + wiz.as_mut().unwrap().focus(Field::Launch); on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Enter)); on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Char('y'))); let job_id = wiz.as_ref().unwrap().active_job.clone().unwrap(); @@ -790,13 +1780,12 @@ mod tests { // NOT claim success (set active_job) when no SpawnJob was emitted. let mut wiz = Some(ServeWizardState::default()); let mut jobs = State::default(); - let launch = FIELDS.iter().position(|f| *f == Field::Launch).unwrap(); // First launch of "qwen": real spawn. for k in typed("qwen") { on_key(&mut wiz, &mut jobs, &[], k); } - wiz.as_mut().unwrap().field = launch; + wiz.as_mut().unwrap().focus(Field::Launch); on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Enter)); on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Char('y'))); assert_eq!( @@ -810,7 +1799,7 @@ mod tests { for k in typed("qwen") { on_key(&mut wiz2, &mut jobs, &[], k); } - wiz2.as_mut().unwrap().field = launch; + wiz2.as_mut().unwrap().focus(Field::Launch); on_key(&mut wiz2, &mut jobs, &[], key(KeyCode::Enter)); let fx = on_key(&mut wiz2, &mut jobs, &[], key(KeyCode::Char('y'))); // No new SpawnJob, no stale console, an informative message instead. @@ -826,6 +1815,8 @@ mod tests { assert_eq!(jobs.jobs.len(), 1, "still just the one job"); } + // ------------------------------------------------------- picker / browser + #[test] fn tab_on_model_opens_folder_browser() { let mut wiz = Some(ServeWizardState::default()); @@ -839,18 +1830,7 @@ mod tests { } #[test] - fn left_right_cycles_engine() { - let mut wiz = Some(ServeWizardState::default()); - let mut jobs = State::default(); - wiz.as_mut().unwrap().field = FIELDS.iter().position(|f| *f == Field::Engine).unwrap(); - on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Right)); - assert_eq!(wiz.as_ref().unwrap().engine_idx, 1); - on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Left)); - assert_eq!(wiz.as_ref().unwrap().engine_idx, 0); - } - - #[test] - fn enter_on_model_opens_picker_and_choice_fills_model_and_engine() { + fn enter_on_model_opens_picker_and_choice_fills_model_only() { let recipes = vec![ModelRecipeSummary { id: "GLM-4".into(), aliases: vec!["glm".into()], @@ -859,18 +1839,17 @@ mod tests { }]; let mut wiz = Some(ServeWizardState::default()); // field 0 = Model let mut jobs = State::default(); - // Enter on Model opens the picker when recipes exist. on_key(&mut wiz, &mut jobs, &recipes, key(KeyCode::Enter)); assert!(wiz.as_ref().unwrap().picker.is_some()); - // Enter in the picker chooses the (only) recipe → fills model + engine. on_key(&mut wiz, &mut jobs, &recipes, key(KeyCode::Enter)); let w = wiz.as_ref().unwrap(); assert!(w.picker.is_none()); assert_eq!(w.model, "GLM-4"); assert_eq!( - ENGINES[w.engine_idx], "vllm", - "preferred engine pre-selected" + w.engine_idx, ENGINE_AUTO, + "a recipe never forces an engine override — the CLI resolves it" ); + assert_eq!(w.build_args().unwrap(), vec!["serve", "GLM-4"]); } #[test] @@ -879,29 +1858,10 @@ mod tests { let mut jobs = State::default(); on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Enter)); assert!(wiz.as_ref().unwrap().picker.is_none()); - assert_eq!(wiz.as_ref().unwrap().field, 1, "Enter advances to Engine"); - } - - #[test] - fn recipe_with_unknown_preferred_engine_leaves_engine_idx() { - // Pins the silent-fallback contract: a preferred_engine the wizard does - // not list must NOT crash and must leave the engine choice untouched - // (model still filled). Guards future ENGINES vs rocm-core divergence. - let recipes = vec![ModelRecipeSummary { - id: "some-model".into(), - aliases: vec![], - task: "chat".into(), - preferred_engine: Some("not-in-engines-list".into()), - }]; - let mut wiz = Some(ServeWizardState::default()); // engine_idx 0 = lemonade - let mut jobs = State::default(); - on_key(&mut wiz, &mut jobs, &recipes, key(KeyCode::Enter)); // open picker - on_key(&mut wiz, &mut jobs, &recipes, key(KeyCode::Enter)); // choose first - let w = wiz.as_ref().unwrap(); - assert_eq!(w.model, "some-model"); assert_eq!( - w.engine_idx, 0, - "unknown preferred engine leaves the choice" + wiz.as_ref().unwrap().current_field(), + Field::Advanced, + "Enter advances to the disclosure row" ); } @@ -923,7 +1883,6 @@ mod tests { ]; let mut wiz = Some(ServeWizardState::default()); let mut jobs = State::default(); - // Type "qwen" on the Model field, then Enter to open the picker. for k in typed("qwen") { on_key(&mut wiz, &mut jobs, &recipes, k); } @@ -931,7 +1890,6 @@ mod tests { let picker = wiz.as_ref().unwrap().picker.as_ref().unwrap(); assert_eq!(picker.query, "qwen"); assert_eq!(picker.filtered(&recipes).len(), 1, "pre-narrowed to Qwen"); - // Enter chooses the single match. on_key(&mut wiz, &mut jobs, &recipes, key(KeyCode::Enter)); assert_eq!(wiz.as_ref().unwrap().model, "Qwen3-4B"); } @@ -954,6 +1912,8 @@ mod tests { assert!(wiz.is_some(), "picker Esc keeps the wizard open"); } + // ---------------------------------------------------------------- render + fn render(w: &ServeWizardState, jobs: &State) -> String { use ratatui::Terminal; use ratatui::backend::TestBackend; @@ -969,14 +1929,98 @@ mod tests { .collect() } + /// A rendered form row starts with the 2-cell focus marker and an + /// 8-cell-padded label, so `Mode` is `"Mode "` and can never be matched + /// by the `Model` row. Bare substrings are not precise enough here. + fn row_label(label: &str) -> String { + format!("{label:<8}") + } + #[test] - fn snapshot_renders_form_fields() { + fn collapsed_render_shows_only_model_advanced_and_launch() { let w = ServeWizardState::default(); let out = render(&w, &State::default()); assert!(out.contains("Serve a model"), "titled overlay"); - assert!(out.contains("Model"), "model field"); - assert!(out.contains("lemonade"), "default engine shown"); - assert!(out.contains("Launch"), "launch action"); + assert!(out.contains(&row_label("Model")), "model row"); + assert!(out.contains("Advanced settings ▸"), "collapsed disclosure"); + assert!(out.contains("Automatic"), "automatic summary"); + assert!(out.contains("[ Launch ]"), "launch action"); + for hidden in ["Engine", "Device", "Host", "Port", "Mode"] { + assert!( + !out.contains(&row_label(hidden)), + "{hidden} row must stay hidden: {out:?}" + ); + } + } + + #[test] + fn expanded_render_inlines_the_advanced_rows_without_a_second_modal() { + let w = ServeWizardState { + advanced_expanded: true, + ..Default::default() + }; + let out = render(&w, &State::default()); + assert!(out.contains("Advanced settings ▾"), "expanded disclosure"); + for shown in ["Model", "Engine", "Device", "Host", "Port", "Mode"] { + assert!( + out.contains(&row_label(shown)), + "{shown} row missing: {out:?}" + ); + } + assert!(out.contains("[ Launch ]"), "launch action"); + // Role affordances: chevrons for choices, brackets for editable text, + // plain read-only text for the GPU-required policy. + assert!(out.contains("‹ automatic ›"), "choice chevrons: {out:?}"); + assert!(out.contains("[127.0.0.1]"), "editable brackets: {out:?}"); + assert!( + out.contains(DEVICE_POLICY), + "read-only device text: {out:?}" + ); + assert!(!out.contains("Review:"), "no second modal"); + } + + #[test] + fn collapsed_render_reports_a_broken_hidden_port() { + let w = ServeWizardState { + model: "m".into(), + port_mode: PortMode::Custom, + port: "abc".into(), + ..Default::default() + }; + let out = render(&w, &State::default()); + assert!(out.contains("port needs attention"), "{out:?}"); + } + + #[test] + fn footer_names_the_role_of_the_focused_row() { + let mut w = ServeWizardState { + advanced_expanded: true, + ..Default::default() + }; + w.focus(Field::Engine); + assert!(render(&w, &State::default()).contains("choice")); + w.focus(Field::Device); + assert!(render(&w, &State::default()).contains("read only")); + w.focus(Field::Host); + assert!(render(&w, &State::default()).contains("editable")); + // Port names the direction it would move in, not a generic pair. + w.focus(Field::Port); + assert!(render(&w, &State::default()).contains("choice · ←→ switch to custom")); + w.port_mode = PortMode::Custom; + assert!( + render(&w, &State::default()) + .contains("choice · ←→ back to automatic · Enter to edit the port") + ); + w.focus(Field::Host); + let host = w.host.clone(); + w.editor = Some(InlineEditor::new(Field::Host, &host)); + assert!(render(&w, &State::default()).contains("Esc cancel")); + } + + #[test] + fn no_generic_change_footer_remains() { + let w = ServeWizardState::default(); + assert!(!render(&w, &State::default()).contains("←→ change")); } #[test] @@ -984,11 +2028,12 @@ mod tests { let mut wiz = Some(ServeWizardState::default()); let mut jobs = State::default(); wiz.as_mut().unwrap().model = "qwen".into(); - wiz.as_mut().unwrap().field = FIELDS.iter().position(|f| *f == Field::Launch).unwrap(); + wiz.as_mut().unwrap().focus(Field::Launch); on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Enter)); let out = render(wiz.as_ref().unwrap(), &jobs); assert!(out.contains("Review:"), "approval modal shown"); assert!(out.contains("serve"), "describes the gated launch"); assert!(out.contains("Approve"), "approve button present"); + assert!(out.contains("endpoint shown after launch"), "{out:?}"); } } diff --git a/crates/rocm-dash-tui/src/ui/tabs/serving.rs b/crates/rocm-dash-tui/src/ui/tabs/serving.rs index 8ce64b34c..5a0dbd7d2 100644 --- a/crates/rocm-dash-tui/src/ui/tabs/serving.rs +++ b/crates/rocm-dash-tui/src/ui/tabs/serving.rs @@ -25,11 +25,11 @@ pub const VERBS: &[Verb] = &[ summary: "Launch a model on a serving engine and expose an OpenAI-style endpoint.", steps: &[ "Pick a model", - "Choose an engine — Lemonade · vLLM", - "Set GPU placement (required / preferred / CPU-only)", - "Launch on 127.0.0.1:11435 and watch it come up", + "Leave the rest automatic, or open Advanced settings", + "GPU is required — ROCm never falls back to CPU", + "Launch on a local endpoint picked for you and watch it come up", ], - cmd: "rocm serve --engine …", + cmd: "rocm serve ", read_only: false, badge: None, }, @@ -156,6 +156,28 @@ mod tests { .collect() } + #[test] + fn serving_first_view_promises_no_concrete_endpoint() { + // The CLI leases the local port at launch time, so the first thing a + // user reads must not promise a fixed `127.0.0.1:11435` endpoint. + let mut s = AppState::new("t".into(), "default-dark".into()); + s.active_tab = ActiveTab::Serving; + s.serving_sel = 0; // "Serve a model" + let out = render(&s, 200, 30); + assert!( + !out.contains("11435"), + "first-view copy must not promise a fixed port: {out:?}" + ); + assert!( + !out.contains("127.0.0.1"), + "first-view copy must not promise a fixed host: {out:?}" + ); + assert!( + out.contains("Launch on a local endpoint picked for you"), + "automatic endpoint wording missing: {out:?}" + ); + } + #[test] fn serving_renders_its_six_rows() { let mut s = AppState::new("t".into(), "default-dark".into()); From f7920a18f94974b05a9d36f7e7342b14416dc725 Mon Sep 17 00:00:00 2001 From: Michael Roy Date: Tue, 18 Aug 2026 09:38:03 -0700 Subject: [PATCH 02/19] fix: address collision-safe serving review Signed-off-by: Michael Roy --- apps/rocm/src/serve_port.rs | 5 +- crates/rocm-dash-tui/src/app/mod.rs | 31 ++++++++++++- crates/rocm-dash-tui/src/ui/serve_wizard.rs | 46 ++++++++++++------- .../features/model_serving.feature | 11 +++++ tests/e2e-cucumber/tests/e2e.rs | 4 ++ tests/e2e-cucumber/tests/e2e/serving_steps.rs | 45 ++++++++++++++++++ 6 files changed, 121 insertions(+), 21 deletions(-) diff --git a/apps/rocm/src/serve_port.rs b/apps/rocm/src/serve_port.rs index e99b47334..baefa089e 100644 --- a/apps/rocm/src/serve_port.rs +++ b/apps/rocm/src/serve_port.rs @@ -152,10 +152,7 @@ pub(crate) fn resolve_serve_port( // — and it is refused by returning an error, never by panicking. let Some(port) = request.explicit() else { validate_port_request(host, request)?; - bail!( - "`--host {host}` needs an explicit `--port <1-65535>`; automatic port \ - selection is supported only on {DEFAULT_LOCAL_HOST}." - ); + unreachable!("custom-host automatic requests are rejected above"); }; return Ok(LoopbackPortLease::engine_owned(port)); } diff --git a/crates/rocm-dash-tui/src/app/mod.rs b/crates/rocm-dash-tui/src/app/mod.rs index 28687dd1b..68c84c8ec 100644 --- a/crates/rocm-dash-tui/src/app/mod.rs +++ b/crates/rocm-dash-tui/src/app/mod.rs @@ -915,7 +915,8 @@ impl AppState { /// (and is ignored while a job runs) before it can eject the manager. pub(crate) fn active_overlay_at_root(&self) -> bool { self.serve_wizard.as_ref().is_none_or(|w| { - w.browser.is_none() + w.editor.is_none() + && w.browser.is_none() && w.picker.is_none() && w.approval.is_none() && w.active_job.is_none() @@ -1795,6 +1796,34 @@ mod tests { ); } + #[test] + fn serve_wizard_editor_handles_escape_before_overlay_back_out() { + let mut state = AppState::new("t".into(), "default-dark".into()); + state.active_tab = ActiveTab::Serving; + let mut wizard = crate::ui::serve_wizard::ServeWizardState { + model: "typed-model".into(), + ..Default::default() + }; + wizard.editor = Some(crate::ui::serve_wizard::InlineEditor { + field: crate::ui::serve_wizard::Field::Host, + original: "127.0.0.1".into(), + value: "localhost".into(), + cursor: 9, + selected: false, + }); + state.serve_wizard = Some(wizard); + + assert!( + !state.should_pane_back_out(crossterm::event::KeyCode::Esc), + "Esc must reach the inline editor instead of closing the wizard" + ); + assert_eq!( + state.serve_wizard.as_ref().unwrap().model, + "typed-model", + "the in-progress form remains available to the editor handler" + ); + } + /// `active_overlay_at_root`'s per-manager clauses are a hand-maintained /// enumeration of each manager's nested sub-view fields (documented on /// `OnboardingState`, which lists every struct this covers). Nothing stops diff --git a/crates/rocm-dash-tui/src/ui/serve_wizard.rs b/crates/rocm-dash-tui/src/ui/serve_wizard.rs index a5d77043c..c5f0e7ff4 100644 --- a/crates/rocm-dash-tui/src/ui/serve_wizard.rs +++ b/crates/rocm-dash-tui/src/ui/serve_wizard.rs @@ -81,18 +81,16 @@ pub const CUSTOM_HOST_NEEDS_PORT: &str = "Custom hosts require a custom port; se /// /// Dash has no public-bind confirmation or endpoint-key UI, so a public bind /// must be requested deliberately from the CLI instead. -pub const LOOPBACK_HOSTS: &[&str] = &["127.0.0.1", "localhost", "::1", "[::1]"]; +pub const LOOPBACK_HOSTS: &[&str] = &["127.0.0.1", "localhost", "::1"]; /// The guidance shown when a public / non-loopback host is typed into Dash. pub const PUBLIC_HOST_NEEDS_CLI: &str = "Dash serves on loopback only; for a public bind run \ `rocm serve --host … --port … --allow-public-bind` from the CLI."; -/// Whether `host` is one of the loopback spellings Dash may bind. +/// Whether `host` is one of the exact loopback spellings accepted by the CLI. #[must_use] pub fn is_loopback_host(host: &str) -> bool { - LOOPBACK_HOSTS - .iter() - .any(|known| known.eq_ignore_ascii_case(host)) + LOOPBACK_HOSTS.contains(&host) } /// The form fields. Not every field is visible at once — see @@ -1156,8 +1154,8 @@ mod tests { } #[test] - fn loopback_host_with_custom_port_is_allowed() { - for host in ["localhost", "::1", "[::1]", "LocalHost"] { + fn exact_cli_loopback_hosts_with_custom_port_are_allowed() { + for host in ["127.0.0.1", "localhost", "::1"] { let w = ServeWizardState { model: "m".into(), host: host.into(), @@ -1165,11 +1163,12 @@ mod tests { port: "9000".into(), ..Default::default() }; - assert_eq!( - w.build_args().unwrap(), - vec!["serve", "m", "--host", host, "--port", "9000"], - "{host}" - ); + let expected = if host == DEFAULT_HOST { + vec!["serve", "m", "--port", "9000"] + } else { + vec!["serve", "m", "--host", host, "--port", "9000"] + }; + assert_eq!(w.build_args().unwrap(), expected, "{host}"); } } @@ -1177,7 +1176,14 @@ mod tests { fn public_host_never_reaches_approval_even_with_a_custom_port() { // Dash has no --allow-public-bind confirmation and no endpoint-key UI, // so a public bind must be requested from the CLI instead. - for host in ["0.0.0.0", "192.168.1.5", "::", "example.invalid"] { + for host in [ + "0.0.0.0", + "192.168.1.5", + "::", + "example.invalid", + "LocalHost", + "[::1]", + ] { let w = ServeWizardState { model: "m".into(), host: host.into(), @@ -1213,11 +1219,19 @@ mod tests { } #[test] - fn is_loopback_host_matches_the_cli_accepted_spellings() { - for ok in ["127.0.0.1", "localhost", "::1", "[::1]", "LOCALHOST"] { + fn is_loopback_host_matches_the_cli_exactly() { + for ok in ["127.0.0.1", "localhost", "::1"] { assert!(is_loopback_host(ok), "{ok}"); } - for bad in ["0.0.0.0", "127.0.0.2", "::", "192.168.1.5", ""] { + for bad in [ + "0.0.0.0", + "127.0.0.2", + "::", + "192.168.1.5", + "", + "LocalHost", + "[::1]", + ] { assert!(!is_loopback_host(bad), "{bad}"); } } diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 90caa1aee..e02c5598e 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -29,6 +29,17 @@ Feature: Model serving When the user lists running services Then the connection details match the actual server port + # Real CLI coverage for automatic port allocation. A loopback listener holds + # the legacy default while one real managed GPU serve starts; the reported + # endpoint must advance to the next candidate rather than failing late in the + # engine. Runs in the merge queue because it launches a real model server. + @id:serve-auto-port-skips-occupied @requires-gpu @merge-queue + Scenario: 15 - Automatic serving skips an occupied default port + Given a managed runtime is active + And the default serve port is occupied + When the user serves a model with automatic port selection + Then the service uses the next automatic port + # vLLM serve + inference (safetensors model). Engine coverage: vLLM. This is the # deliberate vLLM half of a per-engine pair with `serve-lemonade-inference` # below, so it stays pinned to vLLM (the slug names the engine). It is also the diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index cab0f3bef..32f88f669 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -63,6 +63,9 @@ pub struct E2eWorld { /// so the port keeps answering, as a started ComfyUI's does, for the rest /// of the scenario. pub comfyui_port_listener: Option, + /// Listener that deliberately occupies the default serve port while a real + /// `rocm serve` invocation proves automatic allocation advances past it. + pub automatic_port_guard: Option, /// Cache-marker destination discovered from `rocmd`'s own JSON report. pub artifact_marker_path: Option, pub endpoint: Option, @@ -297,6 +300,7 @@ impl Default for E2eWorld { artifact_server: None, paced_download_server: None, comfyui_port_listener: None, + automatic_port_guard: None, artifact_marker_path: None, endpoint: None, model_name: None, diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 7ce18b326..49e0f3c83 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -394,6 +394,18 @@ async fn setup_mock_custom_port(world: &mut E2eWorld) { world.mock = Some(mock); } +#[given("the default serve port is occupied")] +async fn occupy_default_serve_port(world: &mut E2eWorld) { + let _ = ensure_serve_port_free().await; + let next = std::net::TcpListener::bind(("127.0.0.1", SERVE_PORT + 1)) + .expect("the next automatic port must be free before this scenario starts"); + drop(next); + world.automatic_port_guard = Some( + std::net::TcpListener::bind(("127.0.0.1", SERVE_PORT)) + .expect("failed to occupy the default serve port"), + ); +} + /// The (model, engine, ready-substring) this host should serve for an /// engine-agnostic "serve a real model" precondition. /// @@ -858,6 +870,17 @@ async fn user_serves_with_failing_lemonade_preparation(world: &mut E2eWorld) { world.cli_rc = Some(rc); } +#[when("the user serves a model with automatic port selection")] +async fn user_serves_with_automatic_port(world: &mut E2eWorld) { + let (model, engine, _) = host_serve_target(); + let (stdout, stderr, rc) = + crate::run_rocm(world, &["serve", model, "--engine", engine, "--managed"]); + world.automatic_port_guard.take(); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + #[when("the user sends a chat completion request")] async fn user_sends_completion(world: &mut E2eWorld) { crate::send_chat(world).await; @@ -1414,6 +1437,28 @@ async fn assert_endpoint_port(world: &mut E2eWorld) { ); } +#[then("the service uses the next automatic port")] +async fn assert_next_automatic_port(world: &mut E2eWorld) { + let stdout = world.cli_output.as_deref().unwrap_or(""); + let stderr = world.cli_stderr.as_deref().unwrap_or(""); + let rc = world.cli_rc.expect("no serve command was run"); + assert!( + rc == 0, + "{}", + e2e_cucumber::cli_failure_report( + &["serve", "", "--engine", "", "--managed"], + rc, + stdout, + stderr, + ) + ); + let expected = format!("endpoint: http://127.0.0.1:{}/v1", SERVE_PORT + 1); + assert!( + stdout.lines().any(|line| line.trim() == expected), + "expected `{expected}` in real serve output:\n{stdout}" + ); +} + /// Extract the engine name from a serve plan's `engine: ` line. fn selected_engine(output: &str) -> &str { let Some(engine) = output From e9fc8a159464ffe1c38e947f22f80c13ed30916a Mon Sep 17 00:00:00 2001 From: Juho Vainio Date: Tue, 13 Oct 2026 11:23:26 +0300 Subject: [PATCH 03/19] fix(serve): keep failed-launch artifacts when a child actually spawned - abandon_failed_managed_launch no longer deletes the manifest/log/engine-state for a launch that spawned a real child; it marks the record status "failed" and clears only the endpoint key, so `rocm services`/`rocm logs` still has something to show. Pre-spawn failures (no child ever ran) are still fully removed. - document the new automatic-port-selection behavior in README.md and docs/testing.md (range, per-host explicit-port requirement, focused test commands) - add e2e-cucumber coverage for the reviewed gaps: a non-canonical host without an explicit port, and an explicit port that collides with a live reservation or falls outside 1-65535 - fix an orphaned doc comment in serving_steps.rs left over from the previous step insertion, which had migrated onto the wrong function Signed-off-by: Juho Vainio --- README.md | 12 +++ apps/rocm/src/service_launch.rs | 64 +++++++++---- docs/testing.md | 17 ++++ .../features/model_serving.feature | 25 +++++ .../e2e-cucumber/features/networking.feature | 13 +++ tests/e2e-cucumber/tests/e2e/serving_steps.rs | 95 ++++++++++++++++++- 6 files changed, 205 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 082a922af..6bdb1501c 100644 --- a/README.md +++ b/README.md @@ -768,6 +768,18 @@ rocm serve [--engine lemonade|vllm] [--gpu-memory-utilization FRACTION] [--tool-call-parser NAME] ``` +`--port` is optional on the default loopback host (`127.0.0.1`): omit it and +`rocm serve` picks the first free port in `11435`-`11535`, checking both +rocm-cli's own records and the OS so two servers never collide. The deployment +summary shows `Port: automatic; endpoint shown after launch` while the launch +is in flight and prints the concrete endpoint once the port is actually leased. +Pass `--port` explicitly to pin one; an explicit loopback port already +reserved or in use is rejected up front, naming the server holding it and how +to free it (`rocm services stop --yes`). `--host` to anything +other than the default loopback address (`localhost`, `::1`, `0.0.0.0`, or any +other address) requires an explicit `--port`: automatic selection only probes +`127.0.0.1` and cannot promise a free port anywhere else. + `--gpu-memory-utilization` (a fraction above 0.0 and up to 1.0 of each GPU's total VRAM) and `--tool-call-parser` (for example `hermes`, `llama3_json`, or `mistral`, which also turns on automatic tool choice) apply to vLLM only. Lower diff --git a/apps/rocm/src/service_launch.rs b/apps/rocm/src/service_launch.rs index c1786f583..9b253ac29 100644 --- a/apps/rocm/src/service_launch.rs +++ b/apps/rocm/src/service_launch.rs @@ -385,16 +385,17 @@ const MANAGED_LAUNCH_CLEANUP_GRACE: Duration = Duration::from_secs(2); /// so no failed transaction leaves a live reservation or a stray endpoint key. /// /// Order matters: a spawned child is terminated and confirmed dead *first*, so -/// nothing is still holding the port or writing to the log when the artifacts — -/// including the record that reserves the port for other launches — are removed. +/// nothing is still holding the port or writing to the log by the time cleanup +/// runs. If no child was ever spawned (`record.engine_pid` is still `None`), +/// every artifact this attempt created is removed: there is nothing for +/// `rocm services`/`rocm logs` to show. If a child did run, its record and log +/// are kept instead (marked `status = "failed"`), since the error that led here +/// already told the user to go inspect that log; only the endpoint key, which a +/// failed attempt has no use for, is cleared. /// Every step is best-effort and idempotent; cleanup must never mask the /// original failure with a second one. -fn abandon_failed_managed_launch( - paths: &AppPaths, - record: &ManagedServiceRecord, - child_pid: Option, -) { - if let Some(pid) = child_pid.filter(|pid| *pid != 0) { +fn abandon_failed_managed_launch(paths: &AppPaths, record: &ManagedServiceRecord) { + if let Some(pid) = record.engine_pid.filter(|pid| *pid != 0) { let outcome = rocm_core::terminate_verified( &rocm_core::ProcessIdentity::capture(pid), rocm_core::KillScope::Tree, @@ -412,6 +413,11 @@ fn abandon_failed_managed_launch( ), Some(&record.service_id), ); + let mut failed = record.clone(); + failed.status = "failed".to_owned(); + let _ = failed.write(); + endpoint_keys::clear_endpoint_api_key(paths, &record.service_id); + return; } for path in [ record.manifest_path.as_path(), @@ -588,10 +594,11 @@ fn spawn_managed_engine_child( child_pid, }), Err(error) => { - // Pre-spawn failures reach here with nothing running; a post-spawn - // publication failure already terminated its child, and the removals - // are idempotent. - abandon_failed_managed_launch(paths, &record, None); + // The single cleanup choke point for every failure path below the + // record's first write: `record.engine_pid` tells it whether a child + // ever ran, so a pre-spawn failure deletes everything and a post-spawn + // one keeps the record and log (marked `status = "failed"`) instead. + abandon_failed_managed_launch(paths, &record); Err(error) } } @@ -704,6 +711,9 @@ fn launch_managed_engine_child_locked( .spawn() .context("failed to launch managed engine process")?; let child_pid = child.id(); + // Set before the immediate-exit check below, so a bail! here still leaves + // `record.engine_pid` telling cleanup that a child was spawned. + record.engine_pid = Some(child_pid); thread::sleep(Duration::from_millis(200)); if let Some(status) = child .try_wait() @@ -730,7 +740,10 @@ fn launch_managed_engine_child_locked( record.record_engine_identity(child_pid); record.status = "running".to_owned(); if let Err(error) = record.write() { - abandon_failed_managed_launch(paths, record, Some(child_pid)); + // Cleanup is the caller's job now: `record.engine_pid` (set above) is + // already enough for `abandon_failed_managed_launch` to terminate this + // child, so there is nothing left for this inner path to do beyond + // surfacing the error. return Err(error.context("failed to publish the managed service record")); } @@ -2366,7 +2379,7 @@ mod tests { fn abandoned_launch_leaves_no_record_log_or_endpoint_key() -> Result<()> { let (root, paths) = test_paths("abandon-launch-cleanup"); paths.ensure()?; - let record = write_live_service_record(&paths, "lemonade-abandon", "abandon-model", 11_490, "running")?; + let mut record = write_live_service_record(&paths, "lemonade-abandon", "abandon-model", 11_490, "running")?; fs::create_dir_all( record .engine_state_path @@ -2375,8 +2388,10 @@ mod tests { )?; fs::write(&record.engine_state_path, "{}")?; endpoint_keys::store_endpoint_api_key(&paths, &record.service_id, "secret-key")?; + // Simulate a pre-spawn failure: nothing ever ran. + record.engine_pid = None; - abandon_failed_managed_launch(&paths, &record, None); + abandon_failed_managed_launch(&paths, &record); assert!(!record.manifest_path.exists()); assert!(!record.log_path.exists()); @@ -2399,7 +2414,7 @@ mod tests { fn abandoned_launch_terminates_the_spawned_child() -> Result<()> { let (root, paths) = test_paths("abandon-launch-terminates-child"); paths.ensure()?; - let record = + let mut record = write_live_service_record(&paths, "lemonade-abandon-child", "abandon-child-model", 11_491, "running")?; let current_exe = std::env::current_exe().expect("current test exe"); @@ -2412,11 +2427,11 @@ mod tests { .spawn() .expect("spawn sleeper child"); let child_pid = child.id(); + record.engine_pid = Some(child_pid); - abandon_failed_managed_launch(&paths, &record, Some(child_pid)); + abandon_failed_managed_launch(&paths, &record); let wait_result = child.wait(); - let _ = fs::remove_dir_all(root); match wait_result { Ok(status) => assert!( @@ -2425,7 +2440,18 @@ mod tests { ), Err(error) => panic!("failed to wait for sleeper child: {error}"), } - assert!(!record.manifest_path.exists()); + let persisted = load_managed_services(&paths) + .expect("re-read manifests") + .into_iter() + .find(|candidate| candidate.service_id == record.service_id); + let _ = fs::remove_dir_all(root); + let persisted = persisted.expect( + "a launch that spawned a child must keep its record so `rocm services`/`rocm logs` can show the attempt", + ); + assert_eq!( + persisted.status, "failed", + "a spawned child's record must be marked failed, not deleted" + ); Ok(()) } } diff --git a/docs/testing.md b/docs/testing.md index 30b1486e4..0ed561fcd 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -944,6 +944,23 @@ cargo test -p rocmd recovery_respawn_fails_closed cargo test -p rocmd apply_endpoint_key_env ``` +Collision-safe port selection, focused tests: + +```bash +cargo test -p rocm --bin rocm -- serve_port:: +cargo test -p rocm --bin rocm automatic_selects_against_real_loopback_listeners +cargo test -p rocm --bin rocm concurrent_auto_transactions_publish_distinct_ports +cargo test -p rocm --bin rocm concurrent_equivalent_transactions_converge_on_one_record +cargo test -p rocmd recovery_waits_for_the_allocation_lock_and_refuses_an_occupied_port +``` + +Omitting `--port` on the default loopback host scans `11435..=11535` for a +candidate neither reserved by a live managed-service record nor occupied on the +OS (`serve_port.rs`), under the same `services/launch.lock` that serializes +every other managed launch and `rocmd` recovery attempt. An explicit `--port` +on a non-default `--host` is still required: automatic selection only probes +`127.0.0.1` and refuses to guess elsewhere. + A loopback bind (`rocm serve `, default `127.0.0.1`) stays credential-free. A public bind (`--host 0.0.0.0 --allow-public-bind`) requires an API key: pass one with `--api-key` / the `ROCM_SERVE_API_KEY` environment diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index e02c5598e..ded08139b 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -352,3 +352,28 @@ Feature: Model serving Scenario: serve-25 - The recommended model list prints ASCII-only text When the user lists recommended models Then the model list output contains only ASCII characters + + # Parse-time refusal (serve-port rewrite): an out-of-range `--port` (0 here; + # 65536 and non-numeric values share the same parser and message) is rejected + # before any engine or GPU work, so it needs no GPU and gates every PR + # (ungated), like its sibling serve-14. + @id:serve-invalid-port-rejected + Scenario: serve-26 - Serving with an out-of-range port is refused with a clear reason + When the user serves a model with an out-of-range port + Then serving is refused before any engine starts + And the user is told the port must be between 1 and 65535 + + # Collision-safe explicit ports (serve-port rewrite). An explicit `--port` + # equal to one a live managed service already holds must be refused before any + # bind is attempted, naming the service so the user can stop it or pick + # another port. Reaching this check needs a real GPU: `resolve_serve_port` + # only runs once `spawn_managed_engine_child` has a completed model/engine + # resolution in hand, and `serve` has no CPU fallback that gets there. + # Runs in the merge queue, like its automatic-port sibling serve-auto-port-skips-occupied. + @id:serve-explicit-port-reserved @requires-gpu @merge-queue + Scenario: serve-27 - Serving on a port another managed service already holds is refused + Given a managed runtime is active + And another managed service already reserves the default serve port + When the user serves a model explicitly on the reserved port + Then serving is refused before any engine starts + And the user is told to stop the service holding that port diff --git a/tests/e2e-cucumber/features/networking.feature b/tests/e2e-cucumber/features/networking.feature index fe16f4075..7282ba394 100644 --- a/tests/e2e-cucumber/features/networking.feature +++ b/tests/e2e-cucumber/features/networking.feature @@ -39,3 +39,16 @@ Feature: Native HTTP networking When the user serves a model bound to a public interface without allowing public binding Then serving is refused before any engine starts And the user is told to allow public binding first + + # Collision-safe ports (serve-port rewrite). Automatic port selection proves a + # port is free by probing it on loopback before the engine starts; that proof + # only applies to the canonical default host (127.0.0.1), so any other host + # name (even a loopback alias like `localhost` that needs no public-bind + # opt-in) must come with an explicit `--port`. This check runs in + # `validate_port_request`, before any engine or GPU work, so it needs no GPU + # and runs on the mock lane every PR. + @id:networking-custom-host-needs-explicit-port + Scenario: networking-04 - A non-canonical host without an explicit port is refused up front + When the user serves a model bound to a non-canonical host without an explicit port + Then serving is refused before any engine starts + And the user is told to pass an explicit port for that host diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 49e0f3c83..d3c04c77a 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -9,7 +9,7 @@ use std::time::{Duration, Instant}; use cucumber::{given, then, when}; use crate::E2eWorld; -use e2e_cucumber::mock_server::{MockServer, ServiceRecordOptions}; +use e2e_cucumber::mock_server::{MockServer, ServiceRecordOptions, write_service_record_with}; use e2e_cucumber::serve_log::{ ServeAttempt, archive_service_log, serve_attempt_report, service_log_tail, }; @@ -406,6 +406,24 @@ async fn occupy_default_serve_port(world: &mut E2eWorld) { ); } +/// Plant a live managed-service record on the default serve port, with no real +/// listener behind it. `resolve_serve_port`'s explicit-port check matches on the +/// record's `port` field alone, before it ever probes the OS (see +/// `reserved_service_ports`'s "conservative by design" doc comment), so a live +/// record is enough to make the collision deterministic without a second real +/// server. +#[given("another managed service already reserves the default serve port")] +async fn reserve_default_serve_port(world: &mut E2eWorld) { + let root = world.isolated_root.as_ref().expect("no isolated root"); + let services = root.path().join("data").join("services"); + let options = ServiceRecordOptions { + supervisor_pid: std::process::id(), + engine_pid: Some(std::process::id()), + ..ServiceRecordOptions::default() + }; + write_service_record_with(&services, "other-model", SERVE_PORT, options); +} + /// The (model, engine, ready-substring) this host should serve for an /// engine-agnostic "serve a real model" precondition. /// @@ -1051,8 +1069,53 @@ async fn user_serves_public_bind_no_optin(world: &mut E2eWorld) { world.cli_rc = Some(rc); } +/// `localhost` is loopback (`is_public_bind_host` returns false, so no +/// `--allow-public-bind` opt-in is required) but is not the canonical +/// `127.0.0.1` `validate_port_request` requires for automatic port selection. +/// This isolates the host-vs-port-explicitness check from the separate +/// public-bind-opt-in check `networking-03` already covers. +#[when("the user serves a model bound to a non-canonical host without an explicit port")] +async fn user_serves_non_canonical_host_no_port(world: &mut E2eWorld) { + let (stdout, stderr, rc) = + crate::run_rocm(world, &["serve", "some-model", "--host", "localhost"]); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +/// `--port 0` is rejected by `serve_port::parse_serve_port`'s own range check, +/// which also covers 65536 and non-numeric values via the same message. +#[when("the user serves a model with an out-of-range port")] +async fn user_serves_invalid_port(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm(world, &["serve", "some-model", "--port", "0"]); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +#[when("the user serves a model explicitly on the reserved port")] +async fn user_serves_explicit_reserved_port(world: &mut E2eWorld) { + let (model, engine, _) = host_serve_target(); + let port = SERVE_PORT.to_string(); + let (stdout, stderr, rc) = crate::run_rocm( + world, + &[ + "serve", + model, + "--engine", + engine, + "--managed", + "--port", + &port, + ], + ); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + /// Serve naming both a runtime and an environment. These are mutually exclusive -/// selectors, rejected during argument parsing before any engine or GPU work — so +/// selectors, rejected during argument parsing before any engine or GPU work, so /// the refusal is observable with no GPU (mock lane). #[when("the user serves a model selecting both a runtime and an environment")] async fn user_serves_runtime_and_env(world: &mut E2eWorld) { @@ -1144,6 +1207,34 @@ async fn assert_public_bind_message(world: &mut E2eWorld) { ); } +#[then("the user is told to pass an explicit port for that host")] +async fn assert_custom_host_needs_port_message(world: &mut E2eWorld) { + let output = serve_output(world); + assert!( + output.contains("needs an explicit `--port"), + "expected a message asking for an explicit --port, got:\n{output}" + ); +} + +#[then("the user is told the port must be between 1 and 65535")] +async fn assert_invalid_port_message(world: &mut E2eWorld) { + let output = serve_output(world); + assert!( + output.contains("port must be a number between 1 and 65535"), + "expected the port range message, got:\n{output}" + ); +} + +#[then("the user is told to stop the service holding that port")] +async fn assert_port_reserved_message(world: &mut E2eWorld) { + let output = serve_output(world); + assert!( + output.contains("is already reserved by managed service e2e-mock") + && output.contains("rocm services stop e2e-mock --yes"), + "expected the port-reserved message naming e2e-mock, got:\n{output}" + ); +} + #[then("the user is told the two selectors cannot be combined")] async fn assert_selector_conflict_message(world: &mut E2eWorld) { let output = serve_output(world); From febc88fa0c1d709979271e3651449b7944e1c305 Mon Sep 17 00:00:00 2001 From: Juho Vainio Date: Tue, 13 Oct 2026 13:09:33 +0300 Subject: [PATCH 04/19] fix(serve): address remaining non-blocking port-safety review notes Three small follow-ups from PR #243 review, independently re-verified against HEAD before fixing: - Dash's validate() checked the custom-host auto-port rule before the loopback rule, so a public host (e.g. 0.0.0.0) was told to "set Port to Custom" even though that would still be refused afterward. Swap the order so the real blocker is named first. - apps/rocm/src/serve_port.rs and the Dash wizard each declared their own copy of the "Port: automatic" disclosure string, kept in sync by a doc comment's claim rather than the compiler. serve_port.rs now re-exports the wizard's AUTO_PORT_NOTE instead of duplicating it. - The e2e precondition that probes SERVE_PORT+1 is free panicked hard on a stale listener left by a prior scenario. Clear it the same way ensure_serve_port_free already clears the default port. Signed-off-by: Juho Vainio --- apps/rocm/src/serve_port.rs | 6 ++-- crates/rocm-dash-tui/src/ui/serve_wizard.rs | 33 +++++++++++++++---- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 5 +++ 3 files changed, 34 insertions(+), 10 deletions(-) diff --git a/apps/rocm/src/serve_port.rs b/apps/rocm/src/serve_port.rs index baefa089e..903c78685 100644 --- a/apps/rocm/src/serve_port.rs +++ b/apps/rocm/src/serve_port.rs @@ -37,9 +37,9 @@ pub(crate) const AUTO_PORT_SPAN: u16 = 100; pub(crate) const AUTO_PORT_LAST: u16 = AUTO_PORT_FIRST + AUTO_PORT_SPAN; /// The one phrase used wherever a port must be described *before* it is -/// resolved. The CLI plan output and the Dash approval card share it verbatim so -/// neither promises a concrete port it has not yet acquired. -pub(crate) const AUTOMATIC_PORT_DISCLOSURE: &str = "Port: automatic; endpoint shown after launch"; +/// resolved. Re-exported from the Dash approval card's own constant so the CLI +/// plan output and the wizard can never drift apart in wording. +pub(crate) use rocm_dash_tui::ui::serve_wizard::AUTO_PORT_NOTE as AUTOMATIC_PORT_DISCLOSURE; /// What the user asked for on the command line. #[derive(Clone, Copy, Debug, PartialEq, Eq)] diff --git a/crates/rocm-dash-tui/src/ui/serve_wizard.rs b/crates/rocm-dash-tui/src/ui/serve_wizard.rs index c5f0e7ff4..41fc018c2 100644 --- a/crates/rocm-dash-tui/src/ui/serve_wizard.rs +++ b/crates/rocm-dash-tui/src/ui/serve_wizard.rs @@ -369,17 +369,20 @@ impl ServeWizardState { format!("host is required; type an address or restore {DEFAULT_HOST}"), )); } - // Automatic port selection is only supported on the canonical loopback - // host — anything else must name its own port. Never invent one. - if host != DEFAULT_HOST && self.port_mode == PortMode::Auto { - return Err((Field::Port, CUSTOM_HOST_NEEDS_PORT.to_string())); - } // Dash has no `--allow-public-bind` confirmation and no endpoint-key // UI, so it must never stage a public bind — even with an explicit - // port. That deliberate step belongs to the CLI. + // port. That deliberate step belongs to the CLI. Checked before the + // auto-port rule below: a public host is refused outright regardless + // of port mode, so its error must name that, not dead-end the user + // into switching to a custom port that would be refused anyway. if !is_loopback_host(host) { return Err((Field::Host, PUBLIC_HOST_NEEDS_CLI.to_string())); } + // Automatic port selection is only supported on the canonical loopback + // host — anything else must name its own port. Never invent one. + if host != DEFAULT_HOST && self.port_mode == PortMode::Auto { + return Err((Field::Port, CUSTOM_HOST_NEEDS_PORT.to_string())); + } if self.port_mode == PortMode::Custom { let Some(port) = parse_port(&self.port) else { let shown = self.port.trim(); @@ -1140,7 +1143,9 @@ mod tests { #[test] fn noncanonical_host_with_auto_port_is_blocked_with_the_contract_message() { - for host in ["localhost", "::1", "0.0.0.0", "192.168.1.5"] { + // Loopback but not the canonical default: auto-port is refused with the + // "set Port to Custom" guidance, which actually resolves the problem. + for host in ["localhost", "::1"] { let w = ServeWizardState { model: "m".into(), host: host.into(), @@ -1151,6 +1156,20 @@ mod tests { assert_eq!(field, Field::Port, "{host}"); assert_eq!(msg, CUSTOM_HOST_NEEDS_PORT, "{host}"); } + // Not loopback at all: refused outright regardless of port mode, so the + // error must point at the real blocker instead of dead-ending the user + // into a custom port that would be refused anyway. + for host in ["0.0.0.0", "192.168.1.5"] { + let w = ServeWizardState { + model: "m".into(), + host: host.into(), + port_mode: PortMode::Auto, + ..Default::default() + }; + let (field, msg) = w.build_args().unwrap_err(); + assert_eq!(field, Field::Host, "{host}"); + assert_eq!(msg, PUBLIC_HOST_NEEDS_CLI, "{host}"); + } } #[test] diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index d3c04c77a..df4651d1a 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -397,6 +397,11 @@ async fn setup_mock_custom_port(world: &mut E2eWorld) { #[given("the default serve port is occupied")] async fn occupy_default_serve_port(world: &mut E2eWorld) { let _ = ensure_serve_port_free().await; + // A stale listener from a prior scenario (not just an unrelated process) + // is the realistic way this probe finds the next port taken; clear it the + // same way `ensure_serve_port_free` clears the default port instead of + // panicking the whole run over leftover state. + kill_listeners_on_port(SERVE_PORT + 1); let next = std::net::TcpListener::bind(("127.0.0.1", SERVE_PORT + 1)) .expect("the next automatic port must be free before this scenario starts"); drop(next); From 41ad840a8a570f537393cf8533f3dd42c5c06e96 Mon Sep 17 00:00:00 2001 From: Juho Vainio Date: Thu, 15 Oct 2026 12:08:49 +0300 Subject: [PATCH 05/19] fix(serve): address review follow-up for port-collision safety - recovery/restart re-checks live records under the shared launch lock and reports a failed recovery as a server-recover/error event instead of aborting the rest of that watcher tick for every other managed service - a failed collision check during rocm serve leaves no orphaned endpoint-key file behind - Dash accepts only the exact loopback spellings the CLI accepts; LocalHost and [::1] are rejected - the app-level Escape gate treats the inline editor as a nested layer, so Escape cancels the edit instead of closing the wizard - drop a redundant clone in the recovery test harness; document why LoopbackPortLease::is_held stays pub despite no production caller - the lock-hold test tolerates timer coalescing a loaded full-suite run can introduce, instead of asserting an exact wall-clock bound - cargo fmt --all (whitespace drift left over from the main merge) - doc and feature-file updates reflecting the above Signed-off-by: Juho Vainio --- README.md | 21 +- apps/rocm/src/chat.rs | 52 ++ apps/rocm/src/main.rs | 150 +++++- apps/rocm/src/serve_cmd.rs | 14 +- apps/rocm/src/serve_port.rs | 17 +- apps/rocm/src/service_launch.rs | 472 +++++++++++++++++- apps/rocmd/src/mcp.rs | 36 +- apps/rocmd/src/watchers.rs | 306 ++++++++++-- crates/rocm-core/src/lib.rs | 86 +--- crates/rocm-core/src/process.rs | 152 ++++++ crates/rocm-core/src/service_record.rs | 11 + crates/rocm-dash-tui/src/agent/tools.rs | 2 +- crates/rocm-dash-tui/src/app/mod.rs | 1 - crates/rocm-dash-tui/src/ui/serve_wizard.rs | 24 +- docs/manual-testing.md | 13 + docs/testing.md | 2 +- skills/rocm-cli-assistant/SKILL.md | 1 + .../features/model_serving.feature | 6 +- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 27 +- 19 files changed, 1214 insertions(+), 179 deletions(-) diff --git a/README.md b/README.md index 6bdb1501c..0931298d3 100644 --- a/README.md +++ b/README.md @@ -770,15 +770,18 @@ rocm serve [--engine lemonade|vllm] `--port` is optional on the default loopback host (`127.0.0.1`): omit it and `rocm serve` picks the first free port in `11435`-`11535`, checking both -rocm-cli's own records and the OS so two servers never collide. The deployment -summary shows `Port: automatic; endpoint shown after launch` while the launch -is in flight and prints the concrete endpoint once the port is actually leased. -Pass `--port` explicitly to pin one; an explicit loopback port already -reserved or in use is rejected up front, naming the server holding it and how -to free it (`rocm services stop --yes`). `--host` to anything -other than the default loopback address (`localhost`, `::1`, `0.0.0.0`, or any -other address) requires an explicit `--port`: automatic selection only probes -`127.0.0.1` and cannot promise a free port anywhere else. +rocm-cli's own records and the OS. The non-interactive plan output (piped +output, or the Dash approval card) shows `Port: automatic; endpoint shown +after launch` while the launch is in flight and prints the concrete endpoint +once the port is actually leased; the interactive deployment summary table +only ever shows the final resolved endpoint. Pass `--port` explicitly to pin +one; an explicit loopback port already reserved by another managed service is +rejected up front, naming that service and how to free it (`rocm services stop + --yes`). A port held by an untracked process is rejected too, but +without naming the holder. `--host` to anything other than the default +loopback address (`localhost`, `::1`, `0.0.0.0`, or any other address) +requires an explicit `--port`: automatic selection only probes `127.0.0.1` and +cannot promise a free port anywhere else. `--gpu-memory-utilization` (a fraction above 0.0 and up to 1.0 of each GPU's total VRAM) and `--tool-call-parser` (for example `hermes`, `llama3_json`, or diff --git a/apps/rocm/src/chat.rs b/apps/rocm/src/chat.rs index af5878b50..6afd574d9 100644 --- a/apps/rocm/src/chat.rs +++ b/apps/rocm/src/chat.rs @@ -2235,6 +2235,16 @@ fn validate_chat_launch_server_tool_call(call: &providers::ChatToolCall) -> Resu if !is_loopback_host(&host) { bail!("local assistant cannot request non-local host `{host}`"); } + // `rocm serve` proves a port is free only on the exact default host; every + // other spelling (including the loopback aliases `localhost`/`::1` this + // check still accepts) needs an explicit port or the CLI call refuses + // later with a less actionable error. See `is_canonical_loopback_host`. + if object.get("port").is_none() && !rocm_core::is_canonical_loopback_host(&host) { + bail!( + "local assistant must supply an explicit `port` for host `{host}`; automatic port \ + selection is only supported on the default host {DEFAULT_LOCAL_HOST}" + ); + } if object .get("device") .and_then(serde_json::Value::as_str) @@ -6214,6 +6224,48 @@ model recipes assert!(error.contains("non-local host")); } + #[test] + fn chat_tool_call_requires_an_explicit_port_for_non_canonical_loopback_hosts() { + let no_port = providers::ChatToolCall { + id: None, + name: "launch_server".to_owned(), + arguments: serde_json::json!({ + "model": "tiny.gguf", + "engine": "vllm", + "host": "localhost" + }), + }; + let error = validate_chat_tool_call(&no_port).unwrap_err().to_string(); + assert!( + error.contains("explicit `port`"), + "unexpected error: {error}" + ); + + let with_port = providers::ChatToolCall { + id: None, + name: "launch_server".to_owned(), + arguments: serde_json::json!({ + "model": "tiny.gguf", + "engine": "vllm", + "host": "localhost", + "port": 11999 + }), + }; + validate_chat_tool_call(&with_port).expect("explicit port on localhost is allowed"); + + let canonical_no_port = providers::ChatToolCall { + id: None, + name: "launch_server".to_owned(), + arguments: serde_json::json!({ + "model": "tiny.gguf", + "engine": "vllm", + "host": "127.0.0.1" + }), + }; + validate_chat_tool_call(&canonical_no_port) + .expect("automatic port selection still works on the canonical loopback host"); + } + #[test] fn chat_tool_call_rejects_bad_install_suggestions() { for (arguments, expected) in [ diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 739ad12f5..2934f9907 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -4636,7 +4636,9 @@ fn apply_service_prune_plan( /// guard back to its caller, and `start_managed_service`/`run_attached_service` /// only drop it once the record is persisted (see that function's own comment, /// which this one must not drift from). The gap was simply that prune never -/// acquired it: the helper above was the single acquirer in the tree. Taking it +/// acquired it: the helper above was the only acquirer in this binary (`rocmd`'s +/// recovery path takes the same file lock too, bounded, from its own process — +/// see `restart_managed_service`'s doc comment). Taking it /// here closes the window deterministically, for any number of records, which no /// age floor can — the window is not short and is not bounded, because /// `spawn_managed_engine_child`'s idempotency guard runs `load_managed_services` @@ -7202,6 +7204,13 @@ fn restart_internal_managed_service( /// [`restart_internal_managed_service`], with the termination of the recorded /// processes supplied by the caller — see [`stop_internal_managed_service_with`]. +/// +/// The respawn is serialized on the same cross-process launch lock a fresh +/// `rocm serve` launch and `rocmd` recovery both use: the confirmed stop just +/// released this service's (host, port), and nothing reserves it again until +/// the new child binds and its record is published, so a concurrent launch or +/// recovery could otherwise claim the exact port this restart is about to +/// reuse. fn restart_internal_managed_service_with( paths: &AppPaths, service_id: &str, @@ -7251,8 +7260,49 @@ fn restart_internal_managed_service_with( let (stopped_record, _) = read_managed_service_manifest(paths, service_id)?; // Still after the stop, like the device-policy refusal inside it always was: // the callers' "stopped by the attempt" report and pin restore both cover a - // restart that gets this far and fails. + // restart that gets this far and fails. Pure computation — it parses the + // record, it does not touch the network — so it stays ahead of the launch + // lock below and a refused device policy still fails before anything binds. let serve_args = restart_child_argv(&record)?; + // The confirmed stop just released this (host, port): nothing holds it again + // until the new child binds and `record_restarted_child` publishes below. + // Close that gap the same way a fresh launch and `rocmd` recovery do — + // under the shared launch lock, with a re-check against the disk and a + // loopback preflight — so a concurrent launch or recovery can't claim this + // exact port out from under the restart. Unbounded, like every other + // CLI-side acquire of this lock (see `FileLock::acquire_bounded`'s doc + // comment for why only `rocmd`'s recovery bounds its wait). + let launch_lock = rocm_core::FileLock::acquire(paths.managed_launch_lock_path())?; + if let Some(collision) = load_managed_services(paths) + .context("failed to re-read managed services under the launch lock")? + .into_iter() + .find(|other| { + other.service_id != stopped_record.service_id + && other.is_live() + && other.host == stopped_record.host + && other.port == stopped_record.port + }) + { + bail!( + "cannot restart `{service_id}` on {}:{}: {} claimed that port while the restart \ + was waiting for the launch lock", + stopped_record.host, + stopped_record.port, + collision.service_id + ); + } + let port_lease = if rocm_core::is_canonical_loopback_host(&stopped_record.host) { + Some( + rocm_core::lease_loopback_port(stopped_record.port).with_context(|| { + format!( + "cannot restart `{service_id}` on {}:{}: its port is in use", + stopped_record.host, stopped_record.port + ) + })?, + ) + } else { + None + }; fs::OpenOptions::new() .create(true) .append(true) @@ -7270,6 +7320,11 @@ fn restart_internal_managed_service_with( record.runtime_id.as_deref(), record.env_id.as_deref(), )?; + // Hand the port to the respawned child: close the probe socket immediately + // before the spawn, still holding the launch lock. + if let Some(lease) = port_lease { + lease.release(); + } // A restarted public service comes back authenticated: its 0600 key file // persists across restarts (removed only on stop), so hand the child the same // file and reuse the key for the readiness probe. @@ -7321,6 +7376,10 @@ fn restart_internal_managed_service_with( // Recorded before the readiness wait, so a stop issued during it finds the // new child rather than the processes already stopped. let started = record_restarted_child(paths, &stopped_record, runtime_pin, child_pid)?; + // Published; a concurrent launch or recovery now sees this service as live + // on its port again, so there is nothing left for the lock to protect, and + // the readiness wait below can block for many seconds. + drop(launch_lock); let readiness = wait_for_service_http_ready( &started.engine, &started.host, @@ -8178,10 +8237,7 @@ pub(crate) fn managed_service_sidebar_counts( } pub(crate) fn managed_service_is_live(record: &ManagedServiceRecord) -> bool { - matches!( - record.status.as_str(), - "ready" | "running" | "starting" | "recovering" - ) + record.is_live() } /// Idempotency guard for managed launches: returns an already-live managed @@ -12862,6 +12918,7 @@ pub(crate) mod tests { assert!(help.contains("11435"), "{help}"); assert!(help.contains("automatically"), "{help}"); assert!(help.contains("127.0.0.1"), "{help}"); + assert!(help.contains("explicit --port"), "{help}"); } #[test] @@ -17572,6 +17629,87 @@ pub(crate) mod tests { assert_eq!(on_disk.restart_count, 0, "nothing was written"); } + /// The gap a manual restart used to leave open: the confirmed stop just + /// released the service's port, and nothing reserves it again until the + /// new child binds and `record_restarted_child` publishes — a concurrent + /// `rocm serve` launch could claim that exact port in between. Recreate + /// the race under the shared launch lock and require the restart to + /// refuse rather than spawn a competitor onto it, following the same + /// wait-then-publish-then-release shape as + /// `services_prune_waits_for_the_managed_launch_lock_before_sweeping`. + #[test] + fn restart_refuses_a_port_a_launch_claimed_while_it_waited_for_the_lock() -> Result<()> { + use std::sync::mpsc; + + const WAIT_PROOF: Duration = Duration::from_millis(500); + + let (root, paths) = test_paths("restart-launch-claimed-port"); + paths.ensure()?; + let service_id = "svc-restart-launch-claimed-port"; + let record = stop_test_record(&paths, service_id); + let host = record.host.clone(); + let port = record.port; + record.write()?; + + let launch_lock = rocm_core::FileLock::acquire(paths.managed_launch_lock_path())?; + + let (started_tx, started_rx) = mpsc::channel(); + let (done_tx, done_rx) = mpsc::channel(); + let (waited, finished) = std::thread::scope(|scope| { + scope.spawn(|| { + started_tx.send(()).expect("signal about to restart"); + let outcome = + restart_internal_managed_service_with(&paths, service_id, None, |_| { + (Vec::new(), true) + }); + let _ = done_tx.send(outcome.map_err(|error| format!("{error:#}"))); + }); + started_rx + .recv_timeout(Duration::from_secs(10)) + .expect("restart thread started"); + let early = done_rx.recv_timeout(WAIT_PROOF); + let waited = early.is_err(); + // Simulate a fresh `rocm serve --gpu auto` publishing a brand-new + // service on this exact port while the restart was still waiting + // for the lock it itself now holds. + let mut competitor = ManagedServiceRecord::new( + &paths, + "svc-fresh-launch", + "lemonade", + "other-model", + "other-model-canonical", + &host, + port, + "managed", + 0, + None, + None, + None, + ); + competitor.status = "starting".to_owned(); + competitor.write().expect("publish the competing record"); + drop(launch_lock); + let finished = match early { + Ok(finished) => finished, + Err(_) => done_rx + .recv_timeout(Duration::from_secs(10)) + .expect("restart completes once the launch lock is released"), + }; + (waited, finished) + }); + let _ = fs::remove_dir_all(root); + + assert!( + waited, + "the restart must serialize on the shared launch lock" + ); + let error = + finished.expect_err("a live competitor on the same port must refuse the restart"); + assert!(error.contains("svc-fresh-launch"), "{error}"); + assert!(error.contains("claimed that port"), "{error}"); + Ok(()) + } + #[test] fn serve_notes_surface_the_ignored_engine_flag_in_summary_mode() { // The interactive summary is what a default `rocm serve` prints, so a flag diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index c1fb124ba..653ecb4c1 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -878,7 +878,18 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { launch_lock, require_api_key, &mut |_elapsed| spinner.tick(), - )?; + ) + // A pre-spawn failure (port reserved/in use, range exhausted, other bind + // error) returns before any service record exists, so nothing else clears + // the key just stored for this attempt. Idempotent with the post-spawn + // cleanup in `abandon_failed_managed_launch`. + .inspect_err(|_| { + drop_orphaned_endpoint_key_on_already_running( + &paths, + &service_id, + endpoint_auth.as_deref(), + ); + })?; ensure_background_helper_running_quiet(summary_mode)?; // An equivalent service was already running, so nothing was spawned and the @@ -949,6 +960,7 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { } run_attached_service( + &paths, &selected_engine, &service_id, &model, diff --git a/apps/rocm/src/serve_port.rs b/apps/rocm/src/serve_port.rs index 903c78685..a14e8915f 100644 --- a/apps/rocm/src/serve_port.rs +++ b/apps/rocm/src/serve_port.rs @@ -23,6 +23,7 @@ //! Everything downstream of this module — engine requests, service records, //! readiness, supervision, recovery — keeps its concrete `u16` contract. +use crate::service_hints; use anyhow::{Result, bail}; use rocm_core::{DEFAULT_LOCAL_HOST, DEFAULT_LOCAL_PORT, LoopbackPortLease}; use std::io; @@ -160,13 +161,19 @@ pub(crate) fn resolve_serve_port( match request { PortRequest::Explicit(port) => { if let Some(owner) = reserved.iter().find(|entry| entry.port == port) { + // The "stop it" advice below is proven, not just worded: see + // `stopping_the_reserving_record_frees_its_explicit_port_for_reuse` + // in `service_launch.rs`, which recreates the exact on-disk status + // transition `rocm services stop` performs and confirms the same + // explicit port then resolves. A full CLI-level E2E for this needs + // a real GPU, since reaching this branch requires a completed + // model/engine resolution first. bail!( "{DEFAULT_LOCAL_HOST}:{port} is already reserved by managed service {} \ - ({}). Stop it with `rocm services stop {} --yes`, or pass a different \ - `--port`.", + ({}). Stop it with `{}`, or pass a different `--port`.", owner.service_id, owner.status, - owner.service_id + service_hints::service_stop_hint(&owner.service_id) ); } probe(port).map_err(|error| explicit_bind_error(port, &error)) @@ -189,8 +196,8 @@ pub(crate) fn resolve_serve_port( bail!( "no free TCP port for {DEFAULT_LOCAL_HOST} in {AUTO_PORT_FIRST}–{AUTO_PORT_LAST}; \ every candidate is reserved by a live managed service or already in use. Stop an \ - unused server with `rocm services stop --yes`, or pass an explicit \ - `--port`." + unused server with `{}`, or pass an explicit `--port`.", + service_hints::service_action_hint("stop", "") ); } } diff --git a/apps/rocm/src/service_launch.rs b/apps/rocm/src/service_launch.rs index 9b253ac29..6d816a6ff 100644 --- a/apps/rocm/src/service_launch.rs +++ b/apps/rocm/src/service_launch.rs @@ -1082,6 +1082,7 @@ enum AttachOutcome { /// survives the session — Ctrl-D detaches and leaves it running, Ctrl-C stops it. #[allow(clippy::too_many_arguments)] pub(crate) fn run_attached_service( + paths: &AppPaths, engine: &str, service_id: &str, requested_model: &str, @@ -1095,10 +1096,8 @@ pub(crate) fn run_attached_service( launch_lock: rocm_core::FileLock, require_api_key: bool, ) -> Result<()> { - let paths = AppPaths::discover()?; - let spawn = spawn_managed_engine_child( - &paths, + paths, engine, service_id, requested_model, @@ -1111,7 +1110,14 @@ pub(crate) fn run_attached_service( env_id, resolve.engine_recipe.as_ref(), require_api_key, - )?; + ) + // Mirrors the background path's cleanup in `serve_cmd.rs`: a pre-spawn + // failure (port reserved/in use, range exhausted, other bind error) returns + // before any service record exists, so nothing else clears the key just + // stored for this attempt. + .inspect_err(|_| { + drop_orphaned_endpoint_key_on_already_running(paths, service_id, endpoint_api_key); + })?; // The claiming record is persisted (or an existing service was found), so the // selected GPU is now visible to concurrent auto-selection. Release the launch // lock before streaming logs, which blocks for the whole attached session. @@ -1133,7 +1139,7 @@ pub(crate) fn run_attached_service( " stop: {}", service_hints::service_stop_hint(&report.service_id) ); - drop_orphaned_endpoint_key_on_already_running(&paths, service_id, endpoint_api_key); + drop_orphaned_endpoint_key_on_already_running(paths, service_id, endpoint_api_key); return Ok(()); } ManagedSpawn::Spawned { record, child_pid } => ( @@ -1179,7 +1185,7 @@ pub(crate) fn run_attached_service( println!(" logs: rocm logs {service_id}"); println!(" stop: {}", service_hints::service_stop_hint(&service_id)); record_cli_audit_event( - &paths, + paths, "service", "serve_detach", "info", @@ -1191,7 +1197,7 @@ pub(crate) fn run_attached_service( AttachOutcome::Stop => { println!("stopping server..."); match run_internal_sandbox_tool( - &paths, + paths, SandboxToolArg::StopServer, Some(service_id.clone()), true, @@ -1205,7 +1211,7 @@ pub(crate) fn run_attached_service( } } record_cli_audit_event( - &paths, + paths, "service", "serve_stop", "info", @@ -1374,6 +1380,7 @@ fn stream_attached_logs_no_tty(log_path: &Path, child_pid: u32) -> Result Result<()> { + // Drives the real call site's *post*-spawn cleanup branch, which + // neither `abandoned_launch_*` test reaches (they call + // `abandon_failed_managed_launch` directly). An engine name this + // binary does not build in makes the real spawned child (a second + // `rocm` process) exit almost immediately via + // `run_builtin_engine_serve_http`'s `other => bail!(...)` arm, after + // `launch_managed_engine_child_locked` has already set + // `record.engine_pid` — so cleanup must terminate (already-dead) the + // child, keep the record and log for inspection, mark the record + // `failed`, and clear only the endpoint key. + let (root, paths) = test_paths("managed-spawn-unknown-engine-cleanup"); + paths.ensure()?; + let service_id = "no-such-engine-qwen-4000"; + endpoint_keys::store_endpoint_api_key(&paths, service_id, "test-key")?; + + let resolve = ResolveModelResponse { + canonical_model_id: "qwen-canonical".to_owned(), + task: "chat".to_owned(), + source: "hf".to_owned(), + revision: "main".to_owned(), + loader: "llama.cpp".to_owned(), + trust_remote_code: false, + chat_template_mode: "auto".to_owned(), + dtype: "auto".to_owned(), + device_policy: DevicePolicy::GpuPreferred, + estimated_memory: "unknown".to_owned(), + launch_defaults: serde_json::json!({}), + engine_recipe: None, + warnings: Vec::new(), + }; + + let result = spawn_managed_engine_child( + &paths, + "no-such-engine", + service_id, + "qwen", + &resolve, + "127.0.0.1", + serve_port::PortRequest::Explicit(11_514), + &resolve.device_policy, + &[], + None, + None, + None, + false, + ); + + let manifest_path = paths.service_manifest_path(service_id); + let log_path = paths.service_log_path(service_id); + let persisted = load_managed_services(&paths) + .expect("re-read manifests") + .into_iter() + .find(|candidate| candidate.service_id == service_id); + let key = endpoint_keys::endpoint_api_key(&paths, service_id); + let live = live_managed_services(&paths); + + let Err(error) = result else { + panic!("an engine this binary does not build in must not report success") + }; + let message = error.to_string(); + assert!( + message.contains("managed engine exited immediately"), + "the failure must come from the spawned child dying, not an earlier guard: {message}" + ); + assert!( + manifest_path.exists(), + "a post-spawn failure must keep the record for inspection" + ); + assert!( + log_path.exists(), + "a post-spawn failure must keep the log for inspection" + ); + let persisted = + persisted.expect("a post-spawn failure must leave the record on disk, not delete it"); + assert_eq!( + persisted.status, "failed", + "a post-spawn failure must mark the record failed, not leave it stuck" + ); + assert_eq!( + key, None, + "a post-spawn failure must still clear the endpoint key" + ); + assert!( + live.is_empty(), + "a failed record must not still reserve its port" + ); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn failed_port_reservation_leaves_no_orphaned_endpoint_key() -> Result<()> { + // serve() stores the new attempt's endpoint key before calling + // start_managed_service. If spawn_managed_engine_child fails inside + // resolve_managed_port_in_transaction (explicit port already reserved + // by another live service), it returns before any ManagedServiceRecord + // exists for this attempt, so abandon_failed_managed_launch never runs. + // The caller's .inspect_err() cleanup (drop_orphaned_endpoint_key_on_already_running) + // must be what clears the key. + let (root, paths) = test_paths("orphaned-key-on-reserved-port"); + paths.ensure()?; + let mut other = ManagedServiceRecord::new( + &paths, + "lemonade-llama-1000", + "lemonade", + "llama", + "llama-canonical", + "127.0.0.1", + 11513, + "managed", + std::process::id(), + None, + None, + None, + ); + other.status = "ready".to_owned(); + other.engine_pid = Some(std::process::id()); + other.write()?; + + let resolve = ResolveModelResponse { + canonical_model_id: "qwen-canonical".to_owned(), + task: "chat".to_owned(), + source: "hf".to_owned(), + revision: "main".to_owned(), + loader: "llama.cpp".to_owned(), + trust_remote_code: false, + chat_template_mode: "auto".to_owned(), + dtype: "auto".to_owned(), + device_policy: DevicePolicy::GpuPreferred, + estimated_memory: "unknown".to_owned(), + launch_defaults: serde_json::json!({}), + engine_recipe: None, + warnings: Vec::new(), + }; + + let service_id = "lemonade-qwen-2000"; + endpoint_keys::store_endpoint_api_key(&paths, service_id, "test-key")?; + + let result = spawn_managed_engine_child( + &paths, + "lemonade", + service_id, + "qwen", + &resolve, + "127.0.0.1", + serve_port::PortRequest::Explicit(11513), + &resolve.device_policy, + &[], + None, + None, + None, + false, + ); + let Err(error) = result else { + let _ = fs::remove_dir_all(root); + panic!("explicit port already reserved by a live service must be rejected") + }; + assert!( + error + .to_string() + .contains("already reserved by managed service"), + "unexpected error: {error}" + ); + + drop_orphaned_endpoint_key_on_already_running(&paths, service_id, Some("test-key")); + assert!( + endpoint_keys::endpoint_api_key(&paths, service_id).is_none(), + "cleanup must drop the key this failed attempt staged before launching" + ); + + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn attached_launch_failing_before_spawn_leaves_no_orphaned_endpoint_key() -> Result<()> { + // The background path's cleanup lives in serve_cmd.rs, wrapping + // start_managed_service. run_attached_service is the other caller of + // spawn_managed_engine_child and needs the identical .inspect_err() + // cleanup, not a simulated one: drive the real function end to end. + let (root, paths) = test_paths("orphaned-key-attached-reserved-port"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + let mut other = ManagedServiceRecord::new( + &paths, + "lemonade-llama-1000", + "lemonade", + "llama", + "llama-canonical", + "127.0.0.1", + 11514, + "managed", + std::process::id(), + None, + None, + None, + ); + other.status = "ready".to_owned(); + other.engine_pid = Some(std::process::id()); + other.write()?; + + let resolve = ResolveModelResponse { + canonical_model_id: "qwen-canonical".to_owned(), + task: "chat".to_owned(), + source: "hf".to_owned(), + revision: "main".to_owned(), + loader: "llama.cpp".to_owned(), + trust_remote_code: false, + chat_template_mode: "auto".to_owned(), + dtype: "auto".to_owned(), + device_policy: DevicePolicy::GpuPreferred, + estimated_memory: "unknown".to_owned(), + launch_defaults: serde_json::json!({}), + engine_recipe: None, + warnings: Vec::new(), + }; + + let service_id = "lemonade-qwen-attached-2000"; + endpoint_keys::store_endpoint_api_key(&paths, service_id, "test-key")?; + let launch_lock = rocm_core::FileLock::acquire(paths.managed_launch_lock_path())?; + + let result = run_attached_service( + &paths, + "lemonade", + service_id, + "qwen", + &resolve, + "127.0.0.1", + serve_port::PortRequest::Explicit(11514), + &[], + None, + None, + Some("test-key"), + launch_lock, + false, + ); + let Err(error) = result else { + let _ = fs::remove_dir_all(root); + panic!("explicit port already reserved by a live service must be rejected") + }; + assert!( + error + .to_string() + .contains("already reserved by managed service"), + "unexpected error: {error}" + ); + assert!( + endpoint_keys::endpoint_api_key(&paths, service_id).is_none(), + "a pre-spawn failure must drop the key this attempt staged before launching" + ); + + let _ = fs::remove_dir_all(root); Ok(()) } @@ -2241,6 +2538,72 @@ mod tests { Ok(()) } + #[test] + fn stopping_the_reserving_record_frees_its_explicit_port_for_reuse() -> Result<()> { + // Proves the mechanism `rocm services stop` relies on: it never touches + // the port directly, it only flips the stopped record's `status` (see + // the `all_stopped` branch in `main.rs`'s stop handler). This recreates + // that exact disk state and checks the same explicit port a live + // competitor refused now resolves, closing the loop the E2E + // `assert_port_reserved_message` step only checks by wording. + let (root, paths) = test_paths("alloc-explicit-port-freed-by-stop"); + paths.ensure()?; + let mut holder = ManagedServiceRecord::new( + &paths, + "lemonade-a-1", + "lemonade", + "model-a", + "model-a".to_owned(), + DEFAULT_LOCAL_HOST, + 11_435, + "managed", + std::process::id(), + None, + None, + None, + ); + holder.status = "running".to_owned(); + holder.engine_pid = Some(std::process::id()); + holder.write()?; + + let still_reserved = resolve_managed_port_in_transaction( + &paths, + "lemonade", + "model-b", + DEFAULT_LOCAL_HOST, + serve_port::PortRequest::Explicit(11_435), + &mut always_free, + ); + assert!( + matches!(still_reserved, Err(ref error) if error.to_string().contains("already reserved by managed service lemonade-a-1")), + "unexpected result while still running: {still_reserved:?}" + ); + + holder.status = "stopped".to_owned(); + holder.write()?; + + let decision = resolve_managed_port_in_transaction( + &paths, + "lemonade", + "model-b", + DEFAULT_LOCAL_HOST, + serve_port::PortRequest::Explicit(11_435), + &mut always_free, + ); + let _ = fs::remove_dir_all(root); + + match decision? { + ManagedPortDecision::Lease(lease) => assert_eq!(lease.port(), 11_435), + ManagedPortDecision::AlreadyLive(existing) => { + panic!( + "a stopped record must not block reuse: {}", + existing.service_id + ) + } + } + Ok(()) + } + #[test] fn allocation_transaction_rejects_an_explicit_port_a_recovering_service_holds() -> Result<()> { let (root, paths) = test_paths("alloc-txn-rejects-explicit-collision"); @@ -2375,6 +2738,99 @@ mod tests { Ok(()) } + #[test] + fn select_gpu_indices_under_launch_lock_blocks_for_the_full_hold_and_then_sees_the_holders_write() + -> Result<()> { + // The standing gap this closes: nothing exercised the real lock-acquisition + // function production callers use (`select_gpu_indices_under_launch_lock`) + // against a lock another thread actually holds, so a refactor that dropped + // the lock early (or stopped threading it through) would not fail any test. + // A real end-to-end spawn test is avoided here per the fix-forward plan: + // `spawn_managed_engine_child` is still exercised for real, but only via its + // `AlreadyLive` short-circuit (a pre-seeded duplicate record), so no child + // process is ever launched. + use std::sync::{Arc, Barrier}; + use std::time::Instant; + + let (root, paths) = test_paths("launch-lock-holds-across-spawn"); + paths.ensure()?; + write_live_service_record(&paths, "lemonade-held-1", "held-canonical", 11_520, "ready")?; + + let barrier = Arc::new(Barrier::new(2)); + let hold_for = Duration::from_millis(250); + + let holder_paths = paths.clone(); + let holder_barrier = Arc::clone(&barrier); + let holder = thread::spawn(move || -> Result<()> { + let _lock = rocm_core::FileLock::acquire(holder_paths.managed_launch_lock_path())?; + holder_barrier.wait(); + thread::sleep(hold_for); + Ok(()) + }); + + barrier.wait(); + let started = Instant::now(); + let (gpu_indices, launch_lock) = + crate::select_gpu_indices_under_launch_lock(&paths, true, None, || None, None, None)?; + let waited = started.elapsed(); + + let resolve = ResolveModelResponse { + canonical_model_id: "held-canonical".to_owned(), + task: "chat".to_owned(), + source: "hf".to_owned(), + revision: "main".to_owned(), + loader: "llama.cpp".to_owned(), + trust_remote_code: false, + chat_template_mode: "auto".to_owned(), + dtype: "auto".to_owned(), + device_policy: DevicePolicy::GpuPreferred, + estimated_memory: "unknown".to_owned(), + launch_defaults: serde_json::json!({}), + engine_recipe: None, + warnings: Vec::new(), + }; + let spawn = spawn_managed_engine_child( + &paths, + "lemonade", + "lemonade-held-2", + "held-model", + &resolve, + DEFAULT_LOCAL_HOST, + serve_port::PortRequest::Auto, + &resolve.device_policy, + &gpu_indices, + None, + None, + None, + false, + ); + drop(launch_lock); + holder.join().expect("holder thread")?; + let _ = fs::remove_dir_all(root); + + // `thread::sleep` only guarantees *at least* `hold_for`, but under a + // loaded host (e.g. a full parallel `cargo test` run on WSL2) timer + // coalescing can make the measured wait a hair short of the nominal + // hold. Allow a small margin so the test still fails loudly if the + // call returns early, without being flaky on actual block duration. + let margin = Duration::from_millis(20); + assert!( + waited + margin >= hold_for, + "select_gpu_indices_under_launch_lock returned before the holder released the lock: waited {waited:?}, held {hold_for:?}" + ); + match spawn? { + ManagedSpawn::AlreadyRunning(report) => { + assert_eq!(report.service_id, "lemonade-held-1"); + } + ManagedSpawn::Spawned { .. } => { + panic!( + "expected the pre-seeded AlreadyLive record to short-circuit, not a real spawn" + ) + } + } + Ok(()) + } + #[test] fn abandoned_launch_leaves_no_record_log_or_endpoint_key() -> Result<()> { let (root, paths) = test_paths("abandon-launch-cleanup"); diff --git a/apps/rocmd/src/mcp.rs b/apps/rocmd/src/mcp.rs index 8b086a52e..bd6c480a2 100644 --- a/apps/rocmd/src/mcp.rs +++ b/apps/rocmd/src/mcp.rs @@ -445,7 +445,8 @@ pub(crate) fn rocm_mcp_tools() -> Vec { "port": { "type": "integer", "minimum": 1, - "maximum": 65535 + "maximum": 65535, + "description": "Required unless host is exactly 127.0.0.1 (localhost/::1 still need one)." }, "allow_public_bind": { "type": "boolean" @@ -1093,6 +1094,16 @@ fn build_launch_server_args(arguments: &serde_json::Map) -> Resul "launch_server host `{host}` is not loopback; require `allow_public_bind=true` before binding a non-local interface" ); } + // `rocm serve` proves a port is free only on the exact default host; every + // other spelling (including the loopback aliases `localhost`/`::1` the + // check above still accepts) needs an explicit port or the CLI call + // refuses later with a less actionable error. See `is_canonical_loopback_host`. + if arguments.get("port").is_none() && !rocm_core::is_canonical_loopback_host(host) { + bail!( + "launch_server host `{host}` needs an explicit `port`; automatic port selection is \ + only supported on the default host {DEFAULT_LOCAL_HOST}" + ); + } let mut argv = vec!["serve".to_owned(), model.to_owned(), "--managed".to_owned()]; if let Some(engine) = arguments.get("engine").and_then(Value::as_str) { @@ -1421,12 +1432,35 @@ mod tests { ("model".to_owned(), Value::String("tiny-gpt2".to_owned())), ("host".to_owned(), Value::String("0.0.0.0".to_owned())), ("allow_public_bind".to_owned(), Value::Bool(true)), + ("port".to_owned(), Value::Number(11999.into())), ]); let args = build_launch_server_args(&arguments)?; assert!(args.contains(&"--allow-public-bind".to_owned())); Ok(()) } + #[test] + fn launch_server_requires_an_explicit_port_for_non_canonical_loopback_hosts() { + let arguments = serde_json::Map::from_iter([ + ("model".to_owned(), Value::String("tiny-gpt2".to_owned())), + ("host".to_owned(), Value::String("localhost".to_owned())), + ]); + let error = build_launch_server_args(&arguments).unwrap_err(); + assert!(error.to_string().contains("explicit `port`"), "{error:#}"); + } + + #[test] + fn launch_server_allows_localhost_with_an_explicit_port() -> Result<()> { + let arguments = serde_json::Map::from_iter([ + ("model".to_owned(), Value::String("tiny-gpt2".to_owned())), + ("host".to_owned(), Value::String("localhost".to_owned())), + ("port".to_owned(), Value::Number(11999.into())), + ]); + let args = build_launch_server_args(&arguments)?; + assert!(args.contains(&"11999".to_owned())); + Ok(()) + } + #[test] fn install_sdk_rejects_system_prefix_without_ack() { let arguments = serde_json::Map::from_iter([( diff --git a/apps/rocmd/src/watchers.rs b/apps/rocmd/src/watchers.rs index b674682d6..80215114c 100644 --- a/apps/rocmd/src/watchers.rs +++ b/apps/rocmd/src/watchers.rs @@ -1284,6 +1284,15 @@ fn handle_server_recover_event_with_record( // than letting one service take the whole 30s watcher tick (and the // daemon) down. if let Err(error) = restart_managed_service(paths, &mut *record) { + // `restart_managed_service` can fail before it ever writes the + // record (e.g. the recorded port is still occupied), so without + // this stamp the backoff guard above never sees a prior attempt + // and the next 30s tick retries immediately, forever. Best + // effort: a write failure here just means the next tick retries + // sooner, not a worse outcome than today. + let _ = crate::service::update_supervised_record(paths, record, |current| { + current.last_restart_unix_ms = Some(now); + }); return crate::persistence::record_event( paths, state, @@ -1471,6 +1480,31 @@ pub(crate) fn restart_managed_service( ) -> Result<()> { let _allocation = rocm_core::lock_service_allocation(paths)?; + // Re-check against the disk, now that the lock excludes a concurrent + // `rocm serve` publish: the caller's `record` was read before this lock was + // acquired, so a fresh launch could have claimed this exact (host, port) + // in the meantime (the engine itself binds the port only after the model + // loads, so a raw OS bind probe alone would not catch that — see + // `crate::persistence::load_managed_services`'s callers for the shared + // `is_live` reservation this mirrors). Refuse rather than spawn a + // competitor; the next watcher tick reconsiders this service. + let on_disk = crate::persistence::load_managed_services(paths) + .context("failed to re-read managed services under the launch lock")?; + if let Some(collision) = on_disk + .iter() + .find(|other| other.service_id != record.service_id && other.is_live()) + .filter(|other| other.host == record.host && other.port == record.port) + { + anyhow::bail!( + "cannot recover managed service {} on {}:{}: {} claimed that port while recovery \ + was waiting for the launch lock", + record.service_id, + record.host, + record.port, + collision.service_id + ); + } + // On the canonical loopback host, prove the recorded port is genuinely free // before publishing `recovering` or spawning anything: a supervisor that // cannot bind would replace a truthful record with a lie. Custom hosts keep @@ -3284,6 +3318,240 @@ mod tests { ); } + #[test] + fn recovery_refuses_a_port_a_launch_claimed_while_it_waited_for_the_lock() -> Result<()> { + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + + // The gap Finding #2 closes: a crashed service's port is simultaneously + // "free" to a brand-new `rocm serve --gpu auto` launch (which only + // checks *live* records) and "needs recovery" to this function (which + // used to only probe the raw OS bind). Neither process's engine binds + // the port until its model finishes loading, so a probe alone cannot + // see a `starting` competitor that has not bound yet. Recreate that + // exact disk state under the lock and require recovery to refuse, + // rather than racing a second engine onto the same port. + let (_root, paths) = temp_app_paths("recovery-launch-claimed-port"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let port = 21_777_u16; + let mut record = ManagedServiceRecord::new( + &paths, + "lemonade-recover-3", + "lemonade", + "qwen", + "qwen-canonical".to_owned(), + rocm_core::DEFAULT_LOCAL_HOST, + port, + "managed", + 0, + None, + None, + None, + ); + record.status = "failed".to_owned(); + record.write()?; + + let guard = rocm_core::lock_service_allocation(&paths)?; + let finished = Arc::new(AtomicBool::new(false)); + let handle = { + let paths = paths.clone(); + let finished = Arc::clone(&finished); + thread::spawn(move || { + let result = restart_managed_service(&paths, &mut record); + finished.store(true, Ordering::SeqCst); + result + }) + }; + + thread::sleep(Duration::from_millis(250)); + assert!( + !finished.load(Ordering::SeqCst), + "recovery must serialize on the shared allocation lock" + ); + + // Simulate a `rocm serve --gpu auto` publishing a brand-new service on + // this exact port while recovery was still waiting for the lock it + // itself now holds — the one window the launch lock is supposed to + // close. + let mut competitor = ManagedServiceRecord::new( + &paths, + "lemonade-fresh-launch", + "lemonade", + "other-model", + "other-model-canonical".to_owned(), + rocm_core::DEFAULT_LOCAL_HOST, + port, + "managed", + 0, + None, + None, + None, + ); + competitor.status = "starting".to_owned(); + competitor.write()?; + drop(guard); + + let error = handle + .join() + .expect("recovery thread") + .expect_err("a live competitor on the same port must refuse recovery"); + let message = format!("{error:#}"); + assert!(message.contains("lemonade-fresh-launch"), "{message}"); + assert!(message.contains("claimed that port"), "{message}"); + + let on_disk = crate::persistence::load_managed_services(&paths)? + .into_iter() + .find(|candidate| candidate.service_id == "lemonade-recover-3") + .expect("record still present"); + assert_eq!( + on_disk.status, "failed", + "refused recovery must not overwrite the truthful record with `recovering`" + ); + Ok(()) + } + + #[test] + fn contained_recovery_survives_a_failed_restart_and_backs_off() -> Result<()> { + use std::net::TcpListener; + + let (root, paths) = temp_app_paths("recovery-contained-port-collision"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let occupant = TcpListener::bind(("127.0.0.1", 0))?; + let port = occupant.local_addr()?.port(); + + let mut record = ManagedServiceRecord::new( + &paths, + "lemonade-recover-2", + "lemonade", + "qwen", + "qwen-canonical".to_owned(), + rocm_core::DEFAULT_LOCAL_HOST, + port, + "managed", + 0, + None, + None, + None, + ); + record.status = "failed".to_owned(); + record.write()?; + + let mut state = test_runtime_state(vec![test_watcher_snapshot( + "server-recover", + WatcherMode::Contained, + None, + )]); + let event = AutomationTriggerEvent { + at_unix_ms: 1, + kind: "service.manifest_recoverable".to_owned(), + source: "manifest_scan".to_owned(), + watcher_hint: Some("server-recover".to_owned()), + service_id: Some(record.service_id.clone()), + reason: Some("manifest_status_failed".to_owned()), + payload: json!({}), + }; + + // The watcher must not propagate a per-service failure: that would take + // the whole 30s tick (and every other watcher) down with it. + handle_server_recover_event_with_record( + &paths, + WatcherMode::Contained, + &mut state, + &event, + &mut record, + ) + .expect("a failed restart must be caught and reported, not propagated"); + + let event_text = fs::read_to_string(paths.automation_events_path())?; + let logged = serde_json::from_str::(event_text.trim())?; + let reloaded = load_service_record(&paths, "lemonade-recover-2")?; + fs::remove_dir_all(root).ok(); + drop(occupant); + + assert_eq!(logged.level, "error"); + assert_eq!(logged.action, "restart_managed_service_failed"); + assert!( + logged.message.contains("recorded port is in use"), + "{}", + logged.message + ); + // Unchanged other than the backoff stamp: the occupied port must not + // have been reported as recovered. + assert_eq!(reloaded.status, "failed"); + // Stamped despite `restart_managed_service` never writing the record on + // this failure path, so the next 30s tick's backoff guard sees a prior + // attempt instead of retrying immediately forever. + assert!(reloaded.last_restart_unix_ms.is_some()); + Ok(()) + } + + #[test] + fn recovery_refuses_when_the_registry_cannot_be_re_read_leaving_the_prior_record_intact() + -> Result<()> { + use std::net::TcpListener; + + let (_root, paths) = temp_app_paths("recovery-publication-failure"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + // A free recorded port, so a lease was never the obstacle under test. + let probe = TcpListener::bind(("127.0.0.1", 0))?; + let port = probe.local_addr()?.port(); + drop(probe); + + let mut record = ManagedServiceRecord::new( + &paths, + "lemonade-recover-2", + "lemonade", + "qwen", + "qwen-canonical".to_owned(), + rocm_core::DEFAULT_LOCAL_HOST, + port, + "managed", + 4242, + None, + None, + None, + ); + record.status = "failed".to_owned(); + record.write()?; + + // Replace the manifest with a directory. `load_managed_services` scans + // every `.json` entry under `services_dir()` to re-check for a + // collision before publishing `recovering`, and this record's own + // manifest is one of those entries — so the corruption trips that + // re-read, not the publish a few lines later. That is the more + // conservative of the two and still proves the invariant this test is + // for: recovery must fail closed, not spawn a competitor, when it + // cannot trust what else is live. + fs::remove_file(&record.manifest_path)?; + fs::create_dir_all(&record.manifest_path)?; + + let mut attempt = record.clone(); + let error = restart_managed_service(&paths, &mut attempt) + .expect_err("an unreadable registry must refuse, not assume no collision"); + let message = format!("{error:#}"); + assert!( + message.contains("failed to re-read managed services under the launch lock"), + "{message}" + ); + // Nothing was spawned and the on-disk row was never replaced, so the + // error must NOT claim a failed rollback: that phrasing is reserved for + // the post-spawn case where the record really is left mid-transition. + assert!(!message.contains("rollback also failed"), "{message}"); + + // The caller's record must not claim a recovery that never happened. + assert_eq!(attempt.status, "failed"); + assert_eq!(attempt.restart_count, record.restart_count); + assert_eq!(attempt.supervisor_pid, record.supervisor_pid); + fs::remove_dir_all(&record.manifest_path)?; + Ok(()) + } + /// The same guarantee for the daemon's recovery path, which records a PID of /// its own. See `supervise_service_persists_the_supervisor_identity_token` /// for why an unrecorded token is the defect rather than a cosmetic gap. @@ -3522,44 +3790,6 @@ mod tests { Ok(()) } - /// A write failure before anything is spawned must leave the prior record - /// intact: there is nothing to roll back but the in-memory transition. - #[test] - fn recovery_publication_failure_leaves_the_prior_record_intact() -> Result<()> { - use std::net::TcpListener; - - let (root, paths) = temp_app_paths("recovery-publication-failure"); - paths.ensure()?; - fs::create_dir_all(paths.services_dir())?; - - // A free recorded port, so the lease succeeds and the failure under test - // is the record publication itself. - let probe = TcpListener::bind(("127.0.0.1", 0))?; - let port = probe.local_addr()?.port(); - drop(probe); - - let mut record = identity_probe_record(&paths, "svc-recovery-publication-failure", port); - record.status = "failed".to_owned(); - record.write()?; - - // Make the manifest unwritable by replacing it with a directory. The - // `recovering` transition can no longer be published. - let manifest_path = paths.service_manifest_path(&record.service_id); - fs::remove_file(&manifest_path)?; - fs::create_dir_all(&manifest_path)?; - - let error = restart_managed_service(&paths, &mut record) - .expect_err("an unpublishable record must fail recovery"); - let message = format!("{error:#}"); - assert!(message.contains("failed to write service manifest"), "{message}"); - - // The caller's record must not claim a recovery that never happened. - assert_eq!(record.status, "recovering"); - fs::remove_dir_all(&manifest_path).ok(); - fs::remove_dir_all(root).ok(); - Ok(()) - } - #[test] #[ignore = "spawned as a child process by terminate_recovery_supervisor_kills_and_reaps_the_child"] fn recovery_cleanup_sleeper_child() { diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 6ee996355..eae5b1855 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -6,7 +6,6 @@ use anyhow::{Context, Result, bail}; use serde::{Deserialize, Serialize}; use std::ffi::OsString; use std::fs; -use std::net::TcpListener; use std::path::{Path, PathBuf}; use std::thread; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; @@ -143,7 +142,9 @@ pub use proc_lifecycle::{ process_start_ticks, terminate_verified, }; pub use process::{ - FileLock, detach_command_session, process_is_running, terminate_process, terminate_process_tree, + FileLock, LoopbackPortLease, MANAGED_LAUNCH_LOCK_TIMEOUT, detach_command_session, + is_canonical_loopback_host, lease_loopback_port, process_is_running, terminate_process, + terminate_process_tree, }; #[cfg(windows)] pub use process::{ @@ -522,86 +523,6 @@ pub(crate) fn write_atomic(path: &Path, bytes: &[u8], tmp_ext: &str) -> Result<( Ok(()) } -// --------------------------------------------------------------------------- -// Managed-service port leasing and the shared allocation lock. -// -// Both `rocm serve` and `rocmd` recovery contend for the same local TCP ports -// and the same service records, so the primitives that arbitrate them live here -// rather than in either binary. -// --------------------------------------------------------------------------- - -/// True only for the exact canonical default loopback host. -/// -/// Proactive OS collision detection (leasing a real socket before launch) is -/// claimed only for this one host. `localhost`, `::1`, `0.0.0.0`, and other -/// IPv4 addresses are "custom hosts": they require an explicit port and keep the -/// existing engine-owned bind/error behavior, because an IPv4-loopback probe -/// would not prove anything about the address the engine will actually bind. -#[must_use] -pub fn is_canonical_loopback_host(host: &str) -> bool { - host.trim() == DEFAULT_LOCAL_HOST -} - -/// A real, held `127.0.0.1:` listener proving the port is bindable now. -/// -/// The lease is the collision check: it is acquired before any manifest, log, or -/// child process exists, and [`LoopbackPortLease::release`] drops the socket -/// immediately before the engine child is spawned so the child can bind it. The -/// window between release and the child's bind is unavoidable without engine -/// socket-passing; it is far narrower than the previous "bind and hope" launch. -#[derive(Debug)] -pub struct LoopbackPortLease { - port: u16, - listener: Option, -} - -impl LoopbackPortLease { - #[must_use] - pub const fn port(&self) -> u16 { - self.port - } - - /// True while this lease still holds the socket open. - #[must_use] - pub const fn is_held(&self) -> bool { - self.listener.is_some() - } - - /// Close the socket and hand the port to the caller. Call this immediately - /// before spawning the process that will bind it. - pub fn release(mut self) -> u16 { - self.listener = None; - self.port - } - - /// A lease over a port this process never probed, for custom hosts where the - /// engine — not the CLI — owns bind success or failure. - #[must_use] - pub const fn engine_owned(port: u16) -> Self { - Self { - port, - listener: None, - } - } -} - -/// Try to take an exclusive IPv4-loopback lease on `port`. -/// -/// Errors are returned verbatim so callers can distinguish "occupied" -/// ([`std::io::ErrorKind::AddrInUse`], the only condition that should advance an -/// automatic scan) from a privileged or unavailable address, which must fail -/// immediately rather than silently choosing a different port. -pub fn lease_loopback_port(port: u16) -> std::io::Result { - let listener = TcpListener::bind(std::net::SocketAddrV4::new( - std::net::Ipv4Addr::LOCALHOST, - port, - ))?; - Ok(LoopbackPortLease { - port, - listener: Some(listener), - }) -} - /// File name of the shared allocation lock, inside [`AppPaths::services_dir`]. /// /// Not a `.json` file, so `load_managed_services` never mistakes it for a @@ -711,6 +632,7 @@ mod tests { use crate::test_support::workspace_test_artifact_dir; use std::fs; + use std::net::TcpListener; use std::path::Path; use std::process::{Command, Stdio}; diff --git a/crates/rocm-core/src/process.rs b/crates/rocm-core/src/process.rs index 0c715a5fd..809016c02 100644 --- a/crates/rocm-core/src/process.rs +++ b/crates/rocm-core/src/process.rs @@ -24,11 +24,13 @@ //! [#623](https://github.com/ROCm/rocm-cli/issues/623). use anyhow::{Context, Result, bail}; +use crate::DEFAULT_LOCAL_HOST; #[cfg(windows)] use std::collections::BTreeMap; #[cfg(windows)] use std::ffi::{OsStr, OsString}; use std::fs; +use std::net::TcpListener; #[cfg(windows)] use std::os::windows::ffi::OsStrExt; use std::path::{Path, PathBuf}; @@ -428,6 +430,156 @@ impl FileLock { pub fn path(&self) -> &Path { &self.path } + + /// Like [`Self::acquire`], but fails with an actionable error instead of + /// blocking forever once `timeout` has elapsed. + /// + /// Used where an indefinite wait would be a worse failure mode than a + /// clear timeout message — e.g. `rocmd`'s automatic recovery, which must + /// not hang forever behind a `rocm serve` launch holding the same lock. + /// A `rocm serve` launch itself uses the unbounded [`Self::acquire`]: its + /// own worst-case hold time (the already-running probe inside the launch + /// transaction) can exceed [`MANAGED_LAUNCH_LOCK_TIMEOUT`], so bounding it + /// would trade a real, slow-but-working launch for a spurious failure. + /// + /// # Errors + /// Fails when the lock file cannot be created/locked, or when another + /// holder keeps it for the whole timeout. + pub fn acquire_bounded(path: impl Into, timeout: Duration) -> Result { + let path = path.into(); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent) + .with_context(|| format!("failed to create lock directory {}", parent.display()))?; + } + let file = fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(&path) + .with_context(|| format!("failed to open lock file {}", path.display()))?; + let deadline = Instant::now() + timeout; + loop { + match file.try_lock() { + Ok(()) => return Ok(Self { file, path }), + Err(std::fs::TryLockError::WouldBlock) => {} + Err(std::fs::TryLockError::Error(error)) => { + return Err(anyhow::Error::new(error) + .context(format!("failed to lock {}", path.display()))); + } + } + let now = Instant::now(); + if now >= deadline { + bail!( + "timed out after {timeout:?} waiting for the lock at {path}; another \ + `rocm serve` launch or `rocmd` recovery is mid-transaction. Retry in a \ + moment, and check `rocm services` for a stuck launch.", + path = path.display() + ); + } + thread::sleep(FILE_LOCK_BOUNDED_POLL.min(deadline - now)); + } + } +} + +/// Polling interval used while waiting on [`FileLock::acquire_bounded`]. +const FILE_LOCK_BOUNDED_POLL: Duration = Duration::from_millis(25); + +/// Total time `rocmd`'s automatic recovery waits for the launch lock. +/// +/// Bounds contention on [`crate::paths::AppPaths::managed_launch_lock_path`] during the +/// port-allocation window, failing with an actionable error instead of hanging. Only +/// `rocmd`'s recovery/restart path is bounded this way; a `rocm serve` launch holds the +/// same lock with the unbounded [`FileLock::acquire`] (see its doc comment on +/// [`FileLock::acquire_bounded`] for why). +pub const MANAGED_LAUNCH_LOCK_TIMEOUT: Duration = Duration::from_secs(5); + +// --------------------------------------------------------------------------- +// Managed-service port leasing. +// +// Both `rocm serve` and `rocmd` recovery contend for the same local TCP ports +// and the same service records, so the primitives that arbitrate them live here +// rather than in either binary. The transaction itself is serialized by the +// existing [`crate::paths::AppPaths::managed_launch_lock_path`] / [`FileLock`] — see +// [`FileLock::acquire_bounded`] for the bounded variant this contention uses. +// --------------------------------------------------------------------------- + +/// True only for the exact canonical default loopback host. +/// +/// Proactive OS collision detection (leasing a real socket before launch) is +/// claimed only for this one host. `localhost`, `::1`, `0.0.0.0`, and other +/// IPv4 addresses are "custom hosts": they require an explicit port and keep the +/// existing engine-owned bind/error behavior, because an IPv4-loopback probe +/// would not prove anything about the address the engine will actually bind. +#[must_use] +pub fn is_canonical_loopback_host(host: &str) -> bool { + host.trim() == DEFAULT_LOCAL_HOST +} + +/// A real, held `127.0.0.1:` listener proving the port is bindable now. +/// +/// The lease is the collision check: it is acquired before any manifest, log, or +/// child process exists, and [`LoopbackPortLease::release`] drops the socket +/// immediately before the engine child is spawned so the child can bind it. The +/// window between release and the child's bind is unavoidable without engine +/// socket-passing; it is far narrower than the previous "bind and hope" launch. +#[derive(Debug)] +pub struct LoopbackPortLease { + port: u16, + listener: Option, +} + +impl LoopbackPortLease { + #[must_use] + pub const fn port(&self) -> u16 { + self.port + } + + /// True while this lease still holds the socket open. + /// + /// No production caller needs this: `release` is the only thing that ever + /// changes it, and callers already know whether they called `release`. It + /// stays `pub` because downstream crates' tests assert on it directly + /// (e.g. `apps/rocm`'s `serve_port` tests), where `#[cfg(test)]` here + /// would not apply. + #[must_use] + pub const fn is_held(&self) -> bool { + self.listener.is_some() + } + + /// Close the socket and hand the port to the caller. Call this immediately + /// before spawning the process that will bind it. + pub fn release(mut self) -> u16 { + self.listener = None; + self.port + } + + /// A lease over a port this process never probed, for custom hosts where the + /// engine, not the CLI, owns bind success or failure. + #[must_use] + pub const fn engine_owned(port: u16) -> Self { + Self { + port, + listener: None, + } + } +} + +/// Try to take an exclusive IPv4-loopback lease on `port`. +/// +/// Errors are returned verbatim so callers can distinguish "occupied" +/// ([`std::io::ErrorKind::AddrInUse`], the only condition that should advance an +/// automatic scan) from a privileged or unavailable address, which must fail +/// immediately rather than silently choosing a different port. +pub fn lease_loopback_port(port: u16) -> std::io::Result { + let listener = TcpListener::bind(std::net::SocketAddrV4::new( + std::net::Ipv4Addr::LOCALHOST, + port, + ))?; + Ok(LoopbackPortLease { + port, + listener: Some(listener), + }) } impl Drop for FileLock { diff --git a/crates/rocm-core/src/service_record.rs b/crates/rocm-core/src/service_record.rs index ce0459001..134f46890 100644 --- a/crates/rocm-core/src/service_record.rs +++ b/crates/rocm-core/src/service_record.rs @@ -322,6 +322,17 @@ impl ManagedServiceRecord { Ok(self.status != previous) } + /// Whether this record still reserves its port: the sole definition of + /// "live" shared between the CLI's launch-time port check and `rocmd`'s + /// recovery, so the two can never disagree about who owns a port. + #[must_use] + pub fn is_live(&self) -> bool { + matches!( + self.status.as_str(), + "ready" | "running" | "starting" | "recovering" + ) + } + fn with_storage_paths(&self) -> Self { let mut record = self.clone(); record.manifest_path = normalize_runtime_path_for_storage(&record.manifest_path); diff --git a/crates/rocm-dash-tui/src/agent/tools.rs b/crates/rocm-dash-tui/src/agent/tools.rs index 0416d26b3..2f169032d 100644 --- a/crates/rocm-dash-tui/src/agent/tools.rs +++ b/crates/rocm-dash-tui/src/agent/tools.rs @@ -592,7 +592,7 @@ rocm_mutating_tool!( "model": { "type": "string", "description": "Model id/name to serve." }, "engine": { "type": "string", "description": "Optional engine, e.g. 'vllm'." }, "host": { "type": "string", "description": "Loopback host only (e.g. 127.0.0.1)." }, - "port": { "type": "integer", "description": "Optional TCP port." }, + "port": { "type": "integer", "description": "TCP port. Required unless host is exactly 127.0.0.1 (localhost/::1 still need one)." }, "device": { "type": "string", "description": "GPU device selector (CPU is rejected)." } }, "required": ["model"] diff --git a/crates/rocm-dash-tui/src/app/mod.rs b/crates/rocm-dash-tui/src/app/mod.rs index 68c84c8ec..967ecb297 100644 --- a/crates/rocm-dash-tui/src/app/mod.rs +++ b/crates/rocm-dash-tui/src/app/mod.rs @@ -1806,7 +1806,6 @@ mod tests { }; wizard.editor = Some(crate::ui::serve_wizard::InlineEditor { field: crate::ui::serve_wizard::Field::Host, - original: "127.0.0.1".into(), value: "localhost".into(), cursor: 9, selected: false, diff --git a/crates/rocm-dash-tui/src/ui/serve_wizard.rs b/crates/rocm-dash-tui/src/ui/serve_wizard.rs index 41fc018c2..2113231a2 100644 --- a/crates/rocm-dash-tui/src/ui/serve_wizard.rs +++ b/crates/rocm-dash-tui/src/ui/serve_wizard.rs @@ -140,8 +140,6 @@ pub enum PortMode { #[derive(Debug, Clone)] pub struct InlineEditor { pub field: Field, - /// Value to restore on Escape. - pub original: String, pub value: String, /// Cursor position, in characters, in `0..=value.chars().count()`. pub cursor: usize, @@ -153,7 +151,6 @@ impl InlineEditor { fn new(field: Field, value: &str) -> Self { Self { field, - original: value.to_string(), value: value.to_string(), cursor: value.chars().count(), selected: true, @@ -317,6 +314,10 @@ impl ServeWizardState { Field::Mode => self.managed = !self.managed, _ => {} } + // Cycling Port back to Custom can fix the exact complaint an outstanding + // message is naming (e.g. "set Port to Custom"); without this, that + // message would sit on screen after the toggle that resolved it. + refresh_message(self); } /// Direct typing only applies to the Model row; Host and the custom port @@ -567,7 +568,7 @@ pub fn on_key( Vec::new() } -/// Drive the inline editor. Enter accepts, Escape restores, and nothing here +/// Drive the inline editor. Enter accepts, Escape discards, and nothing here /// can open another overlay. fn editor_key(w: &mut ServeWizardState, code: KeyCode) { if w.editor.is_none() { @@ -575,14 +576,9 @@ fn editor_key(w: &mut ServeWizardState, code: KeyCode) { } match code { KeyCode::Esc => { - // Restore the original — an abandoned edit never mutates the form. - if let Some(ed) = w.editor.take() { - match ed.field { - Field::Host => w.host = ed.original, - Field::Port => w.port = ed.original, - _ => {} - } - } + // The form's host/port are only written on Enter, so discarding + // the editor here never mutates them. + w.editor = None; refresh_message(w); } KeyCode::Left | KeyCode::Right => { @@ -1497,7 +1493,7 @@ mod tests { } #[test] - fn editor_escape_restores_the_original_value() { + fn editor_escape_discards_the_edit() { let mut wiz = Some(ServeWizardState { advanced_expanded: true, ..Default::default() @@ -1512,7 +1508,7 @@ mod tests { let w = wiz.as_ref().unwrap(); assert!(w.editor.is_none(), "Esc closes the editor"); assert!(wiz.is_some(), "Esc in the editor never closes the overlay"); - assert_eq!(w.host, "127.0.0.1", "original restored"); + assert_eq!(w.host, "127.0.0.1", "edit was never committed to the form"); } #[test] diff --git a/docs/manual-testing.md b/docs/manual-testing.md index f29df5486..432514d11 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -285,6 +285,19 @@ While the log stream is attached, verify detach and stop behave differently: - Re-run the serve command and press `Ctrl+C` instead. The server shuts down and `rocm services` no longer lists it as running. +Omit `--port` on the default loopback host to exercise automatic selection: + +```powershell +rocm serve qwen --engine lemonade --device gpu_required --managed +``` + +Expected result: + +- The command picks the first free port in `11435`-`11535` instead of failing + or always landing on `11435`. +- `rocm services` shows the launched service bound to the port it picked, and + the deployment summary's endpoint matches it. + ## 4. Local Server Records After a managed or foreground serve attempt, inspect local server records: diff --git a/docs/testing.md b/docs/testing.md index 0ed561fcd..d869038cd 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -956,7 +956,7 @@ cargo test -p rocmd recovery_waits_for_the_allocation_lock_and_refuses_an_occupi Omitting `--port` on the default loopback host scans `11435..=11535` for a candidate neither reserved by a live managed-service record nor occupied on the -OS (`serve_port.rs`), under the same `services/launch.lock` that serializes +OS ([`serve_port.rs`](../apps/rocm/src/serve_port.rs)), under the same `services/launch.lock` that serializes every other managed launch and `rocmd` recovery attempt. An explicit `--port` on a non-default `--host` is still required: automatic selection only probes `127.0.0.1` and refuses to guess elsewhere. diff --git a/skills/rocm-cli-assistant/SKILL.md b/skills/rocm-cli-assistant/SKILL.md index 0b54a2208..9cc280d68 100644 --- a/skills/rocm-cli-assistant/SKILL.md +++ b/skills/rocm-cli-assistant/SKILL.md @@ -28,6 +28,7 @@ Use this skill when answering ROCm CLI local assistant questions. - The built-in assistant is fixed to qwen served by Lemonade with GPU required. Do not switch the built-in assistant to vLLM. - Installing an engine and running a model server are different states. Answer each separately when the user asks. - `rocm serve` accepts `--gpu auto|` to pick the AMD GPU. `auto` (default) prefers a GPU that looks idle from VRAM telemetry (`amd-smi`, or the amdgpu DRM sysfs counters when `amd-smi` is absent) and is not already used by another rocm-cli server (managed or foreground), falling back to the GPU with the most free memory; a single index pins one GPU. Serving one model across multiple GPUs is not supported. Do not suggest CPU fallback when a GPU is busy or out of range. +- Automatic port selection only works on host `127.0.0.1` exactly. `localhost` and `::1` are loopback for auth purposes but still need an explicit `port` when launching a server, or the call is refused. - Which engines actually run on the user's machine is stated in the host facts the CLI appends to this prompt. Answer from those facts; do not assume a platform. Never suggest CPU fallback. - On native Windows, vLLM serving/install live checks are skipped; tell the user to use WSL/Linux for that ROCm GPU engine and do not suggest CPU fallback. diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index ded08139b..ec8223478 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -368,7 +368,11 @@ Feature: Model serving # bind is attempted, naming the service so the user can stop it or pick # another port. Reaching this check needs a real GPU: `resolve_serve_port` # only runs once `spawn_managed_engine_child` has a completed model/engine - # resolution in hand, and `serve` has no CPU fallback that gets there. + # resolution in hand, and `serve` has no CPU fallback that gets there. The + # "stop the service holding that port" advice is checked here by its wording + # only; the advice is actually proven by + # `stopping_the_reserving_record_frees_its_explicit_port_for_reuse` in + # apps/rocm/src/service_launch.rs, per AGENTS.md section 3. # Runs in the merge queue, like its automatic-port sibling serve-auto-port-skips-occupied. @id:serve-explicit-port-reserved @requires-gpu @merge-queue Scenario: serve-27 - Serving on a port another managed service already holds is refused diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index df4651d1a..c566d24ea 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -412,21 +412,26 @@ async fn occupy_default_serve_port(world: &mut E2eWorld) { } /// Plant a live managed-service record on the default serve port, with no real -/// listener behind it. `resolve_serve_port`'s explicit-port check matches on the -/// record's `port` field alone, before it ever probes the OS (see -/// `reserved_service_ports`'s "conservative by design" doc comment), so a live -/// record is enough to make the collision deterministic without a second real -/// server. +/// listener or process behind it. `resolve_serve_port`'s explicit-port check +/// matches on the record's `port` field alone, before it ever probes the OS +/// (see `reserved_service_ports`'s "conservative by design" doc comment), so a +/// live record is enough to make the collision deterministic without a second +/// real server. `ServiceRecordOptions::default()`'s `supervisor_pid: 0` keeps +/// this safe for the World's teardown, which runs `rocm services stop +/// --yes` on every record left in the isolated tree: a real PID here would +/// point that stop at the test process itself (see +/// `plant_unauthenticated_lemonade_service`'s doc comment for the same +/// hazard). #[given("another managed service already reserves the default serve port")] async fn reserve_default_serve_port(world: &mut E2eWorld) { let root = world.isolated_root.as_ref().expect("no isolated root"); let services = root.path().join("data").join("services"); - let options = ServiceRecordOptions { - supervisor_pid: std::process::id(), - engine_pid: Some(std::process::id()), - ..ServiceRecordOptions::default() - }; - write_service_record_with(&services, "other-model", SERVE_PORT, options); + write_service_record_with( + &services, + "other-model", + SERVE_PORT, + ServiceRecordOptions::default(), + ); } /// The (model, engine, ready-substring) this host should serve for an From c4f1ebc1bd34557d0ce863da061b7217c1e7c60b Mon Sep 17 00:00:00 2001 From: Juho Vainio Date: Thu, 15 Oct 2026 13:10:33 +0300 Subject: [PATCH 06/19] fix(serve): gate the zombie-reaping cleanup test out on Windows abandoned_launch_terminates_the_spawned_child and its sleeper-child helper use ProcessCommand/Stdio, which this file already gates #[cfg(not(windows))] at the import site. The test itself was missing the same gate, so it failed to compile on the Windows CI lane. Its own comment already notes it exercises Unix zombie-reaping semantics (kill(pid, 0) on an unreaped zombie), so it has no Windows equivalent to provide. Signed-off-by: Juho Vainio --- apps/rocm/src/service_launch.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/rocm/src/service_launch.rs b/apps/rocm/src/service_launch.rs index 6d816a6ff..c779b09cd 100644 --- a/apps/rocm/src/service_launch.rs +++ b/apps/rocm/src/service_launch.rs @@ -2859,6 +2859,7 @@ mod tests { } #[test] + #[cfg(not(windows))] #[ignore = "spawned as a child process by abandoned_launch_terminates_the_spawned_child"] fn managed_launch_cleanup_sleeper_child() { if std::env::var_os("ROCM_TEST_SLEEPER_CHILD").is_some() { @@ -2867,6 +2868,7 @@ mod tests { } #[test] + #[cfg(not(windows))] fn abandoned_launch_terminates_the_spawned_child() -> Result<()> { let (root, paths) = test_paths("abandon-launch-terminates-child"); paths.ensure()?; From 741a541072f7b9f3112a9aab2457bf3e9c5027d8 Mon Sep 17 00:00:00 2001 From: Juho Vainio Date: Thu, 15 Oct 2026 13:49:08 +0300 Subject: [PATCH 07/19] fix(serve): detect an immediately-dead managed engine child on Windows launch_managed_engine_child_locked's Unix branch already slept 200ms and called try_wait() to catch a child that exited immediately after spawn, bailing with a "managed engine exited immediately" error so the caller's cleanup path can terminate it and keep the record/log for inspection. The Windows branch (spawn_detached_no_inherit only returns a pid, not a Child handle) had no equivalent check and always reported success, even for a child that died on the spot. Add the same liveness check on Windows using the already-cross-platform process_is_running(pid) helper. managed_engine_startup_failure_detail now takes a pre-formatted status clause instead of a Unix-only ExitStatus, so both platforms (and the restart call site in main.rs) can share it. Signed-off-by: Juho Vainio --- apps/rocm/src/main.rs | 5 +++- apps/rocm/src/service_launch.rs | 41 ++++++++++++++++++++++++--------- 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 2934f9907..0a2a7cbbf 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -7366,7 +7366,10 @@ fn restart_internal_managed_service_with( })?; bail!( "{}", - managed_engine_startup_failure_detail(status, &record.log_path) + managed_engine_startup_failure_detail( + &format!(" with status {status}"), + &record.log_path + ) ); } child.id() diff --git a/apps/rocm/src/service_launch.rs b/apps/rocm/src/service_launch.rs index c779b09cd..1ae08549b 100644 --- a/apps/rocm/src/service_launch.rs +++ b/apps/rocm/src/service_launch.rs @@ -34,8 +34,6 @@ use std::fs; use std::io::{self, BufRead, Write}; use std::path::{Path, PathBuf}; #[cfg(not(windows))] -use std::process::ExitStatus; -#[cfg(not(windows))] use std::process::{Command as ProcessCommand, Stdio}; use std::thread; use std::time::Duration; @@ -45,8 +43,11 @@ use rocm_core::{AppPaths, AutomationRuntimeState, EndpointReadiness, ManagedServ use rocm_core::{format_http_base_url, process_is_running}; use rocm_engine_protocol::{DevicePolicy, EngineRecipeHint, ResolveModelResponse}; +#[cfg(not(windows))] +use crate::apply_app_path_env; use crate::endpoint_keys; use crate::engines_cmd::env_root_for_service; +use crate::read_optional_tail_lines; use crate::serve_port; use crate::service_hints; use crate::{ @@ -57,8 +58,6 @@ use crate::{ }; #[cfg(windows)] use crate::{app_path_env_var_refs, app_path_env_var_values}; -#[cfg(not(windows))] -use crate::{apply_app_path_env, read_optional_tail_lines}; pub(crate) fn validate_bind_host(host: &str, allow_public_bind: bool) -> Result<()> { if !is_loopback_host(host) && !allow_public_bind { @@ -242,8 +241,14 @@ pub(crate) fn attach_background_stdio( Ok(()) } -#[cfg(not(windows))] -pub(crate) fn managed_engine_startup_failure_detail(status: ExitStatus, log_path: &Path) -> String { +/// `status_clause` is a pre-formatted fragment inserted right after "exited +/// immediately" (e.g. `" with status exit status: 1"` on Unix, where a real +/// `ExitStatus` is available; empty on Windows, where the liveness check +/// below only has a yes/no answer, not an exit code). +pub(crate) fn managed_engine_startup_failure_detail( + status_clause: &str, + log_path: &Path, +) -> String { let mut recent_lines = read_optional_tail_lines(log_path, 80, "service log"); if recent_lines.is_empty() { for _ in 0..5 { @@ -256,12 +261,12 @@ pub(crate) fn managed_engine_startup_failure_detail(status: ExitStatus, log_path } if recent_lines.is_empty() { return format!( - "managed engine exited immediately with status {status}; inspect {}", + "managed engine exited immediately{status_clause}; inspect {}", log_path.display() ); } format!( - "managed engine exited immediately with status {status}; inspect {}\n\nrecent startup log output:\n{}", + "managed engine exited immediately{status_clause}; inspect {}\n\nrecent startup log output:\n{}", log_path.display(), recent_lines.join("\n") ) @@ -691,8 +696,19 @@ fn launch_managed_engine_child_locked( if let Some(key_file) = endpoint_key_file.as_deref() { env_refs.push((rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV, key_file)); } - rocm_core::spawn_detached_no_inherit(¤t_exe, &serve_args, &env_refs) - .context("failed to launch managed engine process")? + let child_pid = rocm_core::spawn_detached_no_inherit(¤t_exe, &serve_args, &env_refs) + .context("failed to launch managed engine process")?; + // Set before the immediate-exit check below, so a bail! here still leaves + // `record.engine_pid` telling cleanup that a child was spawned. + record.engine_pid = Some(child_pid); + thread::sleep(Duration::from_millis(200)); + if !process_is_running(child_pid) { + bail!( + "{}", + managed_engine_startup_failure_detail("", &record.log_path) + ); + } + child_pid }; #[cfg(not(windows))] let child_pid = { @@ -721,7 +737,10 @@ fn launch_managed_engine_child_locked( { bail!( "{}", - managed_engine_startup_failure_detail(status, &record.log_path) + managed_engine_startup_failure_detail( + &format!(" with status {status}"), + &record.log_path + ) ); } child_pid From 7d0927c03ff415521ec7bd3b7a5a9f02ab95f21a Mon Sep 17 00:00:00 2001 From: Juho Vainio Date: Thu, 15 Oct 2026 15:45:11 +0300 Subject: [PATCH 08/19] fix(serve): unify port-collision checks and fix review findings Address live findings from both pr-review-watcher rounds on PR #243: - Add ManagedServiceRecord::blocks_port(), the single host-blind collision predicate (is_live() && port == port) now shared by the CLI restart re-check and rocmd's recovery re-check, so the two can never disagree about who owns a port. The rocmd side also fixes a latent ordering bug where only the first live record was checked before filtering by host/port. - Extract start_managed_service_dropping_orphaned_key_on_failure() so the orphaned-endpoint-key cleanup wraps the real production call site instead of being hand-simulated in tests. - Rewrite the invalid-key-file test to use a key that is invalid because of embedded CR/LF rather than merely empty, and assert the unusable key file is deleted from disk, not just unreadable. - Add regression tests for mutation gaps the reviews found: a restart refused by a real bound listener (not just a stale record), a rocmd recovery rollback when the first manifest write fails, and a custom-host restart that correctly skips the loopback lease. - Add serve_wizard tests covering space-toggle-while-focused and the outstanding-complaint clearing on port-mode cycling. - Give MockServer/http_server a start_on_port() path so the serve-27 e2e scenario's planted collision record is backed by a real listener instead of being demoted to "stopped" by the liveness probe before the collision check fires. Known gap, intentionally left: the rocmd restart rollback branch for a failed record_spawned_supervisor (including its own nested rollback-also-failed path) is still untested. Covering it deterministically would need either a flaky sub-millisecond race or a test-only production seam, both out of scope for this pass. Signed-off-by: Juho Vainio --- apps/rocm/src/main.rs | 33 +++++- apps/rocm/src/serve_cmd.rs | 22 ++-- apps/rocm/src/service_launch.rs | 107 ++++++++++++++--- apps/rocmd/src/watchers.rs | 110 +++++++++++++++++- crates/rocm-core/src/service_record.rs | 13 +++ crates/rocm-dash-tui/src/ui/serve_wizard.rs | 32 +++++ docs/manual-testing.md | 13 +++ tests/e2e-cucumber/src/http_server.rs | 23 +++- tests/e2e-cucumber/src/mock_server.rs | 29 +++++ tests/e2e-cucumber/tests/e2e/serving_steps.rs | 30 +++-- 10 files changed, 356 insertions(+), 56 deletions(-) diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 0a2a7cbbf..e17f1435e 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -7277,10 +7277,7 @@ fn restart_internal_managed_service_with( .context("failed to re-read managed services under the launch lock")? .into_iter() .find(|other| { - other.service_id != stopped_record.service_id - && other.is_live() - && other.host == stopped_record.host - && other.port == stopped_record.port + other.service_id != stopped_record.service_id && other.blocks_port(stopped_record.port) }) { bail!( @@ -17713,6 +17710,34 @@ pub(crate) mod tests { Ok(()) } + /// The disk re-check above only catches a recorded competitor. Something + /// bound to the exact port without ever publishing a manifest for it — a + /// stray process, a one-off `nc -l` — would sail through that check; only + /// the loopback lease actually binding the port catches it. A real + /// listener held on the port is what proves that lease ran, not just a + /// record on disk. + #[test] + fn restart_refuses_a_port_something_is_actually_bound_to() -> Result<()> { + let (root, paths) = test_paths("restart-port-actually-bound"); + paths.ensure()?; + let service_id = "svc-restart-port-actually-bound"; + let record = stop_test_record(&paths, service_id); + let port = record.port; + record.write()?; + + let _blocker = std::net::TcpListener::bind(("127.0.0.1", port))?; + + let result = + restart_internal_managed_service_with(&paths, service_id, None, |_| (Vec::new(), true)); + + let _ = fs::remove_dir_all(root); + let error = + result.expect_err("a port actually bound by someone else must refuse the restart"); + let rendered = format!("{error:#}"); + assert!(rendered.contains("its port is in use"), "{rendered}"); + Ok(()) + } + #[test] fn serve_notes_surface_the_ignored_engine_flag_in_summary_mode() { // The interactive summary is what a default `rocm serve` prints, so a flag diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index 653ecb4c1..07bc91852 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -13,7 +13,8 @@ //! //! `collect_serve_notes` and `validate_pinned_gpu_index` stay in `main.rs` //! (no entanglement; a later pass can relocate them). The -//! managed-service-spawning tail this file calls into — `start_managed_service`, +//! managed-service-spawning tail this file calls into — +//! `start_managed_service_dropping_orphaned_key_on_failure`, //! `run_attached_service`, `resolve_endpoint_auth`, //! `ensure_public_bind_engine_supported`, //! `drop_orphaned_endpoint_key_on_already_running`, `print_managed_launch_plain`, @@ -43,7 +44,8 @@ use crate::serve_summary; use crate::service_launch::{ drop_orphaned_endpoint_key_on_already_running, ensure_background_helper_running_quiet, ensure_public_bind_engine_supported, print_managed_launch_plain, resolve_endpoint_auth, - run_attached_service, start_managed_service, validate_bind_host, + run_attached_service, start_managed_service_dropping_orphaned_key_on_failure, + validate_bind_host, }; use crate::{ cli_progress, collect_serve_notes, detect_gpu_count, device_policy_name, engine_request, @@ -862,7 +864,8 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { let mut spinner = cli_progress::Spinner::new(format!("Starting {model} on {selected_engine}…")); spinner.tick(); - let report = start_managed_service( + let report = start_managed_service_dropping_orphaned_key_on_failure( + &paths, &selected_engine, &service_id, &model, @@ -878,18 +881,7 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { launch_lock, require_api_key, &mut |_elapsed| spinner.tick(), - ) - // A pre-spawn failure (port reserved/in use, range exhausted, other bind - // error) returns before any service record exists, so nothing else clears - // the key just stored for this attempt. Idempotent with the post-spawn - // cleanup in `abandon_failed_managed_launch`. - .inspect_err(|_| { - drop_orphaned_endpoint_key_on_already_running( - &paths, - &service_id, - endpoint_auth.as_deref(), - ); - })?; + )?; ensure_background_helper_running_quiet(summary_mode)?; // An equivalent service was already running, so nothing was spawned and the diff --git a/apps/rocm/src/service_launch.rs b/apps/rocm/src/service_launch.rs index 1ae08549b..c46a0e531 100644 --- a/apps/rocm/src/service_launch.rs +++ b/apps/rocm/src/service_launch.rs @@ -771,6 +771,7 @@ fn launch_managed_engine_child_locked( #[allow(clippy::too_many_arguments)] pub(crate) fn start_managed_service( + paths: &AppPaths, engine: &str, service_id: &str, requested_model: &str, @@ -787,9 +788,8 @@ pub(crate) fn start_managed_service( require_api_key: bool, on_wait_tick: &mut dyn FnMut(Duration), ) -> Result { - let paths = AppPaths::discover()?; let (mut record, child_pid) = match spawn_managed_engine_child( - &paths, + paths, engine, service_id, requested_model, @@ -837,7 +837,7 @@ pub(crate) fn start_managed_service( record.write()?; let endpoint_url = format!("{}/v1", format_http_base_url(host, port)); record_cli_audit_event( - &paths, + paths, "service", "managed_service_launch", "info", @@ -858,6 +858,58 @@ pub(crate) fn start_managed_service( }) } +/// As [`start_managed_service`], but also drops the endpoint key `serve()` +/// freshly staged for this attempt if the launch fails before any +/// `ManagedServiceRecord` exists to own it (port reserved/in use, range +/// exhausted, other bind error) — such a failure returns before +/// `abandon_failed_managed_launch` ever runs, so nothing else would clear it. +/// Idempotent with that post-spawn cleanup. This is the real call site +/// `serve_cmd.rs`'s background launch path uses, so it is what +/// `failed_port_reservation_leaves_no_orphaned_endpoint_key` drives directly, +/// mirroring how [`run_attached_service`] is driven directly for the +/// foreground path. +#[allow(clippy::too_many_arguments)] +pub(crate) fn start_managed_service_dropping_orphaned_key_on_failure( + paths: &AppPaths, + engine: &str, + service_id: &str, + requested_model: &str, + resolve: &ResolveModelResponse, + host: &str, + port_request: serve_port::PortRequest, + device_policy: &DevicePolicy, + gpu_indices: &[u32], + runtime_id: Option<&str>, + env_id: Option<&str>, + engine_recipe: Option<&EngineRecipeHint>, + endpoint_api_key: Option<&str>, + launch_lock: rocm_core::FileLock, + require_api_key: bool, + on_wait_tick: &mut dyn FnMut(Duration), +) -> Result { + start_managed_service( + paths, + engine, + service_id, + requested_model, + resolve, + host, + port_request, + device_policy, + gpu_indices, + runtime_id, + env_id, + engine_recipe, + endpoint_api_key, + launch_lock, + require_api_key, + on_wait_tick, + ) + .inspect_err(|_| { + drop_orphaned_endpoint_key_on_already_running(paths, service_id, endpoint_api_key); + }) +} + /// Reproduce the original plain, line-by-line managed-launch output. Used for /// non-interactive output (piped, CI, the chat assistant's `serve --managed`), /// where the animated summary is inappropriate. The interactive path renders the @@ -1620,7 +1672,7 @@ mod tests { fn a_managed_spawn_refuses_an_invalid_key_file_on_a_service_that_requires_one() -> Result<()> { // Drives the real call site, not the guard's own arguments. The service // was launched with `--require-api-key`, and its key file is present but - // empty — so `requires_api_key` is true while `key_present` is false, + // invalid — so `requires_api_key` is true while `key_present` is false, // which is the only way to reach the first branch. A test calling // `ensure_public_service_has_endpoint_key` directly cannot catch a // mis-wired call site, which is the defect that has occurred here twice. @@ -1632,9 +1684,18 @@ mod tests { paths.ensure()?; fs::create_dir_all(paths.services_dir())?; - // Empty, so the file exists (requires_api_key = true) but yields no - // usable key (key_present = false). - endpoint_keys::store_endpoint_api_key(&paths, "lemonade-qwen-3000", "")?; + // Non-empty but invalid (embedded CR/LF), so the file exists and is not + // merely empty (requires_api_key = true) but yields no usable key + // (key_present = false) — `endpoint_api_key_from_file` rejects embedded + // control characters. An empty seed here would let the post-condition + // below pass even if the real cleanup call never ran, since + // `endpoint_api_key` already reads back `None` for an empty file whether + // or not it was ever deleted. + endpoint_keys::store_endpoint_api_key( + &paths, + "lemonade-qwen-3000", + "bad-key\r\nX-Injected: value", + )?; let resolve = ResolveModelResponse { canonical_model_id: "qwen-canonical".to_owned(), @@ -1679,7 +1740,7 @@ mod tests { let manifest_path = paths.service_manifest_path("lemonade-qwen-3000"); let log_path = paths.service_log_path("lemonade-qwen-3000"); let engine_state_path = paths.service_engine_state_path("lemonade", "lemonade-qwen-3000"); - let key = endpoint_keys::endpoint_api_key(&paths, "lemonade-qwen-3000"); + let key_file_path = endpoint_keys::endpoint_key_file_path(&paths, "lemonade-qwen-3000"); let live = live_managed_services(&paths); let _ = fs::remove_dir_all(root); @@ -1707,9 +1768,10 @@ mod tests { !engine_state_path.exists(), "a pre-spawn failure must not leave engine state" ); - assert_eq!( - key, None, - "a pre-spawn failure must not leave the unusable key behind" + assert!( + !key_file_path.exists(), + "a pre-spawn failure must delete the unusable key file from disk, not merely \ + leave it unreadable as a key" ); assert!( live.is_empty(), @@ -1814,12 +1876,16 @@ mod tests { #[test] fn failed_port_reservation_leaves_no_orphaned_endpoint_key() -> Result<()> { // serve() stores the new attempt's endpoint key before calling - // start_managed_service. If spawn_managed_engine_child fails inside - // resolve_managed_port_in_transaction (explicit port already reserved - // by another live service), it returns before any ManagedServiceRecord - // exists for this attempt, so abandon_failed_managed_launch never runs. - // The caller's .inspect_err() cleanup (drop_orphaned_endpoint_key_on_already_running) - // must be what clears the key. + // start_managed_service_dropping_orphaned_key_on_failure. If + // spawn_managed_engine_child fails inside resolve_managed_port_in_transaction + // (explicit port already reserved by another live service), it returns + // before any ManagedServiceRecord exists for this attempt, so + // abandon_failed_managed_launch never runs. Driving the real wrapper + // function end to end (not a simulated call to + // drop_orphaned_endpoint_key_on_already_running) proves serve_cmd.rs's own + // call site actually wires the cleanup, mirroring how + // attached_launch_failing_before_spawn_leaves_no_orphaned_endpoint_key + // drives run_attached_service for the foreground path. let (root, paths) = test_paths("orphaned-key-on-reserved-port"); paths.ensure()?; let mut other = ManagedServiceRecord::new( @@ -1858,8 +1924,9 @@ mod tests { let service_id = "lemonade-qwen-2000"; endpoint_keys::store_endpoint_api_key(&paths, service_id, "test-key")?; + let launch_lock = rocm_core::FileLock::acquire(paths.managed_launch_lock_path())?; - let result = spawn_managed_engine_child( + let result = start_managed_service_dropping_orphaned_key_on_failure( &paths, "lemonade", service_id, @@ -1872,7 +1939,10 @@ mod tests { None, None, None, + Some("test-key"), + launch_lock, false, + &mut |_elapsed| {}, ); let Err(error) = result else { let _ = fs::remove_dir_all(root); @@ -1885,7 +1955,6 @@ mod tests { "unexpected error: {error}" ); - drop_orphaned_endpoint_key_on_already_running(&paths, service_id, Some("test-key")); assert!( endpoint_keys::endpoint_api_key(&paths, service_id).is_none(), "cleanup must drop the key this failed attempt staged before launching" diff --git a/apps/rocmd/src/watchers.rs b/apps/rocmd/src/watchers.rs index 80215114c..dbd54d23b 100644 --- a/apps/rocmd/src/watchers.rs +++ b/apps/rocmd/src/watchers.rs @@ -1492,8 +1492,7 @@ pub(crate) fn restart_managed_service( .context("failed to re-read managed services under the launch lock")?; if let Some(collision) = on_disk .iter() - .find(|other| other.service_id != record.service_id && other.is_live()) - .filter(|other| other.host == record.host && other.port == record.port) + .find(|other| other.service_id != record.service_id && other.blocks_port(record.port)) { anyhow::bail!( "cannot recover managed service {} on {}:{}: {} claimed that port while recovery \ @@ -3827,6 +3826,113 @@ mod tests { assert!(child.wait().is_ok()); } + /// If publishing the `recovering` transition fails, nothing has been + /// spawned yet: the rollback at that first write must undo the in-memory + /// transition and leave the on-disk record exactly as it was, not a + /// half-applied `recovering` row. A manifest file made read-only after + /// being seeded fails that exact write without ever reaching the spawn. + /// + /// Unix only and skipped as root, where a read-only mode is not enforced + /// and the write would simply succeed. + #[cfg(unix)] + #[test] + fn restart_rolls_back_the_in_memory_record_when_the_first_write_fails() -> Result<()> { + use std::os::unix::fs::PermissionsExt; + + if rocm_core::openmpi::running_as_root() { + return Ok(()); + } + + let (root, paths) = temp_app_paths("restart-rollback-first-write"); + paths.ensure()?; + let service_id = "svc-restart-rollback-first-write"; + let mut record = identity_probe_record(&paths, service_id, 11459); + record.status = "failed".to_owned(); + record.write()?; + + let manifest_path = paths.service_manifest_path(service_id); + fs::set_permissions(&manifest_path, std::fs::Permissions::from_mode(0o444))?; + + let result = restart_managed_service(&paths, &mut record); + + // Restore write access so the rest of the write (if any landed) can be + // read back, and so cleanup can remove the directory. + fs::set_permissions(&manifest_path, std::fs::Permissions::from_mode(0o644))?; + let persisted = load_service_record(&paths, service_id); + fs::remove_dir_all(root).ok(); + + let error = result.expect_err("a read-only manifest must fail the first write"); + assert!( + format!("{error:#}").contains("failed to publish the recovering service record"), + "{error:#}" + ); + assert_eq!( + record.status, "failed", + "a failed first write must restore the in-memory record rather than leave it \ + \"recovering\"" + ); + assert_eq!( + persisted?.status, "failed", + "nothing must reach disk when the first write itself fails" + ); + Ok(()) + } + + /// The lease branch above proves a bound loopback port refuses recovery; + /// the doc comment on `restart_managed_service`'s lease also claims custom + /// hosts skip that preflight entirely, relying on the engine's own bind + /// error instead. Binding the exact recorded port on a custom host and + /// still reaching a spawned supervisor is what proves that claim, rather + /// than just trusting the comment. + /// + /// Linux-only and ignores the restart's own outcome for the same reason as + /// `restart_managed_service_persists_the_supervisor_identity_token`: the + /// restart re-execs the test binary, which rejects the `supervise` + /// arguments and exits at once, so the restart itself still reports + /// failure — after writing the supervisor identity this test reads. + #[cfg(target_os = "linux")] + #[test] + fn restart_on_a_custom_host_skips_the_loopback_lease() -> Result<()> { + let (root, paths) = temp_app_paths("restart-custom-host-skips-lease"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let service_id = "svc-restart-custom-host"; + let mut record = ManagedServiceRecord::new( + &paths, + service_id, + "llamacpp", + "a-model", + "a-model", + "0.0.0.0", + 11460, + "managed", + std::process::id(), + None, + None, + Some("gpu_required".to_owned()), + ); + record.status = "failed".to_owned(); + record.write()?; + + let _blocker = std::net::TcpListener::bind(("0.0.0.0", 11460))?; + + let outcome = restart_managed_service(&paths, &mut record); + let persisted = load_service_record(&paths, service_id); + fs::remove_dir_all(root).ok(); + + let message = outcome.err().map(|e| format!("{e:#}")).unwrap_or_default(); + assert!( + !message.contains("its recorded port is in use"), + "a custom host must not preflight the port with a loopback lease: {message}" + ); + assert!( + persisted?.supervisor_start_ticks.is_some(), + "recovery must still reach a spawned supervisor despite the bound port" + ); + Ok(()) + } + /// A stop that lands while a restart waits to see whether its supervisor /// survived startup must outlive the restart's `failed` write: a `failed` /// record without the marker is exactly what recovery restarts. diff --git a/crates/rocm-core/src/service_record.rs b/crates/rocm-core/src/service_record.rs index 134f46890..2516fa9dd 100644 --- a/crates/rocm-core/src/service_record.rs +++ b/crates/rocm-core/src/service_record.rs @@ -333,6 +333,19 @@ impl ManagedServiceRecord { ) } + /// Whether this live record reserves `port` — the sole collision check + /// shared by the CLI's restart re-check and `rocmd`'s recovery re-check, so + /// the two can never disagree about who owns a port. + /// + /// Deliberately host-blind: a `0.0.0.0` (or `localhost`) listener also + /// blocks the loopback address a competing bind would target, so comparing + /// hosts would under-detect a real collision rather than merely cost one + /// skipped candidate. + #[must_use] + pub fn blocks_port(&self, port: u16) -> bool { + self.is_live() && self.port == port + } + fn with_storage_paths(&self) -> Self { let mut record = self.clone(); record.manifest_path = normalize_runtime_path_for_storage(&record.manifest_path); diff --git a/crates/rocm-dash-tui/src/ui/serve_wizard.rs b/crates/rocm-dash-tui/src/ui/serve_wizard.rs index 2113231a2..496503db6 100644 --- a/crates/rocm-dash-tui/src/ui/serve_wizard.rs +++ b/crates/rocm-dash-tui/src/ui/serve_wizard.rs @@ -1284,6 +1284,21 @@ mod tests { assert_eq!(wiz.as_ref().unwrap().current_field(), Field::Advanced); } + #[test] + fn space_toggles_advanced_only_while_focused() { + let mut wiz = Some(ServeWizardState::default()); + let mut jobs = State::default(); + // Model has focus: Space must not expand anything. + feed(&mut wiz, &mut jobs, &[KeyCode::Char(' ')]); + assert!(!wiz.as_ref().unwrap().advanced_expanded); + + focus_field(wiz.as_mut().unwrap(), Field::Advanced); + feed(&mut wiz, &mut jobs, &[KeyCode::Char(' ')]); + assert!(wiz.as_ref().unwrap().advanced_expanded); + feed(&mut wiz, &mut jobs, &[KeyCode::Char(' ')]); + assert!(!wiz.as_ref().unwrap().advanced_expanded); + } + #[test] fn expanded_focus_order_is_model_advanced_engine_device_host_port_mode_launch() { let mut w = ServeWizardState { @@ -1397,6 +1412,23 @@ mod tests { assert_eq!(wiz.as_ref().unwrap().port_mode, PortMode::Auto); } + #[test] + fn cycling_port_to_custom_clears_the_outstanding_complaint() { + let mut wiz = Some(ServeWizardState { + advanced_expanded: true, + model: "m".into(), + host: "localhost".into(), + ..Default::default() + }); + wiz.as_mut().unwrap().message = Some(CUSTOM_HOST_NEEDS_PORT.to_string()); + let mut jobs = State::default(); + focus_field(wiz.as_mut().unwrap(), Field::Port); + feed(&mut wiz, &mut jobs, &[KeyCode::Right]); + let w = wiz.as_ref().unwrap(); + assert_eq!(w.port_mode, PortMode::Custom); + assert!(w.message.is_none(), "the fix it named just happened"); + } + #[test] fn mode_toggles_between_managed_and_foreground() { let mut wiz = Some(ServeWizardState { diff --git a/docs/manual-testing.md b/docs/manual-testing.md index 432514d11..567bba8c9 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -298,6 +298,19 @@ Expected result: - `rocm services` shows the launched service bound to the port it picked, and the deployment summary's endpoint matches it. +With that service still running, try to launch a second one pinned to the +same port: + +```powershell +rocm serve qwen --engine lemonade --device gpu_required --managed --port +``` + +Expected result: + +- The command refuses up front, naming the service already holding that port + and how to free it (`rocm services stop --yes`), without + spawning an engine process or writing a new record. + ## 4. Local Server Records After a managed or foreground serve attempt, inspect local server records: diff --git a/tests/e2e-cucumber/src/http_server.rs b/tests/e2e-cucumber/src/http_server.rs index 8ebed5736..c72a385b8 100644 --- a/tests/e2e-cucumber/src/http_server.rs +++ b/tests/e2e-cucumber/src/http_server.rs @@ -76,7 +76,20 @@ impl Drop for ServerHandle { /// For servers started from async code that then yields normally (`.await`s) /// while the server is in use. pub async fn spawn(app: Router) -> ServerHandle { - let listener = bind().await; + spawn_on(app, 0).await +} + +/// Like [`spawn`], but binds a caller-specified port. +/// +/// For scenarios that plant a managed-service record naming an exact port and +/// need a real listener behind it (e.g. so the CLI's liveness probe doesn't +/// demote the record before an explicit-port collision check runs). +pub async fn spawn_on_port(app: Router, port: u16) -> ServerHandle { + spawn_on(app, port).await +} + +async fn spawn_on(app: Router, port: u16) -> ServerHandle { + let listener = bind(port).await; let addr = local_addr(&listener); let (shutdown_tx, shutdown_rx) = oneshot::channel(); tokio::spawn(serve(listener, app, shutdown_rx)); @@ -103,7 +116,7 @@ pub fn spawn_on_own_thread(app: Router) -> ServerHandle { .build() .expect("failed to build test server runtime"); runtime.block_on(async move { - let listener = bind().await; + let listener = bind(0).await; // A dropped receiver means the starter gave up before the bind // completed; there is nothing left to serve. if addr_tx.send(local_addr(&listener)).is_err() { @@ -139,10 +152,10 @@ pub fn loopback_url(port: u16) -> Url { root_url(SocketAddr::from((Ipv4Addr::LOCALHOST, port))) } -async fn bind() -> TcpListener { - TcpListener::bind("127.0.0.1:0") +async fn bind(port: u16) -> TcpListener { + TcpListener::bind(("127.0.0.1", port)) .await - .expect("failed to bind test server listener") + .unwrap_or_else(|e| panic!("failed to bind test server listener on port {port}: {e}")) } fn local_addr(listener: &TcpListener) -> SocketAddr { diff --git a/tests/e2e-cucumber/src/mock_server.rs b/tests/e2e-cucumber/src/mock_server.rs index 74170ce73..71b1501e8 100644 --- a/tests/e2e-cucumber/src/mock_server.rs +++ b/tests/e2e-cucumber/src/mock_server.rs @@ -449,6 +449,35 @@ impl MockServer { Self::spawn(model_name, false).await } + /// Like [`Self::start`], but binds a caller-specified port instead of an + /// ephemeral one — for scenarios that plant a managed-service record + /// naming an exact port and need a real listener behind it, so the CLI's + /// liveness probe reports the model as listed instead of demoting the + /// record before an explicit-port collision check ever runs. + pub async fn start_on_port(model_name: &str, port: u16) -> Self { + let last_chat_request = Arc::new(Mutex::new(None)); + let chat_paths = Arc::new(Mutex::new(Vec::new())); + let state = ServerState { + model_name: model_name.to_string(), + metrics: None, + scripted_metrics: None, + last_chat_request: Arc::clone(&last_chat_request), + chat_paths: Arc::clone(&chat_paths), + }; + let app = Router::new() + .route("/v1/models", get(handle_models)) + .route("/models", get(handle_models)) + .route("/v1/chat/completions", post(handle_chat)) + .route("/chat/completions", post(handle_chat)) + .with_state(state); + Self { + server: http_server::spawn_on_port(app, port).await, + scripted: None, + last_chat_request, + chat_paths, + } + } + /// Like [`Self::start`], but also opts into a deterministic vLLM-flavoured /// `/metrics` route (see [`MetricsCounter`]) — for scenarios that exercise /// the dashboard's live generation-rate / TTFT / TPOT display against a diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index c566d24ea..7d1a1891f 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -411,27 +411,35 @@ async fn occupy_default_serve_port(world: &mut E2eWorld) { ); } -/// Plant a live managed-service record on the default serve port, with no real -/// listener or process behind it. `resolve_serve_port`'s explicit-port check -/// matches on the record's `port` field alone, before it ever probes the OS -/// (see `reserved_service_ports`'s "conservative by design" doc comment), so a -/// live record is enough to make the collision deterministic without a second -/// real server. `ServiceRecordOptions::default()`'s `supervisor_pid: 0` keeps -/// this safe for the World's teardown, which runs `rocm services stop -/// --yes` on every record left in the isolated tree: a real PID here would -/// point that stop at the test process itself (see -/// `plant_unauthenticated_lemonade_service`'s doc comment for the same -/// hazard). +/// Plant a live managed-service record on the default serve port, backed by a +/// real mock listener on that exact port. +/// +/// `ServiceRecordOptions::default()`'s `supervisor_pid: 0` means the CLI's +/// liveness refresh finds no tracked pid to check; without a real listener +/// behind the record, the endpoint probe would then come back `Unreachable` +/// and the record gets demoted to `"stopped"` *before* the explicit-port +/// collision check ever sees it (`refresh_managed_service_runtime_liveness` +/// in `apps/rocm/src/main.rs`). Backing the record with a `MockServer` on +/// `SERVE_PORT` makes `/v1/models` list `other-model`, so the probe reports +/// `Listing`/`Serving` and the record is left `"ready"` instead — the +/// scenario needs the collision detected, so the port has to be reserved for +/// real, not just on paper. `supervisor_pid: 0` still keeps this safe for the +/// World's teardown, which runs `rocm services stop --yes` on every +/// record left in the isolated tree: a real PID here would point that stop at +/// the test process itself (see `plant_unauthenticated_lemonade_service`'s +/// doc comment for the same hazard). #[given("another managed service already reserves the default serve port")] async fn reserve_default_serve_port(world: &mut E2eWorld) { let root = world.isolated_root.as_ref().expect("no isolated root"); let services = root.path().join("data").join("services"); + let mock = MockServer::start_on_port("other-model", SERVE_PORT).await; write_service_record_with( &services, "other-model", SERVE_PORT, ServiceRecordOptions::default(), ); + world.mock = Some(mock); } /// The (model, engine, ready-substring) this host should serve for an From 960d0f596e2d10594bb7255484e8922eaea43730 Mon Sep 17 00:00:00 2001 From: Juho Vainio Date: Thu, 15 Oct 2026 20:29:25 +0300 Subject: [PATCH 09/19] fix(serve): address review feedback on port-collision and lock correctness Extract lease_loopback_port_or_bind_error (rocm-core) so the CLI restart re-check and rocmd's recovery re-check share one AddrInUse-vs-other-error mapping instead of each hand-rolling it. Merge start_managed_service_dropping_orphaned_key_on_failure into start_managed_service: the wrapper had become the only call site, so the split no longer bought anything. Fix doc comments that had drifted from the code or overclaimed: prune's "only acquirer in this binary" claim ignored restart_internal_managed_service_with, restart_managed_service's recovery-lock comment implied a concurrent launch never blocks (it does, just not indefinitely), and live_managed_services' read-failure fallback doc didn't note the starting-record gap. Validate the chat tool's launch_server `port` argument against the same shape the argv builder reads, so a malformed value fails loudly instead of silently falling through to automatic port selection on a host that doesn't support it. Add test coverage for the restart rollback's second-write failure paths (unreadable corrupted content, and a rollback write that itself fails), which were previously only covered for the first-write case. Signed-off-by: Juho Vainio --- apps/rocm/src/chat.rs | 63 ++++++- apps/rocm/src/main.rs | 24 +-- apps/rocm/src/serve_cmd.rs | 8 +- apps/rocm/src/service_launch.rs | 191 ++++++++++---------- apps/rocmd/src/watchers.rs | 176 +++++++++++++++++- crates/rocm-core/src/lib.rs | 80 +-------- crates/rocm-core/src/process.rs | 235 +++++++++++++++++++++++++ crates/rocm-core/src/service_record.rs | 14 +- 8 files changed, 582 insertions(+), 209 deletions(-) diff --git a/apps/rocm/src/chat.rs b/apps/rocm/src/chat.rs index 6afd574d9..dcb57184d 100644 --- a/apps/rocm/src/chat.rs +++ b/apps/rocm/src/chat.rs @@ -2235,11 +2235,28 @@ fn validate_chat_launch_server_tool_call(call: &providers::ChatToolCall) -> Resu if !is_loopback_host(&host) { bail!("local assistant cannot request non-local host `{host}`"); } + // Validate the same way the argv builder reads it + // (`object.get("port").and_then(serde_json::Value::as_u64)`), so a + // malformed `port` (wrong type, out of range) can't slip past this check + // only to have the argv builder silently drop it and fall through to + // automatic port selection. + let port = match object.get("port") { + None => None, + Some(value) => { + let port = value + .as_u64() + .context("ROCm tool `launch_server` argument `port` must be an integer")?; + if !(1..=u64::from(u16::MAX)).contains(&port) { + bail!("ROCm tool `launch_server` argument `port` must be between 1 and 65535"); + } + Some(port) + } + }; // `rocm serve` proves a port is free only on the exact default host; every // other spelling (including the loopback aliases `localhost`/`::1` this // check still accepts) needs an explicit port or the CLI call refuses // later with a less actionable error. See `is_canonical_loopback_host`. - if object.get("port").is_none() && !rocm_core::is_canonical_loopback_host(&host) { + if port.is_none() && !rocm_core::is_canonical_loopback_host(&host) { bail!( "local assistant must supply an explicit `port` for host `{host}`; automatic port \ selection is only supported on the default host {DEFAULT_LOCAL_HOST}" @@ -6266,6 +6283,50 @@ model recipes .expect("automatic port selection still works on the canonical loopback host"); } + #[test] + fn chat_tool_call_rejects_a_malformed_port_instead_of_letting_it_fall_through() { + // A non-integer `port` must not be treated as "no port supplied" here: + // the argv builder (`object.get("port").and_then(as_u64)`) would drop + // it just as silently, so without this check a malformed port on a + // non-canonical host would sail past validation only to fall through + // to automatic port selection the host doesn't support. + let string_port = providers::ChatToolCall { + id: None, + name: "launch_server".to_owned(), + arguments: serde_json::json!({ + "model": "tiny.gguf", + "engine": "vllm", + "host": "localhost", + "port": "11999" + }), + }; + let error = validate_chat_tool_call(&string_port) + .unwrap_err() + .to_string(); + assert!( + error.contains("must be an integer"), + "unexpected error: {error}" + ); + + let out_of_range_port = providers::ChatToolCall { + id: None, + name: "launch_server".to_owned(), + arguments: serde_json::json!({ + "model": "tiny.gguf", + "engine": "vllm", + "host": "127.0.0.1", + "port": 70000 + }), + }; + let error = validate_chat_tool_call(&out_of_range_port) + .unwrap_err() + .to_string(); + assert!( + error.contains("between 1 and 65535"), + "unexpected error: {error}" + ); + } + #[test] fn chat_tool_call_rejects_bad_install_suggestions() { for (arguments, expected) in [ diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index e17f1435e..6e674fa9d 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -4635,10 +4635,11 @@ fn apply_service_prune_plan( /// window — [`select_gpu_indices_under_launch_lock`] acquires it, hands the /// guard back to its caller, and `start_managed_service`/`run_attached_service` /// only drop it once the record is persisted (see that function's own comment, -/// which this one must not drift from). The gap was simply that prune never -/// acquired it: the helper above was the only acquirer in this binary (`rocmd`'s -/// recovery path takes the same file lock too, bounded, from its own process — -/// see `restart_managed_service`'s doc comment). Taking it +/// which this one must not drift from). `restart_internal_managed_service_with` +/// acquires the same lock around its own respawn, for the same reason. The gap +/// was simply that prune never acquired it, while those two already did +/// (`rocmd`'s recovery path takes the same file lock too, bounded, from its own +/// process — see `restart_managed_service`'s doc comment). Taking it /// here closes the window deterministically, for any number of records, which no /// age floor can — the window is not short and is not bounded, because /// `spawn_managed_engine_child`'s idempotency guard runs `load_managed_services` @@ -7289,14 +7290,13 @@ fn restart_internal_managed_service_with( ); } let port_lease = if rocm_core::is_canonical_loopback_host(&stopped_record.host) { - Some( - rocm_core::lease_loopback_port(stopped_record.port).with_context(|| { - format!( - "cannot restart `{service_id}` on {}:{}: its port is in use", - stopped_record.host, stopped_record.port - ) - })?, - ) + Some(rocm_core::lease_loopback_port_or_bind_error( + stopped_record.port, + format_args!( + "cannot restart `{service_id}` on {}:{}: its port is in use", + stopped_record.host, stopped_record.port + ), + )?) } else { None }; diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index 07bc91852..6f496faa0 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -14,8 +14,7 @@ //! `collect_serve_notes` and `validate_pinned_gpu_index` stay in `main.rs` //! (no entanglement; a later pass can relocate them). The //! managed-service-spawning tail this file calls into — -//! `start_managed_service_dropping_orphaned_key_on_failure`, -//! `run_attached_service`, `resolve_endpoint_auth`, +//! `start_managed_service`, `run_attached_service`, `resolve_endpoint_auth`, //! `ensure_public_bind_engine_supported`, //! `drop_orphaned_endpoint_key_on_already_running`, `print_managed_launch_plain`, //! `validate_bind_host`, `ensure_background_helper_running_quiet` — was relocated @@ -44,8 +43,7 @@ use crate::serve_summary; use crate::service_launch::{ drop_orphaned_endpoint_key_on_already_running, ensure_background_helper_running_quiet, ensure_public_bind_engine_supported, print_managed_launch_plain, resolve_endpoint_auth, - run_attached_service, start_managed_service_dropping_orphaned_key_on_failure, - validate_bind_host, + run_attached_service, start_managed_service, validate_bind_host, }; use crate::{ cli_progress, collect_serve_notes, detect_gpu_count, device_policy_name, engine_request, @@ -864,7 +862,7 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { let mut spinner = cli_progress::Spinner::new(format!("Starting {model} on {selected_engine}…")); spinner.tick(); - let report = start_managed_service_dropping_orphaned_key_on_failure( + let report = start_managed_service( &paths, &selected_engine, &service_id, diff --git a/apps/rocm/src/service_launch.rs b/apps/rocm/src/service_launch.rs index c46a0e531..6d4210c61 100644 --- a/apps/rocm/src/service_launch.rs +++ b/apps/rocm/src/service_launch.rs @@ -329,9 +329,17 @@ enum ManagedPortDecision { /// (dead PIDs demote to "stopped" and therefore neither satisfy the duplicate /// check nor reserve a port). /// -/// A read failure degrades to "nothing is live": the OS preflight still guards -/// the port, and refusing to serve because the services directory could not be -/// listed would be a worse failure than launching. +/// A read failure degrades to "nothing is live": for an already-`ready` +/// service the OS preflight still guards its port (the engine holds it), and +/// refusing to serve because the services directory could not be listed would +/// be a worse failure than launching. That guard does not cover a `starting` +/// record: its port is released back to the OS at the end of +/// [`spawn_managed_engine_child`] (see `lease.release()`) before the new +/// engine child has actually bound it, so a concurrent launch that hits this +/// read failure in that narrow window can probe the same port successfully. +/// Accepted as a known gap — narrower than the general services-dir read +/// failure case, and only reachable by a second, near-simultaneous I/O +/// failure on top of one already in flight. fn live_managed_services(paths: &AppPaths) -> Vec { load_managed_services(paths) .unwrap_or_default() @@ -769,6 +777,14 @@ fn launch_managed_engine_child_locked( Ok(child_pid) } +/// Drops the endpoint key `serve()` freshly staged for this attempt if the +/// launch fails before any `ManagedServiceRecord` exists to own it (port +/// reserved/in use, range exhausted, other bind error) — such a failure +/// returns before `abandon_failed_managed_launch` ever runs, so nothing else +/// would clear it. Scoped to the spawn call only, mirroring +/// [`run_attached_service`]'s identical guard: a later failure (the readiness +/// wait, `record.write()`) leaves a record a live child actually owns, so it +/// must not also wipe that key. #[allow(clippy::too_many_arguments)] pub(crate) fn start_managed_service( paths: &AppPaths, @@ -802,7 +818,10 @@ pub(crate) fn start_managed_service( env_id, engine_recipe, require_api_key, - )? { + ) + .inspect_err(|_| { + drop_orphaned_endpoint_key_on_already_running(paths, service_id, endpoint_api_key); + })? { ManagedSpawn::AlreadyRunning(report) => return Ok(report), ManagedSpawn::Spawned { record, child_pid } => (*record, child_pid), }; @@ -858,58 +877,6 @@ pub(crate) fn start_managed_service( }) } -/// As [`start_managed_service`], but also drops the endpoint key `serve()` -/// freshly staged for this attempt if the launch fails before any -/// `ManagedServiceRecord` exists to own it (port reserved/in use, range -/// exhausted, other bind error) — such a failure returns before -/// `abandon_failed_managed_launch` ever runs, so nothing else would clear it. -/// Idempotent with that post-spawn cleanup. This is the real call site -/// `serve_cmd.rs`'s background launch path uses, so it is what -/// `failed_port_reservation_leaves_no_orphaned_endpoint_key` drives directly, -/// mirroring how [`run_attached_service`] is driven directly for the -/// foreground path. -#[allow(clippy::too_many_arguments)] -pub(crate) fn start_managed_service_dropping_orphaned_key_on_failure( - paths: &AppPaths, - engine: &str, - service_id: &str, - requested_model: &str, - resolve: &ResolveModelResponse, - host: &str, - port_request: serve_port::PortRequest, - device_policy: &DevicePolicy, - gpu_indices: &[u32], - runtime_id: Option<&str>, - env_id: Option<&str>, - engine_recipe: Option<&EngineRecipeHint>, - endpoint_api_key: Option<&str>, - launch_lock: rocm_core::FileLock, - require_api_key: bool, - on_wait_tick: &mut dyn FnMut(Duration), -) -> Result { - start_managed_service( - paths, - engine, - service_id, - requested_model, - resolve, - host, - port_request, - device_policy, - gpu_indices, - runtime_id, - env_id, - engine_recipe, - endpoint_api_key, - launch_lock, - require_api_key, - on_wait_tick, - ) - .inspect_err(|_| { - drop_orphaned_endpoint_key_on_already_running(paths, service_id, endpoint_api_key); - }) -} - /// Reproduce the original plain, line-by-line managed-launch output. Used for /// non-interactive output (piped, CI, the chat assistant's `serve --managed`), /// where the animated summary is inappropriate. The interactive path renders the @@ -1793,50 +1760,70 @@ mod tests { // `record.engine_pid` — so cleanup must terminate (already-dead) the // child, keep the record and log for inspection, mark the record // `failed`, and clear only the endpoint key. - let (root, paths) = test_paths("managed-spawn-unknown-engine-cleanup"); - paths.ensure()?; - let service_id = "no-such-engine-qwen-4000"; - endpoint_keys::store_endpoint_api_key(&paths, service_id, "test-key")?; + // + // The child always dies near-instantly, but `spawn_managed_engine_child` + // only reports that within a fixed startup-check window. On a loaded or + // antivirus-scanning Windows CI box, a fresh debug `rocm.exe` can + // occasionally be slow enough to start that the window is missed even + // though the child is already doomed — so poll by retrying a fresh + // attempt (its own service id/port, so no stale manifest collides) up + // to a deadline instead of asserting on one race-prone attempt. + let deadline = std::time::Instant::now() + Duration::from_secs(10); + let mut attempt: u16 = 0; + let (root, paths, service_id, result) = loop { + attempt += 1; + let (root, paths) = + test_paths(&format!("managed-spawn-unknown-engine-cleanup-{attempt}")); + paths.ensure()?; + let service_id = format!("no-such-engine-qwen-{}", 4000 + attempt); + endpoint_keys::store_endpoint_api_key(&paths, &service_id, "test-key")?; + + let resolve = ResolveModelResponse { + canonical_model_id: "qwen-canonical".to_owned(), + task: "chat".to_owned(), + source: "hf".to_owned(), + revision: "main".to_owned(), + loader: "llama.cpp".to_owned(), + trust_remote_code: false, + chat_template_mode: "auto".to_owned(), + dtype: "auto".to_owned(), + device_policy: DevicePolicy::GpuPreferred, + estimated_memory: "unknown".to_owned(), + launch_defaults: serde_json::json!({}), + engine_recipe: None, + warnings: Vec::new(), + }; + + let result = spawn_managed_engine_child( + &paths, + "no-such-engine", + &service_id, + "qwen", + &resolve, + "127.0.0.1", + serve_port::PortRequest::Explicit(11_514 + attempt), + &resolve.device_policy, + &[], + None, + None, + None, + false, + ); - let resolve = ResolveModelResponse { - canonical_model_id: "qwen-canonical".to_owned(), - task: "chat".to_owned(), - source: "hf".to_owned(), - revision: "main".to_owned(), - loader: "llama.cpp".to_owned(), - trust_remote_code: false, - chat_template_mode: "auto".to_owned(), - dtype: "auto".to_owned(), - device_policy: DevicePolicy::GpuPreferred, - estimated_memory: "unknown".to_owned(), - launch_defaults: serde_json::json!({}), - engine_recipe: None, - warnings: Vec::new(), + if result.is_ok() && std::time::Instant::now() < deadline { + let _ = fs::remove_dir_all(&root); + continue; + } + break (root, paths, service_id, result); }; - let result = spawn_managed_engine_child( - &paths, - "no-such-engine", - service_id, - "qwen", - &resolve, - "127.0.0.1", - serve_port::PortRequest::Explicit(11_514), - &resolve.device_policy, - &[], - None, - None, - None, - false, - ); - - let manifest_path = paths.service_manifest_path(service_id); - let log_path = paths.service_log_path(service_id); + let manifest_path = paths.service_manifest_path(&service_id); + let log_path = paths.service_log_path(&service_id); let persisted = load_managed_services(&paths) .expect("re-read manifests") .into_iter() .find(|candidate| candidate.service_id == service_id); - let key = endpoint_keys::endpoint_api_key(&paths, service_id); + let key = endpoint_keys::endpoint_api_key(&paths, &service_id); let live = live_managed_services(&paths); let Err(error) = result else { @@ -1876,14 +1863,14 @@ mod tests { #[test] fn failed_port_reservation_leaves_no_orphaned_endpoint_key() -> Result<()> { // serve() stores the new attempt's endpoint key before calling - // start_managed_service_dropping_orphaned_key_on_failure. If - // spawn_managed_engine_child fails inside resolve_managed_port_in_transaction - // (explicit port already reserved by another live service), it returns - // before any ManagedServiceRecord exists for this attempt, so - // abandon_failed_managed_launch never runs. Driving the real wrapper - // function end to end (not a simulated call to - // drop_orphaned_endpoint_key_on_already_running) proves serve_cmd.rs's own - // call site actually wires the cleanup, mirroring how + // start_managed_service. If spawn_managed_engine_child fails inside + // resolve_managed_port_in_transaction (explicit port already reserved + // by another live service), it returns before any + // ManagedServiceRecord exists for this attempt, so + // abandon_failed_managed_launch never runs. Driving the real function + // end to end (not a simulated call to + // drop_orphaned_endpoint_key_on_already_running) proves serve_cmd.rs's + // own call site actually wires the cleanup, mirroring how // attached_launch_failing_before_spawn_leaves_no_orphaned_endpoint_key // drives run_attached_service for the foreground path. let (root, paths) = test_paths("orphaned-key-on-reserved-port"); @@ -1926,7 +1913,7 @@ mod tests { endpoint_keys::store_endpoint_api_key(&paths, service_id, "test-key")?; let launch_lock = rocm_core::FileLock::acquire(paths.managed_launch_lock_path())?; - let result = start_managed_service_dropping_orphaned_key_on_failure( + let result = start_managed_service( &paths, "lemonade", service_id, diff --git a/apps/rocmd/src/watchers.rs b/apps/rocmd/src/watchers.rs index dbd54d23b..50c6a06c4 100644 --- a/apps/rocmd/src/watchers.rs +++ b/apps/rocmd/src/watchers.rs @@ -1474,6 +1474,19 @@ fn manifest_service_recovery_reason( } } +/// Respawn the supervisor for a managed service that needs recovery. +/// +/// Recovery re-claims the concrete port already written in the record under +/// [`rocm_core::MANAGED_LAUNCH_LOCK_TIMEOUT`]-bounded acquisition of the same +/// cross-process launch lock a `rocm serve` launch holds (unbounded, via +/// [`rocm_core::FileLock::acquire`]). The two still can't publish competing +/// records for this port at the same instant — the lock is exclusive either +/// way. A concurrent `rocm serve` launch does still block for as long as +/// recovery holds the lock; what the bound buys is the other direction — +/// recovery never waits out an interactive launch indefinitely. If a `rocm +/// serve` call is already mid-transaction when this bounded acquire times +/// out, recovery is deferred (not retried in a busy loop) rather than +/// queuing behind the CLI. pub(crate) fn restart_managed_service( paths: &AppPaths, record: &mut ManagedServiceRecord, @@ -1510,14 +1523,13 @@ pub(crate) fn restart_managed_service( // the existing engine-owned bind behavior — the record transition is still // serialized, but the bind is not preflighted. let lease = if rocm_core::is_canonical_loopback_host(&record.host) { - Some( - rocm_core::lease_loopback_port(record.port).with_context(|| { - format!( - "cannot recover managed service {} on {}:{}: its recorded port is in use", - record.service_id, record.host, record.port - ) - })?, - ) + Some(rocm_core::lease_loopback_port_or_bind_error( + record.port, + format_args!( + "cannot recover managed service {} on {}:{}: its recorded port is in use", + record.service_id, record.host, record.port + ), + )?) } else { None }; @@ -3878,6 +3890,154 @@ mod tests { Ok(()) } + /// The first-write rollback above covers a failure before anything is + /// spawned; this one covers the second write, after the supervisor exists. + /// A thread races the restart: as soon as the first write lands + /// (`recovering`, no supervisor yet), it corrupts the manifest's content so + /// `record_spawned_supervisor`'s read fails. That must kill the spawned + /// supervisor and restore the prior record — and the rollback write itself + /// must still land, since corrupting *content* leaves the file a writable + /// regular file. + #[test] + fn restart_rolls_back_and_kills_the_supervisor_when_the_second_write_cannot_be_read() + -> Result<()> { + let (root, paths) = temp_app_paths("restart-rollback-second-write-unreadable"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let service_id = "svc-restart-rollback-second-write-unreadable"; + let mut record = identity_probe_record(&paths, service_id, 11461); + record.status = "failed".to_owned(); + record.write()?; + let prior = record.clone(); + + let manifest_path = paths.service_manifest_path(service_id); + let corrupt = thread::spawn(move || -> bool { + let deadline = std::time::Instant::now() + Duration::from_secs(10); + while std::time::Instant::now() < deadline { + if let Ok(bytes) = fs::read(&manifest_path) + && let Ok(on_disk) = serde_json::from_slice::(&bytes) + && on_disk.status == "recovering" + && on_disk.supervisor_pid == 0 + { + let _ = fs::write(&manifest_path, b"not json"); + return true; + } + // Busy-spin: the window this races (between the restart's + // first write and its spawned child's own write) is only as + // long as a `fork`+`exec`, far shorter than any sleep tick + // could reliably catch. + std::hint::spin_loop(); + } + false + }); + let outcome = restart_managed_service(&paths, &mut record); + let corrupted_in_time = corrupt.join().expect("corrupt thread"); + let persisted = load_service_record(&paths, service_id); + fs::remove_dir_all(root).ok(); + + assert!( + corrupted_in_time, + "precondition: the race won before the second write ran" + ); + let error = outcome.expect_err("an unreadable second write must fail the restart"); + let message = format!("{error:#}"); + assert!( + message.contains("failed to publish the recovering service record"), + "{message}" + ); + assert!( + !message.contains("rollback also failed"), + "corrupting content, not permissions, leaves the rollback write itself able to \ + land: {message}" + ); + assert_eq!( + record.status, prior.status, + "the in-memory record must be restored to what it was before this restart" + ); + let persisted = persisted?; + assert_eq!( + persisted.status, prior.status, + "the rollback write must overwrite the corrupted content with the prior record" + ); + assert_eq!(persisted.supervisor_pid, prior.supervisor_pid); + Ok(()) + } + + /// As above, but the rollback write itself also fails: a manifest made + /// read-only after the first write still lets `record_spawned_supervisor` + /// *read* it (ownership check passes), but both its write and the + /// subsequent rollback write hit the same permission error. That must be + /// reported as a rollback failure, not swallowed as the original one. + /// + /// Unix only and skipped as root, where a read-only mode is not enforced. + #[cfg(unix)] + #[test] + fn restart_reports_when_the_rollback_write_also_fails() -> Result<()> { + use std::os::unix::fs::PermissionsExt; + + if rocm_core::openmpi::running_as_root() { + return Ok(()); + } + + let (root, paths) = temp_app_paths("restart-rollback-also-fails"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let service_id = "svc-restart-rollback-also-fails"; + let mut record = identity_probe_record(&paths, service_id, 11462); + record.status = "failed".to_owned(); + record.write()?; + + let manifest_path = paths.service_manifest_path(service_id); + let lock = { + let manifest_path = manifest_path.clone(); + thread::spawn(move || -> bool { + let deadline = std::time::Instant::now() + Duration::from_secs(10); + while std::time::Instant::now() < deadline { + if let Ok(bytes) = fs::read(&manifest_path) + && let Ok(on_disk) = serde_json::from_slice::(&bytes) + && on_disk.status == "recovering" + && on_disk.supervisor_pid == 0 + && fs::set_permissions( + &manifest_path, + std::fs::Permissions::from_mode(0o444), + ) + .is_ok() + { + return true; + } + std::hint::spin_loop(); + } + false + }) + }; + let outcome = restart_managed_service(&paths, &mut record); + let locked_in_time = lock.join().expect("lock thread"); + fs::set_permissions(&manifest_path, std::fs::Permissions::from_mode(0o644))?; + let persisted = load_service_record(&paths, service_id); + fs::remove_dir_all(root).ok(); + + assert!( + locked_in_time, + "precondition: the race won before the second write ran" + ); + let error = outcome.expect_err("a read-only manifest must fail both writes"); + let message = format!("{error:#}"); + assert!( + message.contains("rollback also failed"), + "both the second write and its rollback hit the same permission error: {message}" + ); + assert!( + message.contains(service_id), + "the rollback-also-failed message must name the affected service: {message}" + ); + // Left exactly as the second write's failure found it: `recovering` + // with no supervisor, matching the "no supervisor" the error names. + assert_eq!(persisted?.status, "recovering"); + Ok(()) + } + /// The lease branch above proves a bound loopback port refuses recovery; /// the doc comment on `restart_managed_service`'s lease also claims custom /// hosts skip that preflight entirely, relying on the engine's own bind diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index eae5b1855..7cb928a0d 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -143,8 +143,8 @@ pub use proc_lifecycle::{ }; pub use process::{ FileLock, LoopbackPortLease, MANAGED_LAUNCH_LOCK_TIMEOUT, detach_command_session, - is_canonical_loopback_host, lease_loopback_port, process_is_running, terminate_process, - terminate_process_tree, + is_canonical_loopback_host, lease_loopback_port, lease_loopback_port_or_bind_error, + process_is_running, terminate_process, terminate_process_tree, }; #[cfg(windows)] pub use process::{ @@ -632,7 +632,6 @@ mod tests { use crate::test_support::workspace_test_artifact_dir; use std::fs; - use std::net::TcpListener; use std::path::Path; use std::process::{Command, Stdio}; @@ -902,7 +901,6 @@ mod tests { const LOCK_CHILD_PATH_ENV: &str = "ROCM_CORE_TEST_LOCK_PATH"; const LOCK_CHILD_READY_ENV: &str = "ROCM_CORE_TEST_LOCK_READY"; - const HANDOFF_PORT_ENV: &str = "ROCM_CORE_TEST_HANDOFF_PORT"; fn lock_test_dir(label: &str) -> PathBuf { let dir = workspace_test_artifact_dir().join(format!( @@ -935,26 +933,6 @@ mod tests { command.spawn().expect("spawn helper test process") } - #[test] - fn canonical_loopback_host_is_only_the_exact_default_address() { - assert!(is_canonical_loopback_host(DEFAULT_LOCAL_HOST)); - assert!(is_canonical_loopback_host(" 127.0.0.1 ")); - // Everything else is a "custom host": explicit port, engine-owned bind. - for host in [ - "localhost", - "::1", - "0.0.0.0", - "127.0.0.2", - "192.168.1.4", - "", - ] { - assert!( - !is_canonical_loopback_host(host), - "{host} must not be treated as the canonical loopback host" - ); - } - } - #[test] fn allocation_lock_path_lives_in_services_dir_and_is_not_a_manifest() { let paths = AppPaths { @@ -1043,58 +1021,4 @@ mod tests { .expect("the OS releases the lock when the holding process exits"); } - #[test] - fn loopback_lease_blocks_a_second_bind_until_released() { - let ephemeral = TcpListener::bind(("127.0.0.1", 0)).expect("ephemeral bind"); - let port = ephemeral.local_addr().expect("local addr").port(); - drop(ephemeral); - - let lease = lease_loopback_port(port).expect("free port leases"); - assert_eq!(lease.port(), port); - assert!(lease.is_held()); - let conflict = lease_loopback_port(port).expect_err("a held port must not lease twice"); - assert_eq!(conflict.kind(), std::io::ErrorKind::AddrInUse); - - assert_eq!(lease.release(), port); - lease_loopback_port(port).expect("the port is bindable again after release"); - } - - #[test] - fn engine_owned_lease_holds_no_socket() { - // Custom hosts get a port carrier, never a loopback probe. - let lease = LoopbackPortLease::engine_owned(8000); - assert_eq!(lease.port(), 8000); - assert!(!lease.is_held()); - } - - #[test] - #[ignore = "spawned as a child process by released_loopback_lease_is_immediately_bindable_by_another_process"] - fn loopback_handoff_child() { - let Ok(port) = std::env::var(HANDOFF_PORT_ENV) else { - return; - }; - let port: u16 = port.parse().expect("handoff port"); - let lease = lease_loopback_port(port).expect("child binds the released port"); - assert_eq!(lease.port(), port); - } - - #[test] - fn released_loopback_lease_is_immediately_bindable_by_another_process() { - let ephemeral = TcpListener::bind(("127.0.0.1", 0)).expect("ephemeral bind"); - let port = ephemeral.local_addr().expect("local addr").port(); - drop(ephemeral); - let lease = lease_loopback_port(port).expect("parent leases the port"); - assert_eq!(lease.release(), port); - - let status = spawn_helper_test( - "tests::loopback_handoff_child", - &[(HANDOFF_PORT_ENV, port.to_string())], - ) - .wait() - .expect("handoff child exits"); - assert!( - status.success(), - "a separate process must bind the port immediately after the lease is released" - ); - } } diff --git a/crates/rocm-core/src/process.rs b/crates/rocm-core/src/process.rs index 809016c02..4746920ce 100644 --- a/crates/rocm-core/src/process.rs +++ b/crates/rocm-core/src/process.rs @@ -582,6 +582,40 @@ pub fn lease_loopback_port(port: u16) -> std::io::Result { }) } +/// [`lease_loopback_port`], but mapped to an `anyhow::Error`. +/// +/// Only uses `in_use_message` for a genuine collision +/// ([`std::io::ErrorKind::AddrInUse`]). Any other bind failure (permission +/// denied, address unavailable, ...) gets its own generic message instead — +/// mislabelling those as "in use" sends the user to stop a service that +/// isn't the actual problem. Mirrors `rocm serve`'s own +/// `explicit_bind_error`; shared by the CLI's restart re-check and +/// `rocmd`'s recovery re-check so neither duplicates (or drifts from) that +/// distinction. +pub fn lease_loopback_port_or_bind_error( + port: u16, + in_use_message: impl std::fmt::Display, +) -> anyhow::Result { + lease_loopback_port(port).map_err(|error| bind_error(port, &error, in_use_message)) +} + +/// Pure error-mapping half of [`lease_loopback_port_or_bind_error`], split out +/// so the `AddrInUse`-vs-other-error distinction is tested against a +/// synthetic `io::Error` rather than needing a real bind failure (a +/// privileged port, an unavailable address) that is not reliably +/// reproducible in CI. +fn bind_error( + port: u16, + error: &std::io::Error, + in_use_message: impl std::fmt::Display, +) -> anyhow::Error { + if error.kind() == std::io::ErrorKind::AddrInUse { + anyhow::anyhow!("{in_use_message}") + } else { + anyhow::anyhow!("failed to reserve port {port}: {error}") + } +} + impl Drop for FileLock { fn drop(&mut self) { // Best-effort: an unlock failure only means the OS releases it slightly @@ -1168,6 +1202,207 @@ fn assemble_output( #[cfg(test)] mod tests { use super::*; + use crate::test_support::workspace_test_artifact_dir; + use std::process::Stdio; + + const LOCK_CHILD_PATH_ENV: &str = "ROCM_CORE_TEST_LOCK_PATH"; + const LOCK_CHILD_READY_ENV: &str = "ROCM_CORE_TEST_LOCK_READY"; + const HANDOFF_PORT_ENV: &str = "ROCM_CORE_TEST_HANDOFF_PORT"; + + fn lock_test_dir(label: &str) -> PathBuf { + let dir = workspace_test_artifact_dir().join(format!( + "{label}-{}-{}", + std::process::id(), + crate::unix_time_millis() + )); + fs::create_dir_all(&dir).expect("create lock test dir"); + dir + } + + /// Re-run this test binary, executing exactly one `#[ignore]`d helper test in + /// a genuinely separate process. Cross-process locking and socket handoff + /// cannot be proven inside one process. + fn spawn_helper_test(name: &str, envs: &[(&str, String)]) -> std::process::Child { + let exe = std::env::current_exe().expect("test binary path"); + let mut command = Command::new(exe); + command + .arg(name) + .arg("--exact") + .arg("--ignored") + .arg("--test-threads") + .arg("1") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + for (key, value) in envs { + command.env(key, value); + } + command.spawn().expect("spawn helper test process") + } + + #[test] + fn canonical_loopback_host_is_only_the_exact_default_address() { + assert!(is_canonical_loopback_host(DEFAULT_LOCAL_HOST)); + assert!(is_canonical_loopback_host(" 127.0.0.1 ")); + // Everything else is a "custom host": explicit port, engine-owned bind. + for host in [ + "localhost", + "::1", + "0.0.0.0", + "127.0.0.2", + "192.168.1.4", + "", + ] { + assert!( + !is_canonical_loopback_host(host), + "{host} must not be treated as the canonical loopback host" + ); + } + } + + #[test] + fn bounded_file_lock_excludes_independent_handles_and_fails_closed_on_timeout() { + let path = lock_test_dir("core-bounded-lock-exclusive").join("launch.lock"); + let held = FileLock::acquire_bounded(&path, Duration::from_secs(5)).expect("first holder"); + assert_eq!(held.path(), path.as_path()); + + // A second, independent file handle must contend on the OS lock rather + // than sail through, and must give up with an actionable error. + let started = Instant::now(); + let error = FileLock::acquire_bounded(&path, Duration::from_millis(200)) + .expect_err("a held lock must not be acquired twice"); + assert!( + started.elapsed() >= Duration::from_millis(200), + "waiter returned before its deadline" + ); + let message = format!("{error:#}"); + assert!(message.contains("timed out"), "{message}"); + + drop(held); + FileLock::acquire_bounded(&path, Duration::from_secs(5)) + .expect("lock is free once the guard is dropped"); + } + + #[test] + #[ignore = "spawned as a child process by bounded_file_lock_is_released_when_the_holder_process_exits"] + fn bounded_file_lock_child_holder() { + let Ok(path) = std::env::var(LOCK_CHILD_PATH_ENV) else { + return; + }; + let ready = std::env::var(LOCK_CHILD_READY_ENV).expect("ready-file path"); + let guard = FileLock::acquire_bounded(Path::new(&path), Duration::from_secs(10)) + .expect("child acquires the shared lock"); + fs::write(&ready, "held").expect("signal that the child holds the lock"); + thread::sleep(Duration::from_millis(300)); + // Leak the guard so the lock is released by process exit alone — that is + // the property a crashed launcher depends on. + std::mem::forget(guard); + } + + #[test] + fn bounded_file_lock_is_released_when_the_holder_process_exits() { + let dir = lock_test_dir("core-bounded-lock-crossproc"); + let path = dir.join("launch.lock"); + let ready = dir.join("child-ready"); + let mut child = spawn_helper_test( + "process::tests::bounded_file_lock_child_holder", + &[ + (LOCK_CHILD_PATH_ENV, path.display().to_string()), + (LOCK_CHILD_READY_ENV, ready.display().to_string()), + ], + ); + + let deadline = Instant::now() + Duration::from_mins(1); + while !ready.exists() { + assert!( + Instant::now() < deadline, + "child never reported holding the lock" + ); + thread::sleep(Duration::from_millis(25)); + } + + let busy = FileLock::acquire_bounded(&path, Duration::from_millis(50)) + .expect_err("the lock must be exclusive across processes"); + assert!(format!("{busy:#}").contains("timed out")); + + let status = child.wait().expect("child holder exits"); + assert!(status.success(), "child holder failed: {status}"); + FileLock::acquire_bounded(&path, Duration::from_secs(10)) + .expect("the OS releases the lock when the holding process exits"); + } + + #[test] + fn loopback_lease_blocks_a_second_bind_until_released() { + let ephemeral = TcpListener::bind(("127.0.0.1", 0)).expect("ephemeral bind"); + let port = ephemeral.local_addr().expect("local addr").port(); + drop(ephemeral); + + let lease = lease_loopback_port(port).expect("free port leases"); + assert_eq!(lease.port(), port); + assert!(lease.is_held()); + let conflict = lease_loopback_port(port).expect_err("a held port must not lease twice"); + assert_eq!(conflict.kind(), std::io::ErrorKind::AddrInUse); + + assert_eq!(lease.release(), port); + lease_loopback_port(port).expect("the port is bindable again after release"); + } + + #[test] + fn bind_error_labels_only_a_genuine_collision_as_in_use() { + let in_use = bind_error( + 9000, + &std::io::Error::from(std::io::ErrorKind::AddrInUse), + "its port is in use", + ); + assert_eq!(in_use.to_string(), "its port is in use"); + + let other = bind_error( + 9000, + &std::io::Error::from(std::io::ErrorKind::PermissionDenied), + "its port is in use", + ); + assert_ne!(other.to_string(), "its port is in use"); + assert!(other.to_string().contains("9000")); + } + + #[test] + fn engine_owned_lease_holds_no_socket() { + // Custom hosts get a port carrier, never a loopback probe. + let lease = LoopbackPortLease::engine_owned(8000); + assert_eq!(lease.port(), 8000); + assert!(!lease.is_held()); + } + + #[test] + #[ignore = "spawned as a child process by released_loopback_lease_is_immediately_bindable_by_another_process"] + fn loopback_handoff_child() { + let Ok(port) = std::env::var(HANDOFF_PORT_ENV) else { + return; + }; + let port: u16 = port.parse().expect("handoff port"); + let lease = lease_loopback_port(port).expect("child binds the released port"); + assert_eq!(lease.port(), port); + } + + #[test] + fn released_loopback_lease_is_immediately_bindable_by_another_process() { + let ephemeral = TcpListener::bind(("127.0.0.1", 0)).expect("ephemeral bind"); + let port = ephemeral.local_addr().expect("local addr").port(); + drop(ephemeral); + let lease = lease_loopback_port(port).expect("parent leases the port"); + assert_eq!(lease.release(), port); + + let status = spawn_helper_test( + "process::tests::loopback_handoff_child", + &[(HANDOFF_PORT_ENV, port.to_string())], + ) + .wait() + .expect("handoff child exits"); + assert!( + status.success(), + "a separate process must bind the port immediately after the lease is released" + ); + } #[test] fn file_lock_creates_missing_parent_dirs_and_lock_file() { diff --git a/crates/rocm-core/src/service_record.rs b/crates/rocm-core/src/service_record.rs index 2516fa9dd..3171683c7 100644 --- a/crates/rocm-core/src/service_record.rs +++ b/crates/rocm-core/src/service_record.rs @@ -324,7 +324,14 @@ impl ManagedServiceRecord { /// Whether this record still reserves its port: the sole definition of /// "live" shared between the CLI's launch-time port check and `rocmd`'s - /// recovery, so the two can never disagree about who owns a port. + /// recovery. Same predicate, not same data: the CLI's `load_managed_services` + /// refreshes each record's PID/endpoint liveness before this runs, while + /// `rocmd::persistence::load_managed_services` reads the file as-is. A + /// record whose process has died since its last write can therefore still + /// read `is_live()` to `rocmd` after the CLI would have refreshed it to + /// `"stopped"`. That fails safe — recovery defers rather than racing a + /// collision — but it can stall on a stale record until something else + /// rewrites it. #[must_use] pub fn is_live(&self) -> bool { matches!( @@ -334,8 +341,9 @@ impl ManagedServiceRecord { } /// Whether this live record reserves `port` — the sole collision check - /// shared by the CLI's restart re-check and `rocmd`'s recovery re-check, so - /// the two can never disagree about who owns a port. + /// shared by the CLI's restart re-check and `rocmd`'s recovery re-check. + /// See [`Self::is_live`]'s doc for the data-freshness gap between the two + /// callers. /// /// Deliberately host-blind: a `0.0.0.0` (or `localhost`) listener also /// blocks the loopback address a competing bind would target, so comparing From 17e931d65d87a45d0b1269af398f445a9ed1e7b4 Mon Sep 17 00:00:00 2001 From: Juho Vainio Date: Thu, 15 Oct 2026 20:30:16 +0300 Subject: [PATCH 10/19] fix(dash-tui): flag a broken hidden host in the serve wizard summary advanced_summary()'s doc comment promised a "needs attention" summary for any broken hidden value, but only a bad port actually produced one. Mirror validate()'s host-invalidity rule (empty, or not a recognized loopback spelling) so a customized-but-collapsed panel with a bad host says so too, instead of silently reading "Customized". Signed-off-by: Juho Vainio --- crates/rocm-dash-tui/src/ui/serve_wizard.rs | 144 ++++++++++++++++++-- 1 file changed, 132 insertions(+), 12 deletions(-) diff --git a/crates/rocm-dash-tui/src/ui/serve_wizard.rs b/crates/rocm-dash-tui/src/ui/serve_wizard.rs index 496503db6..8d24b390e 100644 --- a/crates/rocm-dash-tui/src/ui/serve_wizard.rs +++ b/crates/rocm-dash-tui/src/ui/serve_wizard.rs @@ -299,7 +299,8 @@ impl ServeWizardState { } fn cycle(&mut self, delta: isize) { - match self.current_field() { + let touched = self.current_field(); + match touched { // Advanced is a choice row too: ← collapses, → expands. Only // reachable while Advanced owns focus, and Advanced sits at the // same index in both orders, so focus is preserved either way. @@ -317,7 +318,7 @@ impl ServeWizardState { // Cycling Port back to Custom can fix the exact complaint an outstanding // message is naming (e.g. "set Port to Custom"); without this, that // message would sit on screen after the toggle that resolved it. - refresh_message(self); + restate_or_clear(self, touched); } /// Direct typing only applies to the Model row; Host and the custom port @@ -353,7 +354,12 @@ impl ServeWizardState { if self.port_mode == PortMode::Custom && parse_port(&self.port).is_none() { "Customized · port needs attention" } else { - "Customized" + let host = self.host.trim(); + if host.is_empty() || !is_loopback_host(host) { + "Customized · host needs attention" + } else { + "Customized" + } } } @@ -579,7 +585,7 @@ fn editor_key(w: &mut ServeWizardState, code: KeyCode) { // The form's host/port are only written on Enter, so discarding // the editor here never mutates them. w.editor = None; - refresh_message(w); + recompute_message(w); } KeyCode::Left | KeyCode::Right => { let delta = if code == KeyCode::Left { -1 } else { 1 }; @@ -601,9 +607,15 @@ fn editor_key(w: &mut ServeWizardState, code: KeyCode) { let trimmed = value.trim(); if trimmed.is_empty() { // Keep the editor open with a plain fix rather than - // silently accepting an unusable host. + // silently accepting an unusable host. Esc discards + // the edit and leaves `w.host` as it was before this + // edit started, which is not always `DEFAULT_HOST` — + // the user may be re-editing an already-custom host. + // Name the value Esc actually restores, not a + // hardcoded default that may not match it. + let current_host = w.host.clone(); w.message = Some(format!( - "host cannot be empty; type an address or press Esc to restore {DEFAULT_HOST}" + "host cannot be empty; type an address or press Esc to restore {current_host}" )); return; } @@ -625,7 +637,7 @@ fn editor_key(w: &mut ServeWizardState, code: KeyCode) { } _ => {} } - refresh_message(w); + restate_or_clear(w, field); w.editor = None; } KeyCode::Char(c) => { @@ -640,12 +652,34 @@ fn editor_key(w: &mut ServeWizardState, code: KeyCode) { } } -/// Re-evaluate an *outstanding* validation complaint against the current form: -/// a genuine fix clears it, a still-broken form re-states the reason. Silent -/// when nothing has been complained about yet, so ordinary edits stay quiet. -fn refresh_message(w: &mut ServeWizardState) { +/// Esc discards an edit without ever writing it to the form, so whatever +/// `validate()` says now was equally true before the editor opened. Safe to +/// unconditionally re-state (and re-focus, if the complaint has moved) since +/// nothing new was introduced. +fn recompute_message(w: &mut ServeWizardState) { + if w.message.is_some() { + match w.validate() { + Ok(_) => w.message = None, + Err((field, msg)) => { + w.focus(field); + w.message = Some(msg); + } + } + } +} + +/// An actual edit to `touched` (a committed Enter, or a Port-mode cycle) can +/// only confirm or deny a complaint about `touched` itself. A complaint about +/// some other, untouched field would be a surprise the user never triggered +/// (e.g. fixing a blank host surfacing "model is required"), so clear it +/// silently instead; `request_launch` still surfaces it if they try to launch. +fn restate_or_clear(w: &mut ServeWizardState, touched: Field) { if w.message.is_some() { - w.message = w.validate().err().map(|(_, msg)| msg); + w.message = match w.validate() { + Ok(_) => None, + Err((field, msg)) if field == touched => Some(msg), + Err(_) => None, + }; } } @@ -1353,6 +1387,16 @@ mod tests { assert_eq!(w.advanced_summary(), "Customized · port needs attention"); } + #[test] + fn invalid_hidden_host_says_it_needs_attention() { + let w = ServeWizardState { + host: "0.0.0.0".into(), + ..Default::default() + }; + assert!(!w.advanced_expanded); + assert_eq!(w.advanced_summary(), "Customized · host needs attention"); + } + #[test] fn invalid_combination_expands_advanced_and_focuses_the_problem() { let mut wiz = Some(ServeWizardState { @@ -1576,6 +1620,30 @@ mod tests { assert_eq!(w.host, "10.0.0.7", "trimmed on accept"); } + #[test] + fn blanking_an_already_custom_host_names_that_host_as_the_esc_restore_value() { + // The blank-host message names the value Esc actually restores. If + // the form's host is already a custom value (not DEFAULT_HOST) before + // this edit, the message must name that value, not a hardcoded + // default Esc would not actually restore. + let mut wiz = Some(ServeWizardState { + advanced_expanded: true, + host: "10.0.0.7".to_string(), + ..Default::default() + }); + let mut jobs = State::default(); + focus_field(wiz.as_mut().unwrap(), Field::Host); + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + for k in typed(" ") { + on_key(&mut wiz, &mut jobs, &[], k); + } + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + let w = wiz.as_ref().unwrap(); + let msg = w.message.as_deref().unwrap(); + assert!(msg.contains("restore 10.0.0.7"), "{msg}"); + assert!(!msg.contains("127.0.0.1"), "{msg}"); + } + #[test] fn port_editor_accepts_digits_only() { let mut wiz = Some(ServeWizardState { @@ -1664,6 +1732,58 @@ mod tests { assert_eq!(w.message.as_deref(), Some(CUSTOM_HOST_NEEDS_PORT)); } + #[test] + fn committing_a_host_edit_that_surfaces_an_unrelated_port_complaint_clears_it_instead() { + // An outstanding complaint about the Host row (blank host) is fixed by + // typing a loopback host the CLI accepts — but that host also needs a + // custom port, a complaint about a field the user never touched in + // this edit. Silently clear it rather than surprise the user with a + // new complaint about Port; `request_launch` still catches it later. + let mut wiz = Some(ServeWizardState { + advanced_expanded: true, + model: "m".into(), + ..Default::default() + }); + let mut jobs = State::default(); + focus_field(wiz.as_mut().unwrap(), Field::Host); + wiz.as_mut().unwrap().message = + Some("host cannot be empty; type an address or restore 127.0.0.1".to_string()); + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + for k in typed("localhost") { + on_key(&mut wiz, &mut jobs, &[], k); + } + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + let w = wiz.as_ref().unwrap(); + assert_eq!(w.host, "localhost"); + assert!(w.message.is_none(), "not the user's business right now"); + assert_eq!(w.current_field(), Field::Host, "focus stays put"); + } + + #[test] + fn fixing_a_host_complaint_never_surfaces_an_unrelated_model_complaint() { + // Regression test for the exact bug reported: a fresh form with an + // empty Model and an outstanding blank-host complaint. Fixing Host + // must not raise "model is required" — a complaint the user never + // triggered by editing Host. + let mut wiz = Some(ServeWizardState { + advanced_expanded: true, + ..Default::default() + }); + let mut jobs = State::default(); + focus_field(wiz.as_mut().unwrap(), Field::Host); + wiz.as_mut().unwrap().message = + Some("host cannot be empty; type an address or restore 127.0.0.1".to_string()); + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + for k in typed("localhost") { + on_key(&mut wiz, &mut jobs, &[], k); + } + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + let w = wiz.as_ref().unwrap(); + assert_eq!(w.host, "localhost"); + assert!(w.message.is_none(), "model's emptiness is not Host's news"); + assert_eq!(w.current_field(), Field::Host, "focus stays put"); + } + #[test] fn enter_on_auto_port_advances_instead_of_editing() { let mut wiz = Some(ServeWizardState { From 8a5672b41fe470e1043bf4b7d227ccab8a6b7bec Mon Sep 17 00:00:00 2001 From: Juho Vainio Date: Fri, 16 Oct 2026 14:47:43 +0300 Subject: [PATCH 11/19] fix(serve): replace non-ASCII en dash with hyphen in port range message The xtask ascii_output_contract test flags en dashes that can reach a redirectable sink (ROCMAI-543: mojibake on Windows code page 437). Signed-off-by: Juho Vainio --- apps/rocm/src/serve_port.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/rocm/src/serve_port.rs b/apps/rocm/src/serve_port.rs index a14e8915f..30b5bb0f7 100644 --- a/apps/rocm/src/serve_port.rs +++ b/apps/rocm/src/serve_port.rs @@ -194,7 +194,7 @@ pub(crate) fn resolve_serve_port( } } bail!( - "no free TCP port for {DEFAULT_LOCAL_HOST} in {AUTO_PORT_FIRST}–{AUTO_PORT_LAST}; \ + "no free TCP port for {DEFAULT_LOCAL_HOST} in {AUTO_PORT_FIRST}-{AUTO_PORT_LAST}; \ every candidate is reserved by a live managed service or already in use. Stop an \ unused server with `{}`, or pass an explicit `--port`.", service_hints::service_action_hint("stop", "") @@ -451,7 +451,7 @@ mod tests { let error = resolve_serve_port(DEFAULT_LOCAL_HOST, PortRequest::Auto, &[], &mut probe) .expect_err("an exhausted range must fail"); let message = format!("{error:#}"); - assert!(message.contains("11435–11535"), "{message}"); + assert!(message.contains("11435-11535"), "{message}"); } #[test] From 135f9604b6bfcda4ae75f3671cd2939cedc89baa Mon Sep 17 00:00:00 2001 From: Juho Vainio Date: Fri, 16 Oct 2026 15:57:15 +0300 Subject: [PATCH 12/19] fix(rocmd): repair restart rollback tests for write-via-rename Upstream's write_atomic (write-then-rename) migration of ManagedServiceRecord::write() no longer respects a chmod on the manifest file itself -- fs::rename only needs write permission on the containing directory. Point both tests' write-failure simulation at services_dir() instead, pre-creating the launch lock and service log files first so only the manifest write is blocked. Signed-off-by: Juho Vainio --- apps/rocmd/src/watchers.rs | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/apps/rocmd/src/watchers.rs b/apps/rocmd/src/watchers.rs index 50c6a06c4..e964ff05f 100644 --- a/apps/rocmd/src/watchers.rs +++ b/apps/rocmd/src/watchers.rs @@ -3841,8 +3841,13 @@ mod tests { /// If publishing the `recovering` transition fails, nothing has been /// spawned yet: the rollback at that first write must undo the in-memory /// transition and leave the on-disk record exactly as it was, not a - /// half-applied `recovering` row. A manifest file made read-only after - /// being seeded fails that exact write without ever reaching the spawn. + /// half-applied `recovering` row. The manifest write goes through + /// [`crate::write_atomic`][rocm_core_write_atomic]-style write-then-rename, + /// so a read-only *file* no longer blocks it (rename ignores the + /// destination's own permission bits) — a read-only *directory* still does, + /// since it can't hold the new temp file the rename needs. + /// + /// [rocm_core_write_atomic]: rocm_core /// /// Unix only and skipped as root, where a read-only mode is not enforced /// and the write would simply succeed. @@ -3862,14 +3867,20 @@ mod tests { record.status = "failed".to_owned(); record.write()?; - let manifest_path = paths.service_manifest_path(service_id); - fs::set_permissions(&manifest_path, std::fs::Permissions::from_mode(0o444))?; + let services_dir = paths.services_dir(); + // The launch lock and service log files are both created lazily, on + // first acquire and on first open respectively; pre-create them so + // making the directory read-only below blocks only the manifest's + // write-then-rename, not those earlier opens. + fs::File::create(paths.managed_launch_lock_path())?; + fs::File::create(&record.log_path)?; + fs::set_permissions(&services_dir, std::fs::Permissions::from_mode(0o555))?; let result = restart_managed_service(&paths, &mut record); // Restore write access so the rest of the write (if any landed) can be // read back, and so cleanup can remove the directory. - fs::set_permissions(&manifest_path, std::fs::Permissions::from_mode(0o644))?; + fs::set_permissions(&services_dir, std::fs::Permissions::from_mode(0o755))?; let persisted = load_service_record(&paths, service_id); fs::remove_dir_all(root).ok(); @@ -3970,6 +3981,9 @@ mod tests { /// subsequent rollback write hit the same permission error. That must be /// reported as a rollback failure, not swallowed as the original one. /// + /// The manifest writes go through write-then-rename, so blocking the + /// *directory* (not the file) is what actually denies them. + /// /// Unix only and skipped as root, where a read-only mode is not enforced. #[cfg(unix)] #[test] @@ -3990,8 +4004,9 @@ mod tests { record.write()?; let manifest_path = paths.service_manifest_path(service_id); + let services_dir = paths.services_dir(); let lock = { - let manifest_path = manifest_path.clone(); + let services_dir = services_dir.clone(); thread::spawn(move || -> bool { let deadline = std::time::Instant::now() + Duration::from_secs(10); while std::time::Instant::now() < deadline { @@ -4000,8 +4015,8 @@ mod tests { && on_disk.status == "recovering" && on_disk.supervisor_pid == 0 && fs::set_permissions( - &manifest_path, - std::fs::Permissions::from_mode(0o444), + &services_dir, + std::fs::Permissions::from_mode(0o555), ) .is_ok() { @@ -4014,7 +4029,7 @@ mod tests { }; let outcome = restart_managed_service(&paths, &mut record); let locked_in_time = lock.join().expect("lock thread"); - fs::set_permissions(&manifest_path, std::fs::Permissions::from_mode(0o644))?; + fs::set_permissions(&services_dir, std::fs::Permissions::from_mode(0o755))?; let persisted = load_service_record(&paths, service_id); fs::remove_dir_all(root).ok(); From 331fb471c862bc52653630904f044aba70125829 Mon Sep 17 00:00:00 2001 From: Juho Vainio Date: Thu, 22 Oct 2026 18:10:40 +0300 Subject: [PATCH 13/19] fix(serve): address review feedback on port-collision safety Responds to rominf's CHANGES_REQUESTED review on the serve-port collision-safety work: - unify the launch_server port validators in apps/rocmd/src/mcp.rs and apps/rocm/src/chat.rs so a malformed port is rejected the same way from both, instead of silently falling through to automatic selection on hosts that don't support it - treat Windows PermissionDenied as a bind conflict during automatic port scanning instead of aborting the whole scan - fix a blind whole-record write in restart rollback to only restore the fields restart_managed_service changed - fix a vacuous assertion in the Dash wizard Escape-cancels-edit test - add Gherkin coverage for the reworked Dash serve wizard and for an explicit port held by an untracked process (serve-29), gated @requires-gpu after tracing the real call chain - back the serve-28 "stop the reserving service" remediation with a real stop path instead of a direct disk write - strengthen several existing tests: prove the leased port is released before spawn from inside the injected spawn (both call sites), prove stopped/failed records never block port reuse, assert the automatic- port disclosure line order, assert the restart rollback actually kills the supervisor process - README/SKILL.md wording fixes - fix two clippy lints (type_complexity via a SpawnOverride type alias, a redundant clone) introduced by the above Signed-off-by: Juho Vainio --- README.md | 9 +- apps/rocm/src/chat.rs | 2 +- apps/rocm/src/main.rs | 69 ++++- apps/rocm/src/serve_port.rs | 51 +++- apps/rocm/src/service_launch.rs | 183 ++++++++++-- apps/rocmd/src/mcp.rs | 21 +- apps/rocmd/src/watchers.rs | 281 +++++++++++++++++- crates/rocm-core/src/lib.rs | 233 +-------------- crates/rocm-core/src/process.rs | 2 +- crates/rocm-dash-tui/src/app/mod.rs | 27 +- crates/rocm-dash-tui/src/ui/serve_wizard.rs | 3 + skills/rocm-cli-assistant/SKILL.md | 2 +- tests/e2e-cucumber/features/dash.feature | 26 ++ .../features/model_serving.feature | 78 +++-- tests/e2e-cucumber/tests/e2e/dash_steps.rs | 92 ++++++ tests/e2e-cucumber/tests/e2e/serving_steps.rs | 46 ++- 16 files changed, 799 insertions(+), 326 deletions(-) diff --git a/README.md b/README.md index 0931298d3..d7577bdb0 100644 --- a/README.md +++ b/README.md @@ -775,10 +775,11 @@ output, or the Dash approval card) shows `Port: automatic; endpoint shown after launch` while the launch is in flight and prints the concrete endpoint once the port is actually leased; the interactive deployment summary table only ever shows the final resolved endpoint. Pass `--port` explicitly to pin -one; an explicit loopback port already reserved by another managed service is -rejected up front, naming that service and how to free it (`rocm services stop - --yes`). A port held by an untracked process is rejected too, but -without naming the holder. `--host` to anything other than the default +one; on `127.0.0.1`, an explicit loopback port already reserved by another +managed service is rejected up front, naming that service and how to free it +(`rocm services stop --yes`). A port held by an untracked process +is rejected too, but without naming the holder. `--host` to anything other +than the default loopback address (`localhost`, `::1`, `0.0.0.0`, or any other address) requires an explicit `--port`: automatic selection only probes `127.0.0.1` and cannot promise a free port anywhere else. diff --git a/apps/rocm/src/chat.rs b/apps/rocm/src/chat.rs index dcb57184d..c2f184ef0 100644 --- a/apps/rocm/src/chat.rs +++ b/apps/rocm/src/chat.rs @@ -2241,7 +2241,7 @@ fn validate_chat_launch_server_tool_call(call: &providers::ChatToolCall) -> Resu // only to have the argv builder silently drop it and fall through to // automatic port selection. let port = match object.get("port") { - None => None, + None | Some(serde_json::Value::Null) => None, Some(value) => { let port = value .as_u64() diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 6e674fa9d..de5231fb2 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -6989,7 +6989,10 @@ fn stop_internal_managed_service(paths: &AppPaths, service_id: &str) -> Result (Vec, bool), @@ -17738,6 +17741,70 @@ pub(crate) mod tests { Ok(()) } + #[test] + fn restart_is_not_blocked_by_a_stopped_or_failed_record_on_the_same_port() -> Result<()> { + // Mirror of `restart_refuses_a_port_a_launch_claimed...`: `blocks_port` is + // `is_live() && port == port`, so a `stopped`/`failed` sibling on the same + // port must not trip the "claimed that port" refusal. Mutation-proof: if + // `blocks_port` regressed to ignore status, the restart would bail there + // instead of reaching the real (and, in this test harness, doomed) child + // spawn below. + let (root, paths) = test_paths("restart-non-live-same-port"); + paths.ensure()?; + let service_id = "svc-restart-non-live-same-port"; + let record = stop_test_record(&paths, service_id); + let host = record.host.clone(); + let port = record.port; + record.write()?; + + let mut stopped_sibling = ManagedServiceRecord::new( + &paths, + "svc-stopped-sibling", + "lemonade", + "other-model", + "other-model-canonical", + &host, + port, + "managed", + 0, + None, + None, + None, + ); + stopped_sibling.status = "stopped".to_owned(); + stopped_sibling.write()?; + + let mut failed_sibling = ManagedServiceRecord::new( + &paths, + "svc-failed-sibling", + "lemonade", + "other-model-2", + "other-model-2-canonical", + &host, + port, + "managed", + 0, + None, + None, + None, + ); + failed_sibling.status = "failed".to_owned(); + failed_sibling.write()?; + + let result = + restart_internal_managed_service_with(&paths, service_id, None, |_| (Vec::new(), true)); + + let _ = fs::remove_dir_all(root); + if let Err(error) = &result { + let rendered = format!("{error:#}"); + assert!( + !rendered.contains("claimed that port"), + "a stopped or failed sibling on the same port must not be treated as a collision: {rendered}" + ); + } + Ok(()) + } + #[test] fn serve_notes_surface_the_ignored_engine_flag_in_summary_mode() { // The interactive summary is what a default `rocm serve` prints, so a flag diff --git a/apps/rocm/src/serve_port.rs b/apps/rocm/src/serve_port.rs index 30b5bb0f7..291130ddf 100644 --- a/apps/rocm/src/serve_port.rs +++ b/apps/rocm/src/serve_port.rs @@ -185,8 +185,14 @@ pub(crate) fn resolve_serve_port( } match probe(port) { Ok(lease) => return Ok(lease), - // Occupied is the only outcome that advances the scan. + // Occupied is the only outcome that advances the scan. On + // Windows, a port inside an excluded range (Hyper-V, WSL, + // Docker) or held with exclusive-address-use fails with + // WSAEACCES, which Rust reports as `PermissionDenied`, not + // `AddrInUse` — treat it the same as occupied there. Err(error) if error.kind() == io::ErrorKind::AddrInUse => {} + #[cfg(windows)] + Err(error) if error.kind() == io::ErrorKind::PermissionDenied => {} Err(error) => bail!( "failed to reserve {DEFAULT_LOCAL_HOST}:{port} while choosing an \ automatic port: {error}" @@ -411,16 +417,39 @@ mod tests { #[test] fn automatic_stops_immediately_on_a_non_collision_bind_error() { - for kind in [ - io::ErrorKind::PermissionDenied, - io::ErrorKind::AddrNotAvailable, - ] { - let mut probe = scripted(vec![(11_435, kind)]); - let error = resolve_serve_port(DEFAULT_LOCAL_HOST, PortRequest::Auto, &[], &mut probe) - .expect_err("only AddrInUse may advance the scan"); - let message = format!("{error:#}"); - assert!(message.contains("127.0.0.1:11435"), "{message}"); - } + // `PermissionDenied` is excluded here: on Windows it is how a reserved + // range or an exclusive-address-use holder reports occupied, so it + // advances the scan there (see the Windows-only test below). On every + // other platform it means what it says, and the other-platform half + // of that behavior is covered by + // `permission_denied_does_not_advance_the_scan_off_windows`. + let mut probe = scripted(vec![(11_435, io::ErrorKind::AddrNotAvailable)]); + let error = resolve_serve_port(DEFAULT_LOCAL_HOST, PortRequest::Auto, &[], &mut probe) + .expect_err("only AddrInUse (or, on Windows, PermissionDenied) may advance the scan"); + let message = format!("{error:#}"); + assert!(message.contains("127.0.0.1:11435"), "{message}"); + } + + #[test] + #[cfg(not(windows))] + fn permission_denied_does_not_advance_the_scan_off_windows() { + let mut probe = scripted(vec![(11_435, io::ErrorKind::PermissionDenied)]); + let error = resolve_serve_port(DEFAULT_LOCAL_HOST, PortRequest::Auto, &[], &mut probe) + .expect_err("a permission error must not be papered over by choosing another port"); + let message = format!("{error:#}"); + assert!(message.contains("127.0.0.1:11435"), "{message}"); + } + + #[test] + #[cfg(windows)] + fn permission_denied_advances_the_scan_on_windows() { + // WSAEACCES (reserved range or exclusive-address-use) surfaces as + // `PermissionDenied`, not `AddrInUse`; one such candidate must not + // abort automatic selection on a platform #239 asks to support. + let mut probe = scripted(vec![(11_435, io::ErrorKind::PermissionDenied)]); + let lease = resolve_serve_port(DEFAULT_LOCAL_HOST, PortRequest::Auto, &[], &mut probe) + .expect("the scan advances past a Windows-reserved candidate"); + assert_eq!(lease.port(), 11_436); } #[test] diff --git a/apps/rocm/src/service_launch.rs b/apps/rocm/src/service_launch.rs index 6d4210c61..4f6548a38 100644 --- a/apps/rocm/src/service_launch.rs +++ b/apps/rocm/src/service_launch.rs @@ -368,8 +368,9 @@ fn reserved_service_ports(live: &[ManagedServiceRecord]) -> Vec Ok(ManagedSpawn::Spawned { record: Box::new(record), @@ -623,7 +628,10 @@ fn spawn_managed_engine_child( /// /// Split out of [`spawn_managed_engine_child`] so that *every* failure below the /// record's first write is funnelled through one cleanup path before the -/// allocation lock is released. Returns the child pid. +/// launch lock is released. Returns the child pid. +#[cfg(not(windows))] +type SpawnOverride<'a> = dyn Fn(&mut ProcessCommand) -> io::Result + 'a; + #[allow(clippy::too_many_arguments)] fn launch_managed_engine_child_locked( paths: &AppPaths, @@ -638,6 +646,10 @@ fn launch_managed_engine_child_locked( runtime_id: Option<&str>, env_id: Option<&str>, engine_recipe: Option<&EngineRecipeHint>, + // Test-only seam: lets a test assert the leased port is actually bindable + // (i.e. released) at the exact point production would spawn the child, + // without needing to fake or intercept the real spawn mechanics. + #[cfg(not(windows))] spawn_override: Option<&SpawnOverride<'_>>, ) -> Result { let port = record.port; if let Some(parent) = record.engine_state_path.parent() { @@ -692,9 +704,9 @@ fn launch_managed_engine_child_locked( record.requires_api_key, )?; // Hand the port over: close the probe socket immediately before the spawn, - // still under the allocation lock, so no other ROCm transaction can take it - // in the meantime. On a custom host the lease never held a socket and this - // is a no-op — the engine owns that bind. + // still under the caller's launch lock, so no other ROCm transaction can + // take it in the meantime. On a custom host the lease never held a socket + // and this is a no-op — the engine owns that bind. debug_assert_eq!(lease.port(), port); lease.release(); #[cfg(windows)] @@ -731,9 +743,11 @@ fn launch_managed_engine_child_locked( if let Some(key_file) = endpoint_key_file.as_deref() { command.env(rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV, key_file); } - let mut child = command - .spawn() - .context("failed to launch managed engine process")?; + let mut child = match spawn_override { + Some(spawn) => spawn(&mut command), + None => command.spawn(), + } + .context("failed to launch managed engine process")?; let child_pid = child.id(); // Set before the immediate-exit check below, so a bail! here still leaves // `record.engine_pid` telling cleanup that a child was spawned. @@ -2531,8 +2545,13 @@ mod tests { fn allocation_transaction_answers_an_equivalent_live_service_before_leasing() -> Result<()> { let (root, paths) = test_paths("alloc-txn-already-live"); paths.ensure()?; - let existing = - write_live_service_record(&paths, "lemonade-qwen-9000", "qwen-canonical", 11501, "starting")?; + let existing = write_live_service_record( + &paths, + "lemonade-qwen-9000", + "qwen-canonical", + 11501, + "starting", + )?; let decision = resolve_managed_port_in_transaction( &paths, @@ -2654,8 +2673,14 @@ mod tests { "unexpected result while still running: {still_reserved:?}" ); - holder.status = "stopped".to_owned(); - holder.write()?; + // Go through the real stop path (`rocm services stop --yes`'s own + // implementation) rather than writing `status = "stopped"` to disk + // directly, so this test actually proves the advice serve-28 pins: that + // running the stop command frees the port. The termination closure is + // the same seam production injects a real signal through; here it just + // confirms every recorded process gone without touching the test + // process itself. + crate::stop_internal_managed_service_with(&paths, "lemonade-a-1", |_| (Vec::new(), true))?; let decision = resolve_managed_port_in_transaction( &paths, @@ -2725,7 +2750,7 @@ mod tests { let spawn_one = |model: &'static str| { let paths = std::sync::Arc::clone(&paths); thread::spawn(move || -> Result { - let _allocation = rocm_core::lock_service_allocation(&paths)?; + let _launch_lock = rocm_core::FileLock::acquire(paths.managed_launch_lock_path())?; let lease = match resolve_managed_port_in_transaction( &paths, "lemonade", @@ -2771,7 +2796,7 @@ mod tests { let race_once = || { let paths = std::sync::Arc::clone(&paths); thread::spawn(move || -> Result { - let _allocation = rocm_core::lock_service_allocation(&paths)?; + let _launch_lock = rocm_core::FileLock::acquire(paths.managed_launch_lock_path())?; match resolve_managed_port_in_transaction( &paths, "lemonade", @@ -2910,7 +2935,13 @@ mod tests { fn abandoned_launch_leaves_no_record_log_or_endpoint_key() -> Result<()> { let (root, paths) = test_paths("abandon-launch-cleanup"); paths.ensure()?; - let mut record = write_live_service_record(&paths, "lemonade-abandon", "abandon-model", 11_490, "running")?; + let mut record = write_live_service_record( + &paths, + "lemonade-abandon", + "abandon-model", + 11_490, + "running", + )?; fs::create_dir_all( record .engine_state_path @@ -2927,7 +2958,10 @@ mod tests { assert!(!record.manifest_path.exists()); assert!(!record.log_path.exists()); assert!(!record.engine_state_path.exists()); - assert_eq!(endpoint_keys::endpoint_api_key(&paths, &record.service_id), None); + assert_eq!( + endpoint_keys::endpoint_api_key(&paths, &record.service_id), + None + ); assert!(live_managed_services(&paths).is_empty()); let _ = fs::remove_dir_all(root); Ok(()) @@ -2947,8 +2981,13 @@ mod tests { fn abandoned_launch_terminates_the_spawned_child() -> Result<()> { let (root, paths) = test_paths("abandon-launch-terminates-child"); paths.ensure()?; - let mut record = - write_live_service_record(&paths, "lemonade-abandon-child", "abandon-child-model", 11_491, "running")?; + let mut record = write_live_service_record( + &paths, + "lemonade-abandon-child", + "abandon-child-model", + 11_491, + "running", + )?; let current_exe = std::env::current_exe().expect("current test exe"); let mut child = ProcessCommand::new(current_exe) @@ -2987,4 +3026,96 @@ mod tests { ); Ok(()) } + + #[test] + #[cfg(not(windows))] + fn launch_releases_the_leased_port_before_spawning_the_child() -> Result<()> { + // `lease.release()` must run before the child is spawned, not after, + // or the engine is handed a port the CLI is still holding open. + // Proven via the injected `spawn_override` seam: it asserts a fresh + // bind on the leased port succeeds at the exact instant production + // would spawn, which only holds if the lease was already released. + let (root, paths) = test_paths("launch-releases-port-before-spawn"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let port = 11_599_u16; + let lease = rocm_core::lease_loopback_port(port)?; + let mut record = ManagedServiceRecord::new( + &paths, + "lemonade-release-before-spawn", + "lemonade", + "qwen", + "qwen-canonical", + "127.0.0.1", + port, + "managed", + 0, + None, + None, + None, + ); + record.write()?; + + let resolve = ResolveModelResponse { + canonical_model_id: "qwen-canonical".to_owned(), + task: "chat".to_owned(), + source: "hf".to_owned(), + revision: "main".to_owned(), + loader: "llama.cpp".to_owned(), + trust_remote_code: false, + chat_template_mode: "auto".to_owned(), + dtype: "auto".to_owned(), + device_policy: DevicePolicy::GpuPreferred, + estimated_memory: "unknown".to_owned(), + launch_defaults: serde_json::json!({}), + engine_recipe: None, + warnings: Vec::new(), + }; + + let bind_succeeded = std::cell::Cell::new(false); + let spawn_override: &dyn Fn(&mut ProcessCommand) -> io::Result = + &|_command| { + bind_succeeded.set(std::net::TcpListener::bind(("127.0.0.1", port)).is_ok()); + // A long-lived no-op child stands in for the engine: `_command` + // carries real serve args this test binary cannot execute as + // itself (see `managed_launch_cleanup_sleeper_child`). + ProcessCommand::new(std::env::current_exe().expect("test binary")) + .arg("service_launch::tests::managed_launch_cleanup_sleeper_child") + .arg("--exact") + .arg("--ignored") + .env("ROCM_TEST_SLEEPER_CHILD", "1") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + }; + + let result = launch_managed_engine_child_locked( + &paths, + &mut record, + lease, + "lemonade", + "lemonade-release-before-spawn", + &resolve, + "127.0.0.1", + &resolve.device_policy, + &[], + None, + None, + None, + Some(spawn_override), + ); + if let Ok(child_pid) = result.as_ref() { + let _ = rocm_core::terminate_process(*child_pid); + } + let _ = fs::remove_dir_all(root); + + assert!( + bind_succeeded.get(), + "the port must already be bindable (released) by the time the child is spawned" + ); + result?; + Ok(()) + } } diff --git a/apps/rocmd/src/mcp.rs b/apps/rocmd/src/mcp.rs index bd6c480a2..ab0d4a679 100644 --- a/apps/rocmd/src/mcp.rs +++ b/apps/rocmd/src/mcp.rs @@ -1094,11 +1094,28 @@ fn build_launch_server_args(arguments: &serde_json::Map) -> Resul "launch_server host `{host}` is not loopback; require `allow_public_bind=true` before binding a non-local interface" ); } + // Validate the same way the argv builder reads it + // (`arguments.get("port").and_then(Value::as_u64)`), so a malformed + // `port` (wrong type, out of range) can't slip past this check only to + // have the argv builder silently drop it and fall through to automatic + // port selection. + let port = match arguments.get("port") { + None | Some(Value::Null) => None, + Some(value) => { + let port = value + .as_u64() + .context("launch_server argument `port` must be an integer")?; + if !(1..=u64::from(u16::MAX)).contains(&port) { + bail!("launch_server argument `port` must be between 1 and 65535"); + } + Some(port) + } + }; // `rocm serve` proves a port is free only on the exact default host; every // other spelling (including the loopback aliases `localhost`/`::1` the // check above still accepts) needs an explicit port or the CLI call // refuses later with a less actionable error. See `is_canonical_loopback_host`. - if arguments.get("port").is_none() && !rocm_core::is_canonical_loopback_host(host) { + if port.is_none() && !rocm_core::is_canonical_loopback_host(host) { bail!( "launch_server host `{host}` needs an explicit `port`; automatic port selection is \ only supported on the default host {DEFAULT_LOCAL_HOST}" @@ -1127,7 +1144,7 @@ fn build_launch_server_args(arguments: &serde_json::Map) -> Resul if allow_public_bind { argv.push("--allow-public-bind".to_owned()); } - if let Some(port) = arguments.get("port").and_then(Value::as_u64) { + if let Some(port) = port { argv.push("--port".to_owned()); argv.push(port.to_string()); } diff --git a/apps/rocmd/src/watchers.rs b/apps/rocmd/src/watchers.rs index e964ff05f..ea4d91f46 100644 --- a/apps/rocmd/src/watchers.rs +++ b/apps/rocmd/src/watchers.rs @@ -1279,7 +1279,7 @@ fn handle_server_recover_event_with_record( Some(record.service_id.clone()), ); } - // A contended allocation lock or an occupied recorded port is a + // A contended launch lock or an occupied recorded port is a // recoverable, per-service condition — report it and move on rather // than letting one service take the whole 30s watcher tick (and the // daemon) down. @@ -1491,7 +1491,26 @@ pub(crate) fn restart_managed_service( paths: &AppPaths, record: &mut ManagedServiceRecord, ) -> Result<()> { - let _allocation = rocm_core::lock_service_allocation(paths)?; + restart_managed_service_with(paths, record, None) +} + +/// [`restart_managed_service`], with the supervisor spawn interceptable. +/// +/// `spawn_override`, when set, replaces the real `command.spawn()` call below +/// (falling back to it when `None`) — a test-only seam so a test can assert +/// the leased port is already bindable (released) at the exact instant +/// production would spawn, without faking the whole spawn. +type SpawnOverride<'a> = dyn Fn(&mut ProcessCommand) -> std::io::Result + 'a; + +fn restart_managed_service_with( + paths: &AppPaths, + record: &mut ManagedServiceRecord, + spawn_override: Option<&SpawnOverride<'_>>, +) -> Result<()> { + let _launch_lock = rocm_core::FileLock::acquire_bounded( + paths.managed_launch_lock_path(), + rocm_core::MANAGED_LAUNCH_LOCK_TIMEOUT, + )?; // Re-check against the disk, now that the lock excludes a concurrent // `rocm serve` publish: the caller's `record` was read before this lock was @@ -1545,6 +1564,14 @@ pub(crate) fn restart_managed_service( .try_clone() .context("failed to clone service log file handle")?; + // The truthful state to fall back to. The publish and + // `record_spawned_supervisor` failure paths below restore this rather than + // leave a half-applied `recovering` row, and the latter also makes sure a + // supervisor that exists is never left running unrecorded. The spawn and + // `try_wait` failure paths after that do not restore it: they return before + // any rollback runs, so `recovering` can still be left standing on disk. + let prior = record.clone(); + record.status = "recovering".to_owned(); // Counts the restart and drops the previous run's inference verification. // The respawned child writes a fresh record of its own, and "recovering" is @@ -1572,26 +1599,55 @@ pub(crate) fn restart_managed_service( // Restarting is a request to run, so it supersedes any earlier stop request, // as `rocm services restart` treats it too. record.stop_requested_unix_ms = None; - record.write()?; + if let Err(error) = record.write() { + // Nothing was spawned and the on-disk record was never replaced; just + // undo the in-memory transition so the caller does not report recovery + // that never happened. + *record = prior; + return Err(error.context("failed to publish the recovering service record")); + } // Hand the port to the supervisor: close the probe socket immediately before - // the spawn, still holding the allocation lock. + // the spawn, still holding the launch lock. if let Some(lease) = lease { lease.release(); } - let mut child = detached_rocmd_command(&rocmd_binary) + let recovering_snapshot = record.clone(); + let mut command = detached_rocmd_command(&rocmd_binary); + command .args(recovery_supervise_args(record)) .stdin(Stdio::null()) .stdout(Stdio::from(log_file)) - .stderr(Stdio::from(log_file_err)) - .spawn() - .context("failed to spawn recovery supervisor")?; + .stderr(Stdio::from(log_file_err)); + let mut child = match spawn_override { + Some(spawn) => spawn(&mut command), + None => command.spawn(), + } + .context("failed to spawn recovery supervisor")?; if let Err(error) = record_spawned_supervisor(paths, record, child.id()) { // The supervisor is alive but unrecorded: it would hold the port while - // being invisible to every stop/recovery path. + // being invisible to every stop/recovery path. Kill it, then put the + // prior truthful fields back. Not a blind `record.write()` of `prior`: + // the same discipline as `update_supervised_record` elsewhere in this + // function applies here too, so a stop marker a concurrent stop + // persisted while the spawn was in flight is not reverted by this + // rollback. terminate_recovery_supervisor(&mut child); + let error = error.context("failed to publish the recovering service record"); + if let Err(rollback) = rollback_recovering_record(paths, &recovering_snapshot, &prior) { + // Both writes failed: the on-disk record now still claims + // `recovering` for a supervisor that no longer exists. That is worse + // than the original failure and must not be swallowed — the watcher + // and the manual restart tool both surface whatever comes back. + return Err(error.context(format!( + "rollback also failed: managed service {} is left recorded as recovering \ + with no supervisor ({rollback:#})", + record.service_id + ))); + } + *record = prior; return Err(error); } @@ -1621,6 +1677,20 @@ pub(crate) fn restart_managed_service( /// record could not be published. Short: it is milliseconds old. const RECOVERY_CLEANUP_GRACE: Duration = Duration::from_secs(2); +// Test-only, thread-scoped call counter: `restart_managed_service`'s rollback +// branch asserts this to prove the kill is actually reached (not just that the +// record comes back truthful), since that branch's own supervisor is a +// re-exec'd test binary that exits on its own almost immediately regardless of +// whether it is killed, so `process_is_running` on it would not catch the call +// being dropped. Thread-local, not a global counter: cargo runs tests +// concurrently on separate threads, and this function has another, unrelated +// direct caller (`terminate_recovery_supervisor_kills_and_reaps_the_child`). +#[cfg(test)] +thread_local! { + static TERMINATE_RECOVERY_SUPERVISOR_CALLS: std::cell::Cell = + const { std::cell::Cell::new(0) }; +} + /// Terminate and reap a recovery supervisor that could not be recorded. /// /// Tree scope, forced: the supervisor may already have started an engine child, @@ -1628,6 +1698,8 @@ const RECOVERY_CLEANUP_GRACE: Duration = Duration::from_secs(2); /// no stop or recovery path can find. Reaping afterwards keeps the daemon free /// of zombies. fn terminate_recovery_supervisor(child: &mut std::process::Child) { + #[cfg(test)] + TERMINATE_RECOVERY_SUPERVISOR_CALLS.with(|calls| calls.set(calls.get() + 1)); let _outcome = rocm_core::terminate_verified( &rocm_core::ProcessIdentity::capture(child.id()), rocm_core::KillScope::Tree, @@ -1663,6 +1735,49 @@ fn record_spawned_supervisor( }) } +/// Restore only the fields [`restart_managed_service`]'s transaction owns, +/// after `record_spawned_supervisor` failed to publish them. The same +/// discipline as `update_supervised_record` elsewhere in that function: a +/// stop marker a concurrent stop persisted on the live record must survive +/// this rollback, not be overwritten by a blind copy of `restore`. +/// +/// Differs from that helper in one case: when the manifest cannot be parsed +/// at all (as opposed to merely naming a different supervisor), there is no +/// live data left to selectively preserve, so this falls back to writing +/// `restore` in full rather than no-op'ing. +/// `restart_rolls_back_and_kills_the_supervisor_when_the_second_write_cannot_be_read` +/// exercises exactly that fallback. +fn rollback_recovering_record( + paths: &AppPaths, + recovering_snapshot: &ManagedServiceRecord, + restore: &ManagedServiceRecord, +) -> Result<()> { + let manifest_path = paths.service_manifest_path(&recovering_snapshot.service_id); + if !manifest_path.exists() { + return Ok(()); + } + match load_service_record(paths, &recovering_snapshot.service_id) { + Ok(mut current) + if current.supervisor_pid == recovering_snapshot.supervisor_pid + && current.supervisor_start_ticks == recovering_snapshot.supervisor_start_ticks => + { + current.status = restore.status.clone(); + current.supervisor_pid = restore.supervisor_pid; + current.supervisor_start_ticks = restore.supervisor_start_ticks; + current.engine_pid = restore.engine_pid; + current.engine_start_ticks = restore.engine_start_ticks; + current.restart_count = restore.restart_count; + current.last_restart_unix_ms = restore.last_restart_unix_ms; + current.inference_verified_at_unix_ms = restore.inference_verified_at_unix_ms; + current.inference_probe_attempted_at_unix_ms = + restore.inference_probe_attempted_at_unix_ms; + current.write() + } + Ok(_) => Ok(()), + Err(_) => restore.write(), + } +} + fn recovery_supervise_args(record: &ManagedServiceRecord) -> Vec { let mut args = vec![ "supervise".to_owned(), @@ -3364,7 +3479,10 @@ mod tests { record.status = "failed".to_owned(); record.write()?; - let guard = rocm_core::lock_service_allocation(&paths)?; + let guard = rocm_core::FileLock::acquire_bounded( + paths.managed_launch_lock_path(), + rocm_core::MANAGED_LAUNCH_LOCK_TIMEOUT, + )?; let finished = Arc::new(AtomicBool::new(false)); let handle = { let paths = paths.clone(); @@ -3379,7 +3497,7 @@ mod tests { thread::sleep(Duration::from_millis(250)); assert!( !finished.load(Ordering::SeqCst), - "recovery must serialize on the shared allocation lock" + "recovery must serialize on the shared launch lock" ); // Simulate a `rocm serve --gpu auto` publishing a brand-new service on @@ -3423,6 +3541,80 @@ mod tests { Ok(()) } + #[test] + fn recovery_is_not_blocked_by_a_stopped_or_failed_record_on_the_same_port() -> Result<()> { + // The mirror image of the refusal above: `blocks_port` is + // `is_live() && port == port`, so a `stopped` or `failed` record on + // the same port must not count as a collision. Mutation-proof: if + // `blocks_port` regressed to ignore status, either sibling below + // would trip the "claimed that port" bail and this test would go red. + let (root, paths) = temp_app_paths("recovery-non-live-same-port"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let port = 21_778_u16; + let mut stopped_sibling = ManagedServiceRecord::new( + &paths, + "lemonade-stopped-sibling", + "lemonade", + "other-model", + "other-model-canonical".to_owned(), + rocm_core::DEFAULT_LOCAL_HOST, + port, + "managed", + 0, + None, + None, + None, + ); + stopped_sibling.status = "stopped".to_owned(); + stopped_sibling.write()?; + + let mut failed_sibling = ManagedServiceRecord::new( + &paths, + "lemonade-failed-sibling", + "lemonade", + "other-model-2", + "other-model-2-canonical".to_owned(), + rocm_core::DEFAULT_LOCAL_HOST, + port, + "managed", + 0, + None, + None, + None, + ); + failed_sibling.status = "failed".to_owned(); + failed_sibling.write()?; + + let mut record = identity_probe_record(&paths, "lemonade-recover-non-live", port); + record.status = "failed".to_owned(); + record.write()?; + + // The re-exec'd "supervisor" is this very test binary, which cannot + // actually run as `rocmd supervise` and exits almost immediately; the + // same limitation `restart_managed_service_persists_the_supervisor_identity_token` + // works around. What this proves is that the spawn was ever reached: + // a wrongly-blocking `blocks_port` would bail before `record.write()` + // below ever runs, leaving `supervisor_start_ticks` at `None`. + let result = restart_managed_service(&paths, &mut record); + let persisted = load_service_record(&paths, "lemonade-recover-non-live"); + fs::remove_dir_all(root).ok(); + + if let Err(error) = &result { + let message = format!("{error:#}"); + assert!( + !message.contains("claimed that port"), + "a stopped or failed sibling on the same port must not be treated as a collision: {message}" + ); + } + assert!( + persisted?.supervisor_start_ticks.is_some(), + "recovery must reach the spawn, proving neither sibling blocked it" + ); + Ok(()) + } + #[test] fn contained_recovery_survives_a_failed_restart_and_backs_off() -> Result<()> { use std::net::TcpListener; @@ -3598,6 +3790,49 @@ mod tests { Ok(()) } + /// Mirrors `launch_releases_the_leased_port_before_spawning_the_child` in + /// `apps/rocm/src/service_launch.rs`: the probe lease on the recorded port + /// must already be released by the time the recovery supervisor is + /// spawned, or the supervisor's own bind of that port would fail. + #[test] + fn restart_releases_the_leased_port_before_spawning_the_supervisor() -> Result<()> { + use std::cell::Cell; + use std::net::TcpListener; + + let (root, paths) = temp_app_paths("restart-releases-lease"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let probe = TcpListener::bind(("127.0.0.1", 0))?; + let port = probe.local_addr()?.port(); + drop(probe); + + let service_id = "svc-restart-releases-lease"; + let mut record = identity_probe_record(&paths, service_id, port); + record.status = "failed".to_owned(); + record.write()?; + + let bind_succeeded = Cell::new(false); + let spawn_override = + |command: &mut ProcessCommand| -> std::io::Result { + bind_succeeded.set(TcpListener::bind(("127.0.0.1", port)).is_ok()); + command.spawn() + }; + + // The re-exec'd "supervise" args are rejected by this test binary, + // which exits at once (see + // restart_managed_service_persists_the_supervisor_identity_token), so + // the restart itself is expected to report failure here. + let _ = restart_managed_service_with(&paths, &mut record, Some(&spawn_override)); + fs::remove_dir_all(root).ok(); + + assert!( + bind_succeeded.get(), + "the leased port must already be bindable (released) by the time the supervisor is spawned" + ); + Ok(()) + } + /// An unconfirmed stop leaves the status at `ready` on purpose, which is /// exactly what the daemon's recovery scan looks for. The stop request it /// records is the only thing telling it apart from a crash, so recovery @@ -3738,7 +3973,7 @@ mod tests { } /// Recovery re-claims the concrete port already written in the record, so - /// it must serialize on the same cross-process allocation lock a fresh + /// it must serialize on the same cross-process launch lock a fresh /// `rocm serve` launch takes, and must refuse a recorded port something /// else already holds rather than publish a `recovering` record that /// lies. @@ -3760,7 +3995,10 @@ mod tests { record.status = "failed".to_owned(); record.write()?; - let guard = rocm_core::lock_service_allocation(&paths)?; + let guard = rocm_core::FileLock::acquire_bounded( + paths.managed_launch_lock_path(), + rocm_core::MANAGED_LAUNCH_LOCK_TIMEOUT, + )?; let finished = Arc::new(AtomicBool::new(false)); let handle = { let paths = paths.clone(); @@ -3776,7 +4014,7 @@ mod tests { thread::sleep(Duration::from_millis(250)); assert!( !finished.load(Ordering::SeqCst), - "recovery must serialize on the shared allocation lock" + "recovery must serialize on the shared launch lock" ); drop(guard); @@ -3909,6 +4147,14 @@ mod tests { /// supervisor and restore the prior record — and the rollback write itself /// must still land, since corrupting *content* leaves the file a writable /// regular file. + /// + /// The spawned supervisor here is a re-exec of this very test binary, + /// which rejects its `supervise` arguments and exits on its own almost + /// immediately regardless of whether anything kills it, so asserting + /// `process_is_running` on it afterward would not catch the kill being + /// dropped. `TERMINATE_RECOVERY_SUPERVISOR_CALLS` asserts the call itself + /// is reached instead; `terminate_recovery_supervisor_kills_and_reaps_the_child` + /// below proves what that call does against a real long-lived process. #[test] fn restart_rolls_back_and_kills_the_supervisor_when_the_second_write_cannot_be_read() -> Result<()> { @@ -3922,6 +4168,7 @@ mod tests { record.write()?; let prior = record.clone(); + let terminate_calls_before = TERMINATE_RECOVERY_SUPERVISOR_CALLS.with(std::cell::Cell::get); let manifest_path = paths.service_manifest_path(service_id); let corrupt = thread::spawn(move || -> bool { let deadline = std::time::Instant::now() + Duration::from_secs(10); @@ -3962,6 +4209,12 @@ mod tests { "corrupting content, not permissions, leaves the rollback write itself able to \ land: {message}" ); + let terminate_calls_after = TERMINATE_RECOVERY_SUPERVISOR_CALLS.with(std::cell::Cell::get); + assert_eq!( + terminate_calls_after, + terminate_calls_before + 1, + "an unreadable second write must still kill the spawned supervisor" + ); assert_eq!( record.status, prior.status, "the in-memory record must be restored to what it was before this restart" diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 7cb928a0d..0427ea894 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -7,8 +7,7 @@ use serde::{Deserialize, Serialize}; use std::ffi::OsString; use std::fs; use std::path::{Path, PathBuf}; -use std::thread; -use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +use std::time::{SystemTime, UNIX_EPOCH}; #[cfg(all(test, unix))] mod archive_props; @@ -523,117 +522,12 @@ pub(crate) fn write_atomic(path: &Path, bytes: &[u8], tmp_ext: &str) -> Result<( Ok(()) } -/// File name of the shared allocation lock, inside [`AppPaths::services_dir`]. -/// -/// Not a `.json` file, so `load_managed_services` never mistakes it for a -/// service manifest. -pub const SERVICE_ALLOCATION_LOCK_FILE: &str = "allocation.lock"; -/// Total time a launch or recovery waits for the allocation lock before failing -/// with an actionable error instead of hanging. -pub const SERVICE_ALLOCATION_LOCK_TIMEOUT: Duration = Duration::from_secs(5); -const SERVICE_ALLOCATION_LOCK_POLL: Duration = Duration::from_millis(25); - -#[must_use] -pub fn service_allocation_lock_path(paths: &AppPaths) -> PathBuf { - paths.services_dir().join(SERVICE_ALLOCATION_LOCK_FILE) -} - -/// Cross-process guard serializing the managed-service allocation transaction: -/// duplicate detection, live-reservation refresh, port leasing, child start -/// confirmation, and record publication. -/// -/// The lock is an OS advisory file lock, so it is released both by dropping this -/// guard and by process death — a crashed launcher never wedges the next one. -/// Full HTTP/model readiness deliberately runs *outside* the guard: it takes up -/// to 45 seconds and holds no allocation decision. -#[derive(Debug)] -pub struct ServiceAllocationLock { - file: Option, - path: PathBuf, -} - -impl ServiceAllocationLock { - #[must_use] - pub fn path(&self) -> &Path { - &self.path - } -} - -impl Drop for ServiceAllocationLock { - fn drop(&mut self) { - if let Some(file) = self.file.take() { - let _ = file.unlock(); - } - } -} - -/// Acquire the shared allocation lock for `paths`, waiting at most -/// [`SERVICE_ALLOCATION_LOCK_TIMEOUT`]. -/// -/// # Errors -/// Fails when the lock file cannot be created/locked, or when another process -/// held it for the whole timeout. -pub fn lock_service_allocation(paths: &AppPaths) -> Result { - lock_service_allocation_at( - &service_allocation_lock_path(paths), - SERVICE_ALLOCATION_LOCK_TIMEOUT, - ) -} - -/// [`lock_service_allocation`] against an explicit path and timeout. Exposed for -/// tests that need a private lock file or a short deadline. -/// -/// # Errors -/// See [`lock_service_allocation`]. -pub fn lock_service_allocation_at(path: &Path, timeout: Duration) -> Result { - if let Some(parent) = path.parent() { - fs::create_dir_all(parent) - .with_context(|| format!("failed to create {}", parent.display()))?; - } - let file = fs::OpenOptions::new() - .create(true) - .read(true) - .write(true) - .truncate(false) - .open(path) - .with_context(|| format!("failed to open {}", path.display()))?; - let deadline = Instant::now() + timeout; - loop { - match file.try_lock() { - Ok(()) => { - return Ok(ServiceAllocationLock { - file: Some(file), - path: path.to_path_buf(), - }); - } - Err(std::fs::TryLockError::WouldBlock) => {} - Err(std::fs::TryLockError::Error(error)) => { - return Err( - anyhow::Error::new(error).context(format!("failed to lock {}", path.display())) - ); - } - } - let now = Instant::now(); - if now >= deadline { - bail!( - "timed out after {timeout:?} waiting for the managed-service allocation lock at \ - {path}; another `rocm serve` launch or `rocmd` recovery is mid-transaction. \ - Retry in a moment, and check `rocm services` for a stuck launch.", - path = path.display() - ); - } - thread::sleep(SERVICE_ALLOCATION_LOCK_POLL.min(deadline - now)); - } -} - #[cfg(test)] mod tests { use super::*; - use crate::test_support::workspace_test_artifact_dir; use std::fs; use std::path::Path; - use std::process::{Command, Stdio}; #[test] fn service_id_accepts_generated_and_plain_ids() { @@ -896,129 +790,4 @@ mod tests { ); Ok(()) } - - // -- managed-service allocation lock and loopback leasing --------------- - - const LOCK_CHILD_PATH_ENV: &str = "ROCM_CORE_TEST_LOCK_PATH"; - const LOCK_CHILD_READY_ENV: &str = "ROCM_CORE_TEST_LOCK_READY"; - - fn lock_test_dir(label: &str) -> PathBuf { - let dir = workspace_test_artifact_dir().join(format!( - "{label}-{}-{}", - std::process::id(), - unix_time_millis() - )); - fs::create_dir_all(&dir).expect("create lock test dir"); - dir - } - - /// Re-run this test binary, executing exactly one `#[ignore]`d helper test in - /// a genuinely separate process. Cross-process locking and socket handoff - /// cannot be proven inside one process. - fn spawn_helper_test(name: &str, envs: &[(&str, String)]) -> std::process::Child { - let exe = std::env::current_exe().expect("test binary path"); - let mut command = Command::new(exe); - command - .arg(name) - .arg("--exact") - .arg("--ignored") - .arg("--test-threads") - .arg("1") - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::null()); - for (key, value) in envs { - command.env(key, value); - } - command.spawn().expect("spawn helper test process") - } - - #[test] - fn allocation_lock_path_lives_in_services_dir_and_is_not_a_manifest() { - let paths = AppPaths { - config_dir: PathBuf::from("/tmp/cfg"), - data_dir: PathBuf::from("/tmp/data"), - cache_dir: PathBuf::from("/tmp/cache"), - }; - let path = service_allocation_lock_path(&paths); - assert_eq!(path.parent(), Some(paths.services_dir().as_path())); - // `load_managed_services` only reads `*.json`, so the lock must not be one. - assert_ne!( - path.extension().and_then(|value| value.to_str()), - Some("json") - ); - } - - #[test] - fn allocation_lock_excludes_independent_handles_and_fails_closed_on_timeout() { - let path = lock_test_dir("core-alloc-lock-exclusive").join("allocation.lock"); - let held = lock_service_allocation_at(&path, Duration::from_secs(5)).expect("first holder"); - assert_eq!(held.path(), path.as_path()); - - // A second, independent file handle must contend on the OS lock rather - // than sail through, and must give up with an actionable error. - let started = Instant::now(); - let error = lock_service_allocation_at(&path, Duration::from_millis(200)) - .expect_err("a held lock must not be acquired twice"); - assert!( - started.elapsed() >= Duration::from_millis(200), - "waiter returned before its deadline" - ); - let message = format!("{error:#}"); - assert!(message.contains("timed out"), "{message}"); - assert!(message.contains("allocation lock"), "{message}"); - - drop(held); - lock_service_allocation_at(&path, Duration::from_secs(5)) - .expect("lock is free once the guard is dropped"); - } - - #[test] - #[ignore = "spawned as a child process by allocation_lock_is_released_when_the_holder_process_exits"] - fn allocation_lock_child_holder() { - let Ok(path) = std::env::var(LOCK_CHILD_PATH_ENV) else { - return; - }; - let ready = std::env::var(LOCK_CHILD_READY_ENV).expect("ready-file path"); - let guard = lock_service_allocation_at(Path::new(&path), Duration::from_secs(10)) - .expect("child acquires the shared lock"); - fs::write(&ready, "held").expect("signal that the child holds the lock"); - thread::sleep(Duration::from_millis(300)); - // Leak the guard so the lock is released by process exit alone — that is - // the property a crashed launcher depends on. - std::mem::forget(guard); - } - - #[test] - fn allocation_lock_is_released_when_the_holder_process_exits() { - let dir = lock_test_dir("core-alloc-lock-crossproc"); - let path = dir.join("allocation.lock"); - let ready = dir.join("child-ready"); - let mut child = spawn_helper_test( - "tests::allocation_lock_child_holder", - &[ - (LOCK_CHILD_PATH_ENV, path.display().to_string()), - (LOCK_CHILD_READY_ENV, ready.display().to_string()), - ], - ); - - let deadline = Instant::now() + Duration::from_mins(1); - while !ready.exists() { - assert!( - Instant::now() < deadline, - "child never reported holding the lock" - ); - thread::sleep(Duration::from_millis(25)); - } - - let busy = lock_service_allocation_at(&path, Duration::from_millis(50)) - .expect_err("the lock must be exclusive across processes"); - assert!(format!("{busy:#}").contains("allocation lock")); - - let status = child.wait().expect("child holder exits"); - assert!(status.success(), "child holder failed: {status}"); - lock_service_allocation_at(&path, Duration::from_secs(10)) - .expect("the OS releases the lock when the holding process exits"); - } - } diff --git a/crates/rocm-core/src/process.rs b/crates/rocm-core/src/process.rs index 4746920ce..928e134cc 100644 --- a/crates/rocm-core/src/process.rs +++ b/crates/rocm-core/src/process.rs @@ -23,8 +23,8 @@ //! capture helpers onto it is tracked in //! [#623](https://github.com/ROCm/rocm-cli/issues/623). -use anyhow::{Context, Result, bail}; use crate::DEFAULT_LOCAL_HOST; +use anyhow::{Context, Result, bail}; #[cfg(windows)] use std::collections::BTreeMap; #[cfg(windows)] diff --git a/crates/rocm-dash-tui/src/app/mod.rs b/crates/rocm-dash-tui/src/app/mod.rs index 967ecb297..394aebe47 100644 --- a/crates/rocm-dash-tui/src/app/mod.rs +++ b/crates/rocm-dash-tui/src/app/mod.rs @@ -1816,9 +1816,32 @@ mod tests { !state.should_pane_back_out(crossterm::event::KeyCode::Esc), "Esc must reach the inline editor instead of closing the wizard" ); + + // Drive Esc through the real dispatch (mirrors event_loop.rs's + // `state.serve_wizard.is_some()` arm) instead of only checking the + // routing predicate above, so a removed `editor_key` Esc arm turns + // this test red. + let key = crossterm::event::KeyEvent::new( + crossterm::event::KeyCode::Esc, + crossterm::event::KeyModifiers::NONE, + ); + let _ = crate::ui::serve_wizard::on_key( + &mut state.serve_wizard, + &mut state.jobs, + &state.model_recipes, + key, + ); + + let wizard = state + .serve_wizard + .as_ref() + .expect("Esc must not close the wizard"); + assert!( + wizard.editor.is_none(), + "Esc must discard the inline editor" + ); assert_eq!( - state.serve_wizard.as_ref().unwrap().model, - "typed-model", + wizard.model, "typed-model", "the in-progress form remains available to the editor handler" ); } diff --git a/crates/rocm-dash-tui/src/ui/serve_wizard.rs b/crates/rocm-dash-tui/src/ui/serve_wizard.rs index 8d24b390e..49d7897bc 100644 --- a/crates/rocm-dash-tui/src/ui/serve_wizard.rs +++ b/crates/rocm-dash-tui/src/ui/serve_wizard.rs @@ -1842,6 +1842,9 @@ mod tests { assert_eq!(jobs.jobs.len(), 1); } + // This only pins the shared constant into the approval body; the CLI's own + // plan output proves the phrase actually appears before the resolved + // endpoint, in `serve-auto-port-skips-occupied` (serve-26). #[test] fn approval_shows_exact_argv_and_the_automatic_port_phrase() { let mut w = ServeWizardState { diff --git a/skills/rocm-cli-assistant/SKILL.md b/skills/rocm-cli-assistant/SKILL.md index 9cc280d68..b080c50e0 100644 --- a/skills/rocm-cli-assistant/SKILL.md +++ b/skills/rocm-cli-assistant/SKILL.md @@ -28,7 +28,7 @@ Use this skill when answering ROCm CLI local assistant questions. - The built-in assistant is fixed to qwen served by Lemonade with GPU required. Do not switch the built-in assistant to vLLM. - Installing an engine and running a model server are different states. Answer each separately when the user asks. - `rocm serve` accepts `--gpu auto|` to pick the AMD GPU. `auto` (default) prefers a GPU that looks idle from VRAM telemetry (`amd-smi`, or the amdgpu DRM sysfs counters when `amd-smi` is absent) and is not already used by another rocm-cli server (managed or foreground), falling back to the GPU with the most free memory; a single index pins one GPU. Serving one model across multiple GPUs is not supported. Do not suggest CPU fallback when a GPU is busy or out of range. -- Automatic port selection only works on host `127.0.0.1` exactly. `localhost` and `::1` are loopback for auth purposes but still need an explicit `port` when launching a server, or the call is refused. +- Automatic port selection only works on host `127.0.0.1` exactly. `localhost` and `::1` are loopback for auth purposes but still need an explicit `--port` when launching a server, or the call is refused. - Which engines actually run on the user's machine is stated in the host facts the CLI appends to this prompt. Answer from those facts; do not assume a platform. Never suggest CPU fallback. - On native Windows, vLLM serving/install live checks are skipped; tell the user to use WSL/Linux for that ROCm GPU engine and do not suggest CPU fallback. diff --git a/tests/e2e-cucumber/features/dash.feature b/tests/e2e-cucumber/features/dash.feature index 524b3ee43..6191252d6 100644 --- a/tests/e2e-cucumber/features/dash.feature +++ b/tests/e2e-cucumber/features/dash.feature @@ -639,3 +639,29 @@ Feature: Interactive dashboard When the user closes the Diagnose screen And the user quits the dashboard Then the dashboard exits successfully + + @id:dash-serve-wizard-opens-collapsed @requires-os:linux + Scenario: dash-38 - Opening the serve wizard shows only the model, Advanced settings, and Launch + When the user opens the dashboard with demo data + And the user opens the Observe view + And the user opens the serve wizard + Then the serve wizard shows only the model, Advanced settings, and Launch + When the user presses Escape + Then the serve wizard is closed + When the user quits the dashboard + Then the dashboard exits successfully + + @id:dash-serve-wizard-escape-cancels-edit-without-closing @requires-os:linux + Scenario: dash-39 - Escape cancels an in-progress host edit without closing the serve wizard + When the user opens the dashboard with demo data + And the user opens the Observe view + And the user opens the serve wizard + And the user expands the serve wizard's Advanced settings + And the user starts editing the serve wizard's Host field + And the user types "nope" into the serve wizard + And the user presses Escape + Then the serve wizard is still open with the host unchanged + When the user presses Escape + Then the serve wizard is closed + When the user quits the dashboard + Then the dashboard exits successfully diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index ec8223478..4ee0a6b9a 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -34,7 +34,7 @@ Feature: Model serving # endpoint must advance to the next candidate rather than failing late in the # engine. Runs in the merge queue because it launches a real model server. @id:serve-auto-port-skips-occupied @requires-gpu @merge-queue - Scenario: 15 - Automatic serving skips an occupied default port + Scenario: serve-05 - Automatic serving skips an occupied default port Given a managed runtime is active And the default serve port is occupied When the user serves a model with automatic port selection @@ -48,7 +48,7 @@ Feature: Model serving # (`serve-default-engine-working-endpoint`, `serve-default-engine-inference`, # and `serve-readiness-contract`) run only in the merge queue (not on MI300X). @id:serve-vllm-inference @requires-gpu @requires-engine:vllm - Scenario: serve-05 - A served model responds to inference requests on vLLM + Scenario: serve-06 - A served model responds to inference requests on vLLM Given a managed runtime is active And a model is being served on GPU When the user sends a chat completion request @@ -63,7 +63,7 @@ Feature: Model serving # ordinary per-PR path, and the longer readiness timeout also gives the first # inference request enough time to complete. @id:serve-large-model-inference @requires-gpu @serve-timeout:2400 @nightly - Scenario: serve-06 - A large platform-specific model serves and responds to inference + Scenario: serve-07 - A large platform-specific model serves and responds to inference Given a managed runtime is active And a large model is being served on GPU When the user sends a chat completion request @@ -74,7 +74,7 @@ Feature: Model serving # lemonade per-PR canary: one real lemonade serve runs on every PR (the # counterpart to the vLLM canary above). @id:serve-lemonade-inference @requires-gpu @requires-engine:lemonade - Scenario: serve-07 - A model served on lemonade responds to inference requests + Scenario: serve-08 - A model served on lemonade responds to inference requests Given a managed runtime is active And a GGUF model is being served on lemonade When the user sends a chat completion request @@ -90,7 +90,7 @@ Feature: Model serving # `serve-lemonade-inference` (cache-shared, no extra download) so this stays a # fast per-PR canary rather than needing the @nightly large-checkpoint path. @id:serve-hf-checkpoint-inference @requires-gpu @requires-engine:lemonade - Scenario: serve-08 - A canonical Hugging Face checkpoint serves and responds to inference + Scenario: serve-09 - A canonical Hugging Face checkpoint serves and responds to inference Given a managed runtime is active And a canonical Hugging Face GGUF checkpoint is being served on lemonade When the user sends a chat completion request @@ -101,15 +101,15 @@ Feature: Model serving # default from the capability probe, so this covers whichever engine the host # would actually pick. @id:serve-default-engine-working-endpoint @requires-gpu @merge-queue - Scenario: serve-09 - Serving a model without specifying an engine produces a working endpoint + Scenario: serve-10 - Serving a model without specifying an engine produces a working endpoint Given a managed runtime is active When the user serves a model without specifying an engine Then an engine is selected automatically And the model is reachable - # The inference half of serve-09. + # The inference half of serve-10. @id:serve-default-engine-inference @requires-gpu @merge-queue - Scenario: serve-10 - A default-engine served model responds to inference requests + Scenario: serve-11 - A default-engine served model responds to inference requests Given a managed runtime is active When the user serves a model without specifying an engine Then the model responds to inference requests @@ -121,7 +121,7 @@ Feature: Model serving # skips it on lemonade-default hosts (Strix Halo), where asserting a vLLM # default would be a guaranteed false failure. @id:serve-vllm-default-on-instinct @requires-gpu @requires-engine:vllm - Scenario: serve-11 - vLLM is the default serving engine on Instinct + Scenario: serve-12 - vLLM is the default serving engine on Instinct Given a managed runtime is active When the user serves a vLLM-capable model without specifying an engine Then vLLM is selected as the default engine @@ -132,7 +132,7 @@ Feature: Model serving # platform that runs it (the merge queue skips MI300X). Readiness is gated on a real inference probe, which is what makes # this contract hold rather than race the model load. @id:serve-readiness-contract @requires-gpu @merge-queue - Scenario: serve-12 - A service reported ready can immediately serve inference + Scenario: serve-13 - A service reported ready can immediately serve inference Given a managed runtime is active And a model is being served on GPU When the CLI reports the service as ready @@ -143,7 +143,7 @@ Feature: Model serving # launched — with an actionable message, never a CPU or device-0 fallback. Runs # on the no-GPU mock host, so it gates every PR (@requires-no-gpu, no GPU needed). @id:serve-no-gpu-fails-fast @requires-no-gpu - Scenario: serve-13 - Serving is refused on a host with no AMD GPU + Scenario: serve-14 - Serving is refused on a host with no AMD GPU When the user serves a model under the GPU-required default Then serving is refused before any engine starts And the user is told no AMD GPU was detected @@ -153,7 +153,7 @@ Feature: Model serving # check runs inside argument parsing, before engine selection or any GPU # pre-flight, so it needs no GPU and no engine and gates every PR (ungated). @id:serve-negative-temperature-rejected - Scenario: serve-14 - Serving with a temperature below zero is refused with a clear reason + Scenario: serve-15 - Serving with a temperature below zero is refused with a clear reason When the user serves a model with a negative sampling temperature Then serving is refused before any engine starts And the CLI explains that temperature cannot be negative @@ -162,7 +162,7 @@ Feature: Model serving # GPU-required serve must treat it as "no GPU" and refuse, not fall back. Runs on # GPU hardware (Strix Halo / Instinct). @id:serve-masked-devices-fail @requires-gpu @requires-os:linux - Scenario: serve-15 - Serving is refused when every GPU is masked from view + Scenario: serve-16 - Serving is refused when every GPU is masked from view When the user serves a model with every GPU masked from view Then serving is refused before any engine starts And the user is told no AMD GPU was detected @@ -175,12 +175,12 @@ Feature: Model serving # # This is the live coverage for an index the host does not have. The *masked* # sub-case — an index that exists but is hidden by an active visibility mask — - # is a different refusal and owns its own scenarios, serve-19 and serve-20. + # is a different refusal and owns its own scenarios, serve-20 and serve-21. # The step here accepts several refusal wordings only because which authority # answers depends on the lane: the visible set when it could be enumerated, # the raw amd-smi count when it could not. @id:serve-absent-gpu-index-rejected @requires-gpu @requires-os:linux - Scenario: serve-16 - Serving pinned to a GPU that does not exist is refused + Scenario: serve-17 - Serving pinned to a GPU that does not exist is refused When the user serves a model pinned to a GPU index that does not exist Then serving is refused before any engine starts And the user is told that GPU index is unavailable @@ -190,7 +190,7 @@ Feature: Model serving # during argument parsing — before any engine or GPU work. No device needed, so # this runs on the mock lane every PR. @id:serve-runtime-and-env-selectors-conflict - Scenario: serve-17 - Selecting both a runtime and an environment at once is refused + Scenario: serve-18 - Selecting both a runtime and an environment at once is refused When the user serves a model selecting both a runtime and an environment Then serving is refused before any engine starts And the user is told the two selectors cannot be combined @@ -201,7 +201,7 @@ Feature: Model serving # deterministic on the blocking no-GPU lane rather than relying on a real 3 GiB # transfer to fail at just the right moment. @id:serve-lemonade-preparation-recovery @requires-no-gpu - Scenario: serve-18 - Repeated Lemonade preparation failure gives the user a recovery path + Scenario: serve-19 - Repeated Lemonade preparation failure gives the user a recovery path Given Lemonade preparation cannot complete When the user serves a model with Lemonade Then serving stops after one automatic retry @@ -216,12 +216,12 @@ Feature: Model serving # `HIP_VISIBLE_DEVICES=0` is in range but masked, exercising the visible-set # rejection directly. It holds on a single-GPU host too, so this stays ungated: # the mask names device 0, which exists there, so the visible set still resolves - # to [0] and ordinal 1 is refused against it. (Contrast serve-20, whose mask + # to [0] and ordinal 1 is refused against it. (Contrast serve-21, whose mask # names a device a single-GPU host does not have — see the note there.) Either # way an honest refusal rather than a remap, which is what this asserts. Runs on # GPU hardware. @id:serve-masked-gpu-index-rejected @requires-gpu @requires-os:linux - Scenario: serve-19 - Serving pinned to a GPU hidden by the visibility mask is refused + Scenario: serve-20 - Serving pinned to a GPU hidden by the visibility mask is refused When the user serves a model pinned to a GPU hidden by the visibility mask Then serving is refused before any engine starts And the user is told the pinned GPU is unavailable @@ -251,11 +251,11 @@ Feature: Model serving # outcome on one GPU turns on whether amd-smi happens to be present, and only a # second device makes the refusal follow from the mask itself. The permissive # unknown-mask fallback is deliberate (an unprobeable host must not be blocked - # from serving) and EAI-7194 does not change it. Sibling scenario serve-19 uses + # from serving) and EAI-7194 does not change it. Sibling scenario serve-20 uses # a `HIP_VISIBLE_DEVICES` mask naming a device that DOES exist, so its visible # set resolves on one GPU too and it stays ungated. @id:serve-rocr-reindexed-gpu-index-rejected @requires-gpu @requires-multi-gpu @requires-os:linux - Scenario: serve-20 - Serving pinned past the ROCR-reindexed visible set is refused + Scenario: serve-21 - Serving pinned past the ROCR-reindexed visible set is refused When the user serves a model pinned past the ROCR-reindexed visible set Then serving is refused before any engine starts And the user is told the pinned GPU is unavailable @@ -264,13 +264,13 @@ Feature: Model serving # alternatives. ROCr applies first and HIP only re-indexes and selects among the # survivors, so `ROCR_VISIBLE_DEVICES=` (hide everything) leaves HIP nothing to # see and `HIP_VISIBLE_DEVICES=0` cannot bring a device back. The GPU-required - # serve must therefore refuse, exactly as serve-15 does for a HIP-only empty + # serve must therefore refuse, exactly as serve-16 does for a HIP-only empty # mask. Before the fix the probe preferred the HIP mask and discarded the ROCR # one entirely, resolved the visible set to [0], and let the serve proceed onto a # device the runtime had already hidden. Runs on GPU hardware: on a no-GPU host # the same refusal fires for having no device at all and would prove nothing. @id:serve-rocr-empty-mask-beats-hip-mask @requires-gpu @requires-os:linux - Scenario: serve-21 - Serving is refused when ROCR hides every GPU a HIP mask names + Scenario: serve-22 - Serving is refused when ROCR hides every GPU a HIP mask names When the user serves a model with ROCR hiding every GPU a HIP mask names Then serving is refused before any engine starts And the user is told no AMD GPU was detected @@ -283,7 +283,7 @@ Feature: Model serving # what it hides and names the two commands that reach it. No GPU and no real # serve: the record is planted, so this runs on every lane. @id:serve-past-attempts-surfaced - Scenario: serve-22 - A failed local server is counted in the default list + Scenario: serve-23 - A failed local server is counted in the default list Given a local server attempt has failed When the user lists running services Then the list reports the attempt and how to look at it @@ -301,7 +301,7 @@ Feature: Model serving # runtime is active — hence the same `a managed runtime is active` precondition # every other real-serve scenario on this lane opens with. @id:serve-vllm-low-vram-oom-guidance @requires-gpu @requires-engine:vllm @requires-os:linux - Scenario: serve-23 - A vLLM serve plan on a nearly-full GPU points at the memory knob + Scenario: serve-24 - A vLLM serve plan on a nearly-full GPU points at the memory knob Given a managed runtime is active And the selected GPU is reported nearly out of VRAM When the user previews a vLLM serve plan pinned to that GPU @@ -327,7 +327,7 @@ Feature: Model serving # Windows detached-spawn primitive), so this scenario cannot reach the # refusal it exercises there. @id:serve-require-api-key-refuses-unauthenticated-reuse @requires-gpu @requires-engine:lemonade @requires-os:linux - Scenario: serve-24 - Reusing a running server without a key refuses a --require-api-key request + Scenario: serve-25 - Reusing a running server without a key refuses a --require-api-key request Given a managed runtime is active And an unauthenticated Lemonade service is already running for that model When the user serves that same model again with --require-api-key @@ -349,16 +349,16 @@ Feature: Model serving # match a narrower wording would make this scenario weaker than the # whole-tree contract in xtask/src/ascii_output_contract.rs that it mirrors. @id:serve-model-list-output-is-ascii - Scenario: serve-25 - The recommended model list prints ASCII-only text + Scenario: serve-26 - The recommended model list prints ASCII-only text When the user lists recommended models Then the model list output contains only ASCII characters # Parse-time refusal (serve-port rewrite): an out-of-range `--port` (0 here; # 65536 and non-numeric values share the same parser and message) is rejected # before any engine or GPU work, so it needs no GPU and gates every PR - # (ungated), like its sibling serve-14. + # (ungated), like its sibling serve-15. @id:serve-invalid-port-rejected - Scenario: serve-26 - Serving with an out-of-range port is refused with a clear reason + Scenario: serve-27 - Serving with an out-of-range port is refused with a clear reason When the user serves a model with an out-of-range port Then serving is refused before any engine starts And the user is told the port must be between 1 and 65535 @@ -375,9 +375,27 @@ Feature: Model serving # apps/rocm/src/service_launch.rs, per AGENTS.md section 3. # Runs in the merge queue, like its automatic-port sibling serve-auto-port-skips-occupied. @id:serve-explicit-port-reserved @requires-gpu @merge-queue - Scenario: serve-27 - Serving on a port another managed service already holds is refused + Scenario: serve-28 - Serving on a port another managed service already holds is refused Given a managed runtime is active And another managed service already reserves the default serve port When the user serves a model explicitly on the reserved port Then serving is refused before any engine starts And the user is told to stop the service holding that port + + # Collision-safe explicit ports, untracked-process case (serve-port rewrite, + # sibling of serve-28). No managed-service record exists here, so + # `resolve_serve_port` never reaches the "reserved by managed service" branch + # and instead gets a plain OS-level `AddrInUse` back from the bind probe, + # routing to `explicit_bind_error`'s "already in use by another process" + # message. Needs the same `@requires-gpu` as serve-28 for the same reason: + # `resolve_serve_port` only runs once `spawn_managed_engine_child` has a + # completed model/GPU resolution in hand (GPU indices are selected earlier in + # `serve_cmd.rs`, under the launch lock, before any port is ever probed), and + # `serve` has no CPU fallback that gets there. + @id:serve-explicit-port-in-use-by-untracked-process @requires-gpu @merge-queue + Scenario: serve-29 - Serving on a port an untracked process already holds is refused + Given a managed runtime is active + And the default serve port is occupied + When the user serves a model explicitly on the occupied port + Then serving is refused before any engine starts + And the user is told the port is already in use by another process diff --git a/tests/e2e-cucumber/tests/e2e/dash_steps.rs b/tests/e2e-cucumber/tests/e2e/dash_steps.rs index 4a9d1aefd..a81727a24 100644 --- a/tests/e2e-cucumber/tests/e2e/dash_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/dash_steps.rs @@ -503,6 +503,98 @@ async fn choose_serving(world: &mut E2eWorld) { .unwrap_or_else(|e| panic!("failed to open Serving: {e}")); } +#[when("the user opens the serve wizard")] +async fn open_serve_wizard(world: &mut E2eWorld) { + // `w` opens the serve wizard, but only from the Observe tab (`actions.rs`), + // so the preceding `the user opens the Observe view` step is load-bearing + // — it also proves the event loop is reading input, the same way + // `open_services_overlay` relies on it for `s`. + let tui = session(world); + tui.send("w") + .unwrap_or_else(|e| panic!("failed to open the serve wizard: {e}")); + tui.wait_for_screen("Serve a model", default_timeout()) + .await + .unwrap_or_else(|e| panic!("serve wizard did not open: {e}")); +} + +#[then("the serve wizard shows only the model, Advanced settings, and Launch")] +async fn serve_wizard_shows_only_basic_fields(world: &mut E2eWorld) { + let screen = session(world).screen_text(); + assert!( + screen.contains("Model ") + && screen.contains("Advanced settings ▸ Automatic") + && screen.contains("[ Launch ]"), + "serve wizard is missing one of its collapsed-view fields:\n{screen}" + ); + assert!( + !screen.contains("Engine ") + && !screen.contains("Device ") + && !screen.contains("Host ") + && !screen.contains("Port ") + && !screen.contains("Mode "), + "serve wizard shows an advanced field while collapsed:\n{screen}" + ); +} + +// `Field::Model` is at index 0 of the collapsed `BASIC_FIELDS`, so one Down +// reaches `Field::Advanced`. Arrow keys are sent as their raw VT100 escape +// sequences, same as a real terminal would report them; the wizard's `on_key` +// only matches literal `KeyCode::Up`/`Down`, not vim-style `j`/`k`, so those +// would be typed straight into the focused field instead of moving focus. +#[when("the user expands the serve wizard's Advanced settings")] +async fn expand_serve_wizard_advanced(world: &mut E2eWorld) { + let tui = session(world); + tui.send("\x1b[B") + .unwrap_or_else(|e| panic!("failed to move to Advanced settings: {e}")); + tui.send("\r") + .unwrap_or_else(|e| panic!("failed to expand Advanced settings: {e}")); + tui.wait_for_screen("Advanced settings ▾", default_timeout()) + .await + .unwrap_or_else(|e| panic!("Advanced settings did not expand: {e}")); +} + +// With Advanced expanded, focus is still on `Field::Advanced` (index 1 of the +// full `FIELDS` order); three more Downs walk Engine → Device → Host. +#[when("the user starts editing the serve wizard's Host field")] +async fn start_editing_serve_wizard_host(world: &mut E2eWorld) { + let tui = session(world); + for _ in 0..3 { + tui.send("\x1b[B") + .unwrap_or_else(|e| panic!("failed to move toward the Host field: {e}")); + } + tui.send("\r") + .unwrap_or_else(|e| panic!("failed to start editing the Host field: {e}")); + tui.wait_for_screen("editing host", default_timeout()) + .await + .unwrap_or_else(|e| panic!("Host field editor did not open: {e}")); +} + +#[when(expr = "the user types {string} into the serve wizard")] +async fn type_into_serve_wizard(world: &mut E2eWorld, text: String) { + session(world) + .send(&text) + .unwrap_or_else(|e| panic!("failed to type {text:?} into the serve wizard: {e}")); +} + +#[then("the serve wizard is still open with the host unchanged")] +async fn serve_wizard_still_open_host_unchanged(world: &mut E2eWorld) { + let screen = session(world).screen_text(); + assert!( + screen.contains("Serve a model") + && screen.contains("[127.0.0.1]") + && !screen.contains("editing host"), + "Escape did not cleanly cancel the Host edit:\n{screen}" + ); +} + +#[then("the serve wizard is closed")] +async fn serve_wizard_closed(world: &mut E2eWorld) { + session(world) + .wait_until_gone("Serve a model", default_timeout()) + .await + .unwrap_or_else(|e| panic!("the serve wizard is still open after Escape: {e}")); +} + #[when("the user opens onboarding setup")] async fn open_onboarding_setup(world: &mut E2eWorld) { // `n` opens the onboarding wizard from the Observe tab diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 7d1a1891f..cadc537bd 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -1132,6 +1132,31 @@ async fn user_serves_explicit_reserved_port(world: &mut E2eWorld) { world.cli_rc = Some(rc); } +/// Same CLI invocation as [`user_serves_explicit_reserved_port`]; the two +/// scenarios differ only in what is holding the port (a plain listener here, +/// vs. a live managed-service record there), which `resolve_serve_port` +/// routes to two different refusal messages. +#[when("the user serves a model explicitly on the occupied port")] +async fn user_serves_explicit_occupied_port(world: &mut E2eWorld) { + let (model, engine, _) = host_serve_target(); + let port = SERVE_PORT.to_string(); + let (stdout, stderr, rc) = crate::run_rocm( + world, + &[ + "serve", + model, + "--engine", + engine, + "--managed", + "--port", + &port, + ], + ); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + /// Serve naming both a runtime and an environment. These are mutually exclusive /// selectors, rejected during argument parsing before any engine or GPU work, so /// the refusal is observable with no GPU (mock lane). @@ -1253,6 +1278,17 @@ async fn assert_port_reserved_message(world: &mut E2eWorld) { ); } +#[then("the user is told the port is already in use by another process")] +async fn assert_port_in_use_message(world: &mut E2eWorld) { + let output = serve_output(world); + assert!( + output.contains(&format!( + "127.0.0.1:{SERVE_PORT} is already in use by another process" + )) && output.contains("Choose a free `--port`"), + "expected the port-in-use message, got:\n{output}" + ); +} + #[then("the user is told the two selectors cannot be combined")] async fn assert_selector_conflict_message(world: &mut E2eWorld) { let output = serve_output(world); @@ -1562,10 +1598,18 @@ async fn assert_next_automatic_port(world: &mut E2eWorld) { ) ); let expected = format!("endpoint: http://127.0.0.1:{}/v1", SERVE_PORT + 1); + let disclosure_line = stdout + .lines() + .position(|line| line.trim() == "Port: automatic; endpoint shown after launch"); + let endpoint_line = stdout.lines().position(|line| line.trim() == expected); assert!( - stdout.lines().any(|line| line.trim() == expected), + endpoint_line.is_some(), "expected `{expected}` in real serve output:\n{stdout}" ); + assert!( + disclosure_line.is_some_and(|d| d < endpoint_line.unwrap()), + "the automatic-port disclosure must print before the resolved endpoint:\n{stdout}" + ); } /// Extract the engine name from a serve plan's `engine: ` line. From fea5e5376daa9bdbd532ca4e1c37f3279d14b89b Mon Sep 17 00:00:00 2001 From: Juho Vainio Date: Mon, 26 Oct 2026 16:17:59 +0200 Subject: [PATCH 14/19] fix(serve): address review 5651221433 on port policy, tests and docs - serve(): reject automatic port on a custom host right after validate_bind_host, before any GPU or engine work (networking-04). - dash e2e: wait for the screen to settle before asserting in dash-38/39. - rocmd tests: gate the Linux-only recovery test; drive the rollback tests through the spawn seam instead of racing a thread; retry the lease-release bind briefly for sibling-fork races. - Add a spawn seam to the CLI restart path and tests for lease release, recovery self-exclusion, the rollback merge and the spawn-failure rollback. - rocmd recovery: roll the recovering record back when the spawn or try_wait fails, so it stops reserving the port. - Share one launch_port_argument validator (rocm-core) between chat and MCP launch_server, with malformed-port tests. - Restore the original serve-NN numbering and append the new scenarios as serve-26..29, fixing the stale references. - Dash wizard: Esc on a host-editor error no longer surfaces or focuses another field's complaint. - Use ephemeral ports in tests that bind for real; share the FileLock opener; fix misplaced and duplicated doc comments. Signed-off-by: Juho Vainio --- apps/rocm/src/chat.rs | 21 +- apps/rocm/src/main.rs | 100 +++++- apps/rocm/src/serve_cmd.rs | 1 + apps/rocm/src/serve_port.rs | 37 +- apps/rocm/src/service_launch.rs | 26 +- apps/rocmd/src/mcp.rs | 51 ++- apps/rocmd/src/test_support.rs | 12 + apps/rocmd/src/watchers.rs | 337 ++++++++++++------ crates/rocm-core/src/lib.rs | 5 +- crates/rocm-core/src/process.rs | 96 ++++- crates/rocm-dash-tui/src/ui/serve_wizard.rs | 50 ++- .../features/model_serving.feature | 86 ++--- tests/e2e-cucumber/tests/e2e/dash_steps.rs | 12 +- 13 files changed, 573 insertions(+), 261 deletions(-) diff --git a/apps/rocm/src/chat.rs b/apps/rocm/src/chat.rs index c2f184ef0..2947f2622 100644 --- a/apps/rocm/src/chat.rs +++ b/apps/rocm/src/chat.rs @@ -2235,23 +2235,10 @@ fn validate_chat_launch_server_tool_call(call: &providers::ChatToolCall) -> Resu if !is_loopback_host(&host) { bail!("local assistant cannot request non-local host `{host}`"); } - // Validate the same way the argv builder reads it - // (`object.get("port").and_then(serde_json::Value::as_u64)`), so a - // malformed `port` (wrong type, out of range) can't slip past this check - // only to have the argv builder silently drop it and fall through to - // automatic port selection. - let port = match object.get("port") { - None | Some(serde_json::Value::Null) => None, - Some(value) => { - let port = value - .as_u64() - .context("ROCm tool `launch_server` argument `port` must be an integer")?; - if !(1..=u64::from(u16::MAX)).contains(&port) { - bail!("ROCm tool `launch_server` argument `port` must be between 1 and 65535"); - } - Some(port) - } - }; + // A malformed `port` must be rejected here, not silently dropped by the + // argv builder and left to fall through to automatic port selection. + let port = rocm_core::launch_port_argument(object.get("port")) + .map_err(|reason| anyhow::anyhow!("ROCm tool `launch_server` argument `port` {reason}"))?; // `rocm serve` proves a port is free only on the exact default host; every // other spelling (including the loopback aliases `localhost`/`::1` this // check still accepts) needs an explicit port or the CLI call refuses diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index de5231fb2..f435ba85a 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -7221,6 +7221,24 @@ fn restart_internal_managed_service_with( runtime_pin: Option<&str>, terminate: impl FnOnce(&ManagedServiceRecord) -> (Vec, bool), ) -> Result { + restart_internal_managed_service_inner(paths, service_id, runtime_pin, terminate, None) +} + +/// Replaces the child `spawn()` of a restart (non-Windows only; Windows spawns +/// through `rocm_core::spawn_detached_no_inherit`). Test-only seam, so a test +/// can observe the leased port at the exact instant production would spawn. +type RestartSpawnOverride<'a> = + dyn Fn(&mut std::process::Command) -> io::Result + 'a; + +fn restart_internal_managed_service_inner( + paths: &AppPaths, + service_id: &str, + runtime_pin: Option<&str>, + terminate: impl FnOnce(&ManagedServiceRecord) -> (Vec, bool), + spawn_override: Option<&RestartSpawnOverride<'_>>, +) -> Result { + #[cfg(windows)] + let _ = spawn_override; let record = load_service_for_restart(paths, service_id, runtime_pin)?; // Preserve the endpoint key across the restart: `stop_internal_managed_service` // deletes the key file (correct on a real stop), but a restart must bring the @@ -7353,9 +7371,11 @@ fn restart_internal_managed_service_with( if let Some(key_file) = endpoint_key_file.as_deref() { command.env(rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV, key_file); } - let mut child = command - .spawn() - .context("failed to restart managed engine process")?; + let mut child = match spawn_override { + Some(spawn) => spawn(&mut command), + None => command.spawn(), + } + .context("failed to restart managed engine process")?; thread::sleep(Duration::from_millis(200)); if let Some(status) = child .try_wait() @@ -17741,6 +17761,55 @@ pub(crate) mod tests { Ok(()) } + /// Mirrors `launch_releases_the_leased_port_before_spawning_the_child` and + /// its rocmd twin: the restart's probe lease must be closed by the time the + /// child is spawned, or the child's own bind of the port fails. Without + /// the `lease.release()` call every loopback restart would die at the + /// engine bind, and nothing else here would notice. + #[cfg(not(windows))] + #[test] + fn restart_releases_the_leased_port_before_spawning_the_child() -> Result<()> { + let (root, paths) = test_paths("restart-releases-lease"); + paths.ensure()?; + let service_id = "svc-restart-releases-lease"; + let record = stop_test_record(&paths, service_id); + let port = record.port; + record.write()?; + + let bind_succeeded = std::cell::Cell::new(false); + let spawn: &RestartSpawnOverride<'_> = &|_command| { + // Retry briefly: a sibling test's concurrent fork can hold a copy of + // the just-released socket until its exec closes it. + let deadline = std::time::Instant::now() + Duration::from_secs(2); + while !bind_succeeded.get() && std::time::Instant::now() < deadline { + bind_succeeded.set(std::net::TcpListener::bind(("127.0.0.1", port)).is_ok()); + } + // Fail the spawn on purpose: the observation above is all this + // test needs, and no real engine should start. + Err(io::Error::other("spawn intercepted by test")) + }; + let result = restart_internal_managed_service_inner( + &paths, + service_id, + None, + |_| (Vec::new(), true), + Some(spawn), + ); + let _ = fs::remove_dir_all(root); + + assert!( + bind_succeeded.get(), + "the leased port must already be bindable (released) by the time the child is spawned" + ); + let rendered = format!("{:#}", result.expect_err("the intercepted spawn must fail")); + assert!( + rendered.contains("spawn intercepted by test"), + "the restart must have reached the spawn: {rendered}" + ); + Ok(()) + } + + #[cfg(not(windows))] #[test] fn restart_is_not_blocked_by_a_stopped_or_failed_record_on_the_same_port() -> Result<()> { // Mirror of `restart_refuses_a_port_a_launch_claimed...`: `blocks_port` is @@ -17791,17 +17860,24 @@ pub(crate) mod tests { failed_sibling.status = "failed".to_owned(); failed_sibling.write()?; - let result = - restart_internal_managed_service_with(&paths, service_id, None, |_| (Vec::new(), true)); + // Fail the spawn on purpose: reaching it is what proves neither sibling + // blocked the restart, so any other error is a real failure here. + let spawn: &RestartSpawnOverride<'_> = + &|_command| Err(io::Error::other("spawn intercepted by test")); + let result = restart_internal_managed_service_inner( + &paths, + service_id, + None, + |_| (Vec::new(), true), + Some(spawn), + ); let _ = fs::remove_dir_all(root); - if let Err(error) = &result { - let rendered = format!("{error:#}"); - assert!( - !rendered.contains("claimed that port"), - "a stopped or failed sibling on the same port must not be treated as a collision: {rendered}" - ); - } + let rendered = format!("{:#}", result.expect_err("the intercepted spawn must fail")); + assert!( + rendered.contains("spawn intercepted by test"), + "a stopped or failed sibling on the same port must not block the restart: {rendered}" + ); Ok(()) } diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index 6f496faa0..ffcf2cfef 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -543,6 +543,7 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { let port_request = serve_port::PortRequest::from_flag(port); let _ = managed; // background is now the default; --managed is accepted as an explicit synonym. validate_bind_host(&host, allow_public_bind)?; + serve_port::validate_port_request(&host, port_request)?; // Loopback stays credential-free; a public bind must be authenticated. Resolve // (or generate) the endpoint key now so every downstream path — engine spawn, // readiness probe, smoke test, and the client-config we print — shares one value. diff --git a/apps/rocm/src/serve_port.rs b/apps/rocm/src/serve_port.rs index 291130ddf..200147f21 100644 --- a/apps/rocm/src/serve_port.rs +++ b/apps/rocm/src/serve_port.rs @@ -485,27 +485,28 @@ mod tests { #[test] fn automatic_selects_against_real_loopback_listeners() { - // Real sockets, not a script: hold the first two candidates with live - // listeners and prove the production probe walks past them. If either is - // already owned by something outside this test the scenario cannot be set - // up deterministically, so skip rather than assert on a moving target. - let (Ok(first), Ok(second)) = ( - TcpListener::bind(("127.0.0.1", AUTO_PORT_FIRST)), - TcpListener::bind(("127.0.0.1", AUTO_PORT_FIRST + 1)), - ) else { - return; + // Real sockets, not a script: two live listeners hold the first two + // candidates and the production probe must walk past them. The + // candidates are remapped onto ephemeral ports so a real `rocm serve` + // sitting in 11435-11535 on this machine can neither fail nor + // vacuously skip the test. + let first = TcpListener::bind(("127.0.0.1", 0)).expect("bind"); + let second = TcpListener::bind(("127.0.0.1", 0)).expect("bind"); + let free = TcpListener::bind(("127.0.0.1", 0)).expect("bind"); + let targets = [ + first.local_addr().expect("addr").port(), + second.local_addr().expect("addr").port(), + free.local_addr().expect("addr").port(), + ]; + drop(free); + let mut probe = |candidate: u16| { + let index = usize::from(candidate - AUTO_PORT_FIRST); + loopback_bind_probe(targets[index.min(targets.len() - 1)]) }; - let mut probe = loopback_bind_probe; let lease = resolve_serve_port(DEFAULT_LOCAL_HOST, PortRequest::Auto, &[], &mut probe) - .expect("a candidate above the held ones is free"); - assert!( - lease.port() >= AUTO_PORT_FIRST + 2, - "selected {} while {AUTO_PORT_FIRST} and {} were held", - lease.port(), - AUTO_PORT_FIRST + 1 - ); + .expect("the third candidate is free"); + assert_eq!(lease.port(), targets[2], "must walk past both held ports"); assert!(lease.is_held(), "the production probe must hold the socket"); - // Guards outlive every assertion above. drop((first, second)); } } diff --git a/apps/rocm/src/service_launch.rs b/apps/rocm/src/service_launch.rs index 4f6548a38..4512bb97a 100644 --- a/apps/rocm/src/service_launch.rs +++ b/apps/rocm/src/service_launch.rs @@ -5,12 +5,13 @@ //! Spawn, attach to, and stream logs from a managed background service //! process, plus the endpoint-key/auth checks that gate it. //! -//! Mechanically relocated from `main.rs` with no behavior change. This module +//! Relocated from `main.rs`, since extended with the port-allocation and +//! launch-lock transaction. This module //! is the managed-service-spawning tail that was deliberately deferred out of //! the Phase 6a (ROCMAI-82) split: `start_managed_service`, //! `run_attached_service`, `spawn_managed_engine_child`, `stream_attached_logs`, //! and the rest of this file. It was entangled with crate-root launch helpers -//! — `record_cli_audit_event`, `existing_live_managed_service`, and the stop +//! — `record_cli_audit_event`, `existing_live_managed_service_in`, and the stop //! action of the service-record-management cluster (this module reaches only //! stop, via `AttachOutcome::Stop`; restart/remove/prune are not called here) //! — and this later pass (ROCMAI-91, Phase 6b) separates the spawning tail out @@ -21,7 +22,7 @@ //! `ServicesCommand`/`services()` stay in `main.rs` (a dedicated clap //! subcommand's enum and dispatch function usually do, per //! `docs/architecture.md`'s convention note). `record_cli_audit_event`, -//! `existing_live_managed_service`, and the broader "service record management" +//! `existing_live_managed_service_in`, and the broader "service record management" //! cluster (stop/restart/remove/prune, reached from `services()`; stop/restart //! are also reachable from the chat/LLM sandbox tool-calling path) stay at the //! crate root too — reached via `crate::`, same as other mechanically relocated @@ -622,6 +623,9 @@ fn spawn_managed_engine_child( } } +#[cfg(not(windows))] +type SpawnOverride<'a> = dyn Fn(&mut ProcessCommand) -> io::Result + 'a; + /// The side-effecting half of the launch transaction: create the log, prepare /// concrete argv, hand the leased port to the child, spawn it, reject a child /// that has already exited, and publish the running record. @@ -629,9 +633,6 @@ fn spawn_managed_engine_child( /// Split out of [`spawn_managed_engine_child`] so that *every* failure below the /// record's first write is funnelled through one cleanup path before the /// launch lock is released. Returns the child pid. -#[cfg(not(windows))] -type SpawnOverride<'a> = dyn Fn(&mut ProcessCommand) -> io::Result + 'a; - #[allow(clippy::too_many_arguments)] fn launch_managed_engine_child_locked( paths: &AppPaths, @@ -772,11 +773,6 @@ fn launch_managed_engine_child_locked( // alive: a stop pairs every recorded PID with its own token, and the two // PIDs diverge as soon as `rocmd` restarts the service — at which point an // engine PID recorded without one could only be signalled blind. - // The child is both roles until the engine state file names a separate - // server process. Each role gets its own token, captured while the child is - // alive: a stop pairs every recorded PID with its own token, and the two - // PIDs diverge as soon as `rocmd` restarts the service — at which point an - // engine PID recorded without one could only be signalled blind. record.record_supervisor_identity(child_pid); record.record_engine_identity(child_pid); record.status = "running".to_owned(); @@ -2675,7 +2671,7 @@ mod tests { // Go through the real stop path (`rocm services stop --yes`'s own // implementation) rather than writing `status = "stopped"` to disk - // directly, so this test actually proves the advice serve-28 pins: that + // directly, so this test actually proves the advice serve-27 pins: that // running the stop command frees the port. The termination closure is // the same seam production injects a real signal through; here it just // confirms every recorded process gone without touching the test @@ -3039,7 +3035,11 @@ mod tests { paths.ensure()?; fs::create_dir_all(paths.services_dir())?; - let port = 11_599_u16; + // Ephemeral, not a fixed number in the automatic range, which a real + // `rocm serve` on the developer's machine could be holding. + let port = std::net::TcpListener::bind(("127.0.0.1", 0))? + .local_addr()? + .port(); let lease = rocm_core::lease_loopback_port(port)?; let mut record = ManagedServiceRecord::new( &paths, diff --git a/apps/rocmd/src/mcp.rs b/apps/rocmd/src/mcp.rs index ab0d4a679..8a3526b94 100644 --- a/apps/rocmd/src/mcp.rs +++ b/apps/rocmd/src/mcp.rs @@ -1094,23 +1094,10 @@ fn build_launch_server_args(arguments: &serde_json::Map) -> Resul "launch_server host `{host}` is not loopback; require `allow_public_bind=true` before binding a non-local interface" ); } - // Validate the same way the argv builder reads it - // (`arguments.get("port").and_then(Value::as_u64)`), so a malformed - // `port` (wrong type, out of range) can't slip past this check only to - // have the argv builder silently drop it and fall through to automatic - // port selection. - let port = match arguments.get("port") { - None | Some(Value::Null) => None, - Some(value) => { - let port = value - .as_u64() - .context("launch_server argument `port` must be an integer")?; - if !(1..=u64::from(u16::MAX)).contains(&port) { - bail!("launch_server argument `port` must be between 1 and 65535"); - } - Some(port) - } - }; + // A malformed `port` must be rejected here, not silently dropped by the + // argv builder and left to fall through to automatic port selection. + let port = rocm_core::launch_port_argument(arguments.get("port")) + .map_err(|reason| anyhow::anyhow!("launch_server argument `port` {reason}"))?; // `rocm serve` proves a port is free only on the exact default host; every // other spelling (including the loopback aliases `localhost`/`::1` the // check above still accepts) needs an explicit port or the CLI call @@ -1466,6 +1453,36 @@ mod tests { assert!(error.to_string().contains("explicit `port`"), "{error:#}"); } + #[test] + fn launch_server_rejects_a_malformed_port_instead_of_choosing_one_automatically() { + // Each value would be dropped by a lenient argv builder and leave the + // launch to automatic port selection on a host that cannot do it. + for (port, expected) in [ + (Value::String("8000".to_owned()), "must be an integer"), + (Value::Number(0.into()), "between 1 and 65535"), + (Value::Number(65536.into()), "between 1 and 65535"), + ] { + let arguments = serde_json::Map::from_iter([ + ("model".to_owned(), Value::String("tiny-gpt2".to_owned())), + ("host".to_owned(), Value::String("localhost".to_owned())), + ("port".to_owned(), port.clone()), + ]); + let error = build_launch_server_args(&arguments).unwrap_err(); + assert!(error.to_string().contains(expected), "{port}: {error:#}"); + } + } + + #[test] + fn launch_server_treats_a_null_port_as_automatic() -> Result<()> { + let arguments = serde_json::Map::from_iter([ + ("model".to_owned(), Value::String("tiny-gpt2".to_owned())), + ("port".to_owned(), Value::Null), + ]); + let args = build_launch_server_args(&arguments)?; + assert!(!args.contains(&"--port".to_owned()), "{args:?}"); + Ok(()) + } + #[test] fn launch_server_allows_localhost_with_an_explicit_port() -> Result<()> { let arguments = serde_json::Map::from_iter([ diff --git a/apps/rocmd/src/test_support.rs b/apps/rocmd/src/test_support.rs index f1c000bfe..a20aea292 100644 --- a/apps/rocmd/src/test_support.rs +++ b/apps/rocmd/src/test_support.rs @@ -94,6 +94,18 @@ pub(crate) fn assert_each_call_gets_its_own_root(make_root: impl Fn() -> PathBuf ); } +/// A loopback port the OS just handed out and nothing holds now. +/// +/// For tests that make the code under test bind the port for real: a fixed +/// number inside the automatic range 11435-11535 fails whenever a real +/// `rocm serve` is running on the developer's machine. +pub(crate) fn free_loopback_port() -> u16 { + std::net::TcpListener::bind(("127.0.0.1", 0)) + .and_then(|listener| listener.local_addr()) + .expect("bind an ephemeral loopback port") + .port() +} + /// Build the record a supervisor would construct for `service_id`, to learn /// the paths it derives before the real call does. Not platform-gated: /// the supervisor exit-write tests use it on every platform. diff --git a/apps/rocmd/src/watchers.rs b/apps/rocmd/src/watchers.rs index ea4d91f46..2b7fab96d 100644 --- a/apps/rocmd/src/watchers.rs +++ b/apps/rocmd/src/watchers.rs @@ -1494,14 +1494,12 @@ pub(crate) fn restart_managed_service( restart_managed_service_with(paths, record, None) } -/// [`restart_managed_service`], with the supervisor spawn interceptable. -/// -/// `spawn_override`, when set, replaces the real `command.spawn()` call below -/// (falling back to it when `None`) — a test-only seam so a test can assert -/// the leased port is already bindable (released) at the exact instant -/// production would spawn, without faking the whole spawn. +/// Replaces the real `command.spawn()` of a recovery supervisor: a test-only +/// seam for asserting on state at the exact instant production would spawn. type SpawnOverride<'a> = dyn Fn(&mut ProcessCommand) -> std::io::Result + 'a; +/// [`restart_managed_service`], with the supervisor spawn interceptable via +/// `spawn_override` (the real `command.spawn()` when `None`). fn restart_managed_service_with( paths: &AppPaths, record: &mut ManagedServiceRecord, @@ -1564,12 +1562,11 @@ fn restart_managed_service_with( .try_clone() .context("failed to clone service log file handle")?; - // The truthful state to fall back to. The publish and - // `record_spawned_supervisor` failure paths below restore this rather than - // leave a half-applied `recovering` row, and the latter also makes sure a - // supervisor that exists is never left running unrecorded. The spawn and - // `try_wait` failure paths after that do not restore it: they return before - // any rollback runs, so `recovering` can still be left standing on disk. + // The truthful state to fall back to. Every failure after the `recovering` + // write below (spawn, `try_wait`, recording the supervisor) restores this + // through `fail_recovery` rather than leave a half-applied `recovering` row + // that would keep reserving the port, and a supervisor that exists is never + // left running unrecorded. let prior = record.clone(); record.status = "recovering".to_owned(); @@ -1620,42 +1617,55 @@ fn restart_managed_service_with( .stdin(Stdio::null()) .stdout(Stdio::from(log_file)) .stderr(Stdio::from(log_file_err)); - let mut child = match spawn_override { + let spawned = match spawn_override { Some(spawn) => spawn(&mut command), None => command.spawn(), - } - .context("failed to spawn recovery supervisor")?; + }; + let mut child = match spawned { + Ok(child) => child, + Err(error) => { + let error = anyhow::Error::new(error).context("failed to spawn recovery supervisor"); + return Err(fail_recovery( + paths, + record, + &recovering_snapshot, + prior, + error, + )); + } + }; if let Err(error) = record_spawned_supervisor(paths, record, child.id()) { // The supervisor is alive but unrecorded: it would hold the port while - // being invisible to every stop/recovery path. Kill it, then put the - // prior truthful fields back. Not a blind `record.write()` of `prior`: - // the same discipline as `update_supervised_record` elsewhere in this - // function applies here too, so a stop marker a concurrent stop - // persisted while the spawn was in flight is not reverted by this - // rollback. + // being invisible to every stop/recovery path. Kill it first. terminate_recovery_supervisor(&mut child); let error = error.context("failed to publish the recovering service record"); - if let Err(rollback) = rollback_recovering_record(paths, &recovering_snapshot, &prior) { - // Both writes failed: the on-disk record now still claims - // `recovering` for a supervisor that no longer exists. That is worse - // than the original failure and must not be swallowed — the watcher - // and the manual restart tool both surface whatever comes back. - return Err(error.context(format!( - "rollback also failed: managed service {} is left recorded as recovering \ - with no supervisor ({rollback:#})", - record.service_id - ))); - } - *record = prior; - return Err(error); + return Err(fail_recovery( + paths, + record, + &recovering_snapshot, + prior, + error, + )); } thread::sleep(Duration::from_millis(200)); - if let Some(status) = child - .try_wait() - .context("failed to check recovery supervisor startup state")? - { + let exited = match child.try_wait() { + Ok(exited) => exited, + Err(error) => { + terminate_recovery_supervisor(&mut child); + let error = anyhow::Error::new(error) + .context("failed to check recovery supervisor startup state"); + return Err(fail_recovery( + paths, + record, + &recovering_snapshot, + prior, + error, + )); + } + }; + if let Some(status) = exited { // Not a write-back of `record`: a stop issued since the spawn has // persisted its marker, and a `failed` record without one is what // recovery restarts. Same discipline as the supervisor's own writes. @@ -1671,8 +1681,33 @@ fn restart_managed_service_with( Ok(()) } -/// Record the supervisor [`restart_managed_service`] just spawned, as `pid`. +/// Undo [`restart_managed_service`]'s `recovering` publication after a later +/// step failed, and return the error to report. /// +/// Not a blind `record.write()` of `prior`: the same discipline as +/// `update_supervised_record` applies, so a stop marker a concurrent stop +/// persisted since the first write is not reverted. When the rollback itself +/// fails the on-disk record still claims `recovering` for a supervisor that no +/// longer exists, which is worse than the original failure and is reported with +/// it rather than swallowed. +fn fail_recovery( + paths: &AppPaths, + record: &mut ManagedServiceRecord, + recovering_snapshot: &ManagedServiceRecord, + prior: ManagedServiceRecord, + error: anyhow::Error, +) -> anyhow::Error { + if let Err(rollback) = rollback_recovering_record(paths, recovering_snapshot, &prior) { + return error.context(format!( + "rollback also failed: managed service {} is left recorded as recovering \ + with no supervisor ({rollback:#})", + record.service_id + )); + } + *record = prior; + error +} + /// Grace given to a recovery supervisor that must be terminated because its /// record could not be published. Short: it is milliseconds old. const RECOVERY_CLEANUP_GRACE: Duration = Duration::from_secs(2); @@ -1903,10 +1938,10 @@ fn detached_rocmd_command(rocmd_binary: &std::path::Path) -> ProcessCommand { #[cfg(test)] mod tests { use super::*; - use crate::test_support::identity_probe_record; use crate::test_support::{ UNCONFIRMED_STOP_PID, seed_keyed_service, stop_with_outcome, temp_app_paths, }; + use crate::test_support::{free_loopback_port, identity_probe_record}; use rocm_core::{AuditEventRecord, AutomationEventRecord}; fn test_watcher_snapshot( @@ -3449,7 +3484,7 @@ mod tests { use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; - // The gap Finding #2 closes: a crashed service's port is simultaneously + // The gap this closes: a crashed service's port is simultaneously // "free" to a brand-new `rocm serve --gpu auto` launch (which only // checks *live* records) and "needs recovery" to this function (which // used to only probe the raw OS bind). Neither process's engine binds @@ -3541,6 +3576,7 @@ mod tests { Ok(()) } + #[cfg(target_os = "linux")] #[test] fn recovery_is_not_blocked_by_a_stopped_or_failed_record_on_the_same_port() -> Result<()> { // The mirror image of the refusal above: `blocks_port` is @@ -3815,7 +3851,12 @@ mod tests { let bind_succeeded = Cell::new(false); let spawn_override = |command: &mut ProcessCommand| -> std::io::Result { - bind_succeeded.set(TcpListener::bind(("127.0.0.1", port)).is_ok()); + // Retry briefly: a sibling test's concurrent fork can hold a + // copy of the just-released socket until its exec closes it. + let deadline = std::time::Instant::now() + Duration::from_secs(2); + while !bind_succeeded.get() && std::time::Instant::now() < deadline { + bind_succeeded.set(TcpListener::bind(("127.0.0.1", port)).is_ok()); + } command.spawn() }; @@ -4101,7 +4142,7 @@ mod tests { let (root, paths) = temp_app_paths("restart-rollback-first-write"); paths.ensure()?; let service_id = "svc-restart-rollback-first-write"; - let mut record = identity_probe_record(&paths, service_id, 11459); + let mut record = identity_probe_record(&paths, service_id, free_loopback_port()); record.status = "failed".to_owned(); record.write()?; @@ -4141,8 +4182,8 @@ mod tests { /// The first-write rollback above covers a failure before anything is /// spawned; this one covers the second write, after the supervisor exists. - /// A thread races the restart: as soon as the first write lands - /// (`recovering`, no supervisor yet), it corrupts the manifest's content so + /// The spawn seam corrupts the manifest's content right after the first + /// write (`recovering`, no supervisor yet), so /// `record_spawned_supervisor`'s read fails. That must kill the spawned /// supervisor and restore the prior record — and the rollback write itself /// must still land, since corrupting *content* leaves the file a writable @@ -4163,41 +4204,23 @@ mod tests { fs::create_dir_all(paths.services_dir())?; let service_id = "svc-restart-rollback-second-write-unreadable"; - let mut record = identity_probe_record(&paths, service_id, 11461); + let mut record = identity_probe_record(&paths, service_id, free_loopback_port()); record.status = "failed".to_owned(); record.write()?; let prior = record.clone(); let terminate_calls_before = TERMINATE_RECOVERY_SUPERVISOR_CALLS.with(std::cell::Cell::get); let manifest_path = paths.service_manifest_path(service_id); - let corrupt = thread::spawn(move || -> bool { - let deadline = std::time::Instant::now() + Duration::from_secs(10); - while std::time::Instant::now() < deadline { - if let Ok(bytes) = fs::read(&manifest_path) - && let Ok(on_disk) = serde_json::from_slice::(&bytes) - && on_disk.status == "recovering" - && on_disk.supervisor_pid == 0 - { - let _ = fs::write(&manifest_path, b"not json"); - return true; - } - // Busy-spin: the window this races (between the restart's - // first write and its spawned child's own write) is only as - // long as a `fork`+`exec`, far shorter than any sleep tick - // could reliably catch. - std::hint::spin_loop(); - } - false - }); - let outcome = restart_managed_service(&paths, &mut record); - let corrupted_in_time = corrupt.join().expect("corrupt thread"); + // The spawn seam runs after the restart's first write and before + // `record_spawned_supervisor`, which is exactly the window to corrupt. + let corrupt = |command: &mut ProcessCommand| -> std::io::Result { + fs::write(&manifest_path, b"not json")?; + command.spawn() + }; + let outcome = restart_managed_service_with(&paths, &mut record, Some(&corrupt)); let persisted = load_service_record(&paths, service_id); fs::remove_dir_all(root).ok(); - assert!( - corrupted_in_time, - "precondition: the race won before the second write ran" - ); let error = outcome.expect_err("an unreadable second write must fail the restart"); let message = format!("{error:#}"); assert!( @@ -4252,44 +4275,22 @@ mod tests { fs::create_dir_all(paths.services_dir())?; let service_id = "svc-restart-rollback-also-fails"; - let mut record = identity_probe_record(&paths, service_id, 11462); + let mut record = identity_probe_record(&paths, service_id, free_loopback_port()); record.status = "failed".to_owned(); record.write()?; - let manifest_path = paths.service_manifest_path(service_id); let services_dir = paths.services_dir(); - let lock = { - let services_dir = services_dir.clone(); - thread::spawn(move || -> bool { - let deadline = std::time::Instant::now() + Duration::from_secs(10); - while std::time::Instant::now() < deadline { - if let Ok(bytes) = fs::read(&manifest_path) - && let Ok(on_disk) = serde_json::from_slice::(&bytes) - && on_disk.status == "recovering" - && on_disk.supervisor_pid == 0 - && fs::set_permissions( - &services_dir, - std::fs::Permissions::from_mode(0o555), - ) - .is_ok() - { - return true; - } - std::hint::spin_loop(); - } - false - }) + // Same seam as above: deny directory writes once the first write has + // landed, so the second write and its rollback both fail. + let lock = |command: &mut ProcessCommand| -> std::io::Result { + fs::set_permissions(&services_dir, std::fs::Permissions::from_mode(0o555))?; + command.spawn() }; - let outcome = restart_managed_service(&paths, &mut record); - let locked_in_time = lock.join().expect("lock thread"); + let outcome = restart_managed_service_with(&paths, &mut record, Some(&lock)); fs::set_permissions(&services_dir, std::fs::Permissions::from_mode(0o755))?; let persisted = load_service_record(&paths, service_id); fs::remove_dir_all(root).ok(); - assert!( - locked_in_time, - "precondition: the race won before the second write ran" - ); let error = outcome.expect_err("a read-only manifest must fail both writes"); let message = format!("{error:#}"); assert!( @@ -4306,6 +4307,142 @@ mod tests { Ok(()) } + /// A spawn that fails after the `recovering` write must not leave that + /// write behind: `is_live` counts `recovering`, and rocmd reads records + /// without refreshing them, so a stale one would reserve the port. + #[test] + fn restart_rolls_back_the_recovering_record_when_the_spawn_fails() -> Result<()> { + let (root, paths) = temp_app_paths("restart-spawn-fails-rolls-back"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let service_id = "svc-restart-spawn-fails"; + let mut record = identity_probe_record(&paths, service_id, free_loopback_port()); + record.status = "failed".to_owned(); + record.write()?; + + let failing = |_command: &mut ProcessCommand| -> std::io::Result { + Err(std::io::Error::other("spawn refused by test")) + }; + let outcome = restart_managed_service_with(&paths, &mut record, Some(&failing)); + let persisted = load_service_record(&paths, service_id); + fs::remove_dir_all(root).ok(); + + let message = format!( + "{:#}", + outcome.expect_err("a failed spawn fails the restart") + ); + assert!( + message.contains("failed to spawn recovery supervisor"), + "{message}" + ); + assert_eq!( + persisted?.status, "failed", + "the on-disk record must not stay `recovering`" + ); + assert_eq!( + record.status, "failed", + "the in-memory record must be restored too" + ); + Ok(()) + } + + /// The disk re-check excludes the service's own record: rocmd reads records + /// as-is, so a crashed service can still read `running` on its own port, and + /// counting that as a competitor would make it impossible to ever recover. + #[test] + fn recovery_ignores_its_own_live_looking_record_on_its_own_port() -> Result<()> { + use std::net::TcpListener; + + let (root, paths) = temp_app_paths("recovery-self-exclusion"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let probe = TcpListener::bind(("127.0.0.1", 0))?; + let port = probe.local_addr()?.port(); + drop(probe); + + let service_id = "svc-recovery-self-exclusion"; + let mut record = identity_probe_record(&paths, service_id, port); + record.status = "running".to_owned(); + record.write()?; + + let failing = |_command: &mut ProcessCommand| -> std::io::Result { + Err(std::io::Error::other("spawn refused by test")) + }; + let outcome = restart_managed_service_with(&paths, &mut record, Some(&failing)); + fs::remove_dir_all(root).ok(); + + let message = format!("{:#}", outcome.expect_err("the intercepted spawn fails")); + assert!( + message.contains("spawn refused by test"), + "recovery must reach the spawn, not refuse over its own record: {message}" + ); + Ok(()) + } + + /// The rollback merges onto the live record: a stop marker persisted since + /// the first write survives, and the restart's own fields are restored. + #[test] + fn rollback_keeps_a_concurrent_stop_marker_and_restores_the_restarts_fields() -> Result<()> { + let (root, paths) = temp_app_paths("rollback-merge"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let mut prior = identity_probe_record(&paths, "svc-rollback-merge", 11464); + prior.status = "failed".to_owned(); + let mut recovering = prior.clone(); + recovering.status = "recovering".to_owned(); + recovering.supervisor_pid = 0; + // What a concurrent stop persisted on the live `recovering` record. + let mut on_disk = recovering.clone(); + on_disk.stop_requested_unix_ms = Some(1_234); + on_disk.write()?; + + rollback_recovering_record(&paths, &recovering, &prior)?; + let persisted = load_service_record(&paths, "svc-rollback-merge"); + fs::remove_dir_all(root).ok(); + + let persisted = persisted?; + assert_eq!(persisted.status, "failed"); + assert_eq!( + persisted.stop_requested_unix_ms, + Some(1_234), + "a stop marker persisted meanwhile must survive the rollback" + ); + Ok(()) + } + + /// If another writer has since recorded a different supervisor, that + /// record is truer than this restart's view and the rollback leaves it. + #[test] + fn rollback_leaves_a_record_that_names_a_different_supervisor() -> Result<()> { + let (root, paths) = temp_app_paths("rollback-other-supervisor"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let mut prior = identity_probe_record(&paths, "svc-rollback-other", 11465); + prior.status = "failed".to_owned(); + let mut recovering = prior.clone(); + recovering.status = "recovering".to_owned(); + recovering.supervisor_pid = 0; + let mut on_disk = recovering.clone(); + on_disk.supervisor_pid = 4_242_424; + on_disk.write()?; + + rollback_recovering_record(&paths, &recovering, &prior)?; + let persisted = load_service_record(&paths, "svc-rollback-other"); + fs::remove_dir_all(root).ok(); + + let persisted = persisted?; + assert_eq!( + persisted.status, "recovering", + "must not be blindly restored" + ); + assert_eq!(persisted.supervisor_pid, 4_242_424); + Ok(()) + } + /// The lease branch above proves a bound loopback port refuses recovery; /// the doc comment on `restart_managed_service`'s lease also claims custom /// hosts skip that preflight entirely, relying on the engine's own bind diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 0427ea894..e213ccd2d 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -142,8 +142,9 @@ pub use proc_lifecycle::{ }; pub use process::{ FileLock, LoopbackPortLease, MANAGED_LAUNCH_LOCK_TIMEOUT, detach_command_session, - is_canonical_loopback_host, lease_loopback_port, lease_loopback_port_or_bind_error, - process_is_running, terminate_process, terminate_process_tree, + is_canonical_loopback_host, launch_port_argument, lease_loopback_port, + lease_loopback_port_or_bind_error, process_is_running, terminate_process, + terminate_process_tree, }; #[cfg(windows)] pub use process::{ diff --git a/crates/rocm-core/src/process.rs b/crates/rocm-core/src/process.rs index 928e134cc..fc012b6ae 100644 --- a/crates/rocm-core/src/process.rs +++ b/crates/rocm-core/src/process.rs @@ -402,13 +402,8 @@ pub struct FileLock { } impl FileLock { - /// Acquire an exclusive lock on `path`, creating the lock file and any - /// missing parent directories first. Blocks until the lock is available. - /// - /// The lock file itself carries no data; it is a rendezvous point, so an - /// existing file is reused (never truncated) and its contents are ignored. - pub fn acquire(path: impl Into) -> Result { - let path = path.into(); + /// Create (never truncate) the lock file and any missing parent directories. + fn open(path: PathBuf) -> Result<(PathBuf, fs::File)> { if let Some(parent) = path.parent() { fs::create_dir_all(parent) .with_context(|| format!("failed to create lock directory {}", parent.display()))?; @@ -420,6 +415,16 @@ impl FileLock { .truncate(false) .open(&path) .with_context(|| format!("failed to open lock file {}", path.display()))?; + Ok((path, file)) + } + + /// Acquire an exclusive lock on `path`, creating the lock file and any + /// missing parent directories first. Blocks until the lock is available. + /// + /// The lock file itself carries no data; it is a rendezvous point, so an + /// existing file is reused (never truncated) and its contents are ignored. + pub fn acquire(path: impl Into) -> Result { + let (path, file) = Self::open(path.into())?; file.lock() .with_context(|| format!("failed to acquire lock {}", path.display()))?; Ok(Self { file, path }) @@ -446,18 +451,7 @@ impl FileLock { /// Fails when the lock file cannot be created/locked, or when another /// holder keeps it for the whole timeout. pub fn acquire_bounded(path: impl Into, timeout: Duration) -> Result { - let path = path.into(); - if let Some(parent) = path.parent() { - fs::create_dir_all(parent) - .with_context(|| format!("failed to create lock directory {}", parent.display()))?; - } - let file = fs::OpenOptions::new() - .read(true) - .write(true) - .create(true) - .truncate(false) - .open(&path) - .with_context(|| format!("failed to open lock file {}", path.display()))?; + let (path, file) = Self::open(path.into())?; let deadline = Instant::now() + timeout; loop { match file.try_lock() { @@ -516,6 +510,28 @@ pub fn is_canonical_loopback_host(host: &str) -> bool { host.trim() == DEFAULT_LOCAL_HOST } +/// Validate the optional `port` argument of a `launch_server` tool call. +/// +/// `None` and JSON `null` mean "not supplied" (automatic selection). Anything +/// else must be an integer in 1-65535: a malformed value must not be read as +/// "not supplied" and silently fall through to automatic port selection. +/// Shared by the chat assistant and the MCP server so the two cannot drift. +/// +/// # Errors +/// Returns the reason, to be prefixed with the caller's own argument name. +pub fn launch_port_argument( + value: Option<&serde_json::Value>, +) -> Result, &'static str> { + let Some(value) = value.filter(|value| !value.is_null()) else { + return Ok(None); + }; + let port = value.as_u64().ok_or("must be an integer")?; + match u16::try_from(port) { + Ok(port) if port != 0 => Ok(Some(port)), + _ => Err("must be between 1 and 65535"), + } +} + /// A real, held `127.0.0.1:` listener proving the port is bindable now. /// /// The lease is the collision check: it is acquired before any manifest, log, or @@ -2408,3 +2424,45 @@ mod tests { assert_the_child_is_given_an_empty_stdin(fixture.command()); } } + +#[cfg(test)] +mod launch_port_argument_tests { + use super::launch_port_argument; + use serde_json::json; + + #[test] + fn absent_or_null_means_automatic() { + assert_eq!(launch_port_argument(None), Ok(None)); + assert_eq!(launch_port_argument(Some(&json!(null))), Ok(None)); + } + + #[test] + fn accepts_the_full_valid_range() { + assert_eq!(launch_port_argument(Some(&json!(1))), Ok(Some(1))); + assert_eq!(launch_port_argument(Some(&json!(65535))), Ok(Some(65535))); + } + + #[test] + fn rejects_malformed_values_instead_of_treating_them_as_absent() { + for value in [ + json!("8000"), + json!(8000.5), + json!(-1), + json!(true), + json!([8000]), + ] { + assert_eq!( + launch_port_argument(Some(&value)), + Err("must be an integer"), + "{value}" + ); + } + for value in [json!(0), json!(65536), json!(u64::MAX)] { + assert_eq!( + launch_port_argument(Some(&value)), + Err("must be between 1 and 65535"), + "{value}" + ); + } + } +} diff --git a/crates/rocm-dash-tui/src/ui/serve_wizard.rs b/crates/rocm-dash-tui/src/ui/serve_wizard.rs index 49d7897bc..e87e4b067 100644 --- a/crates/rocm-dash-tui/src/ui/serve_wizard.rs +++ b/crates/rocm-dash-tui/src/ui/serve_wizard.rs @@ -585,7 +585,14 @@ fn editor_key(w: &mut ServeWizardState, code: KeyCode) { // The form's host/port are only written on Enter, so discarding // the editor here never mutates them. w.editor = None; - recompute_message(w); + // Keep the complaint only while it is still true; a different one + // (about a field this Esc never touched) is cleared, not surfaced + // or focused. `request_launch` still reports it on launch. + if let Some(current) = w.message.take() + && w.validate().is_err_and(|(_, msg)| msg == current) + { + w.message = Some(current); + } } KeyCode::Left | KeyCode::Right => { let delta = if code == KeyCode::Left { -1 } else { 1 }; @@ -652,22 +659,6 @@ fn editor_key(w: &mut ServeWizardState, code: KeyCode) { } } -/// Esc discards an edit without ever writing it to the form, so whatever -/// `validate()` says now was equally true before the editor opened. Safe to -/// unconditionally re-state (and re-focus, if the complaint has moved) since -/// nothing new was introduced. -fn recompute_message(w: &mut ServeWizardState) { - if w.message.is_some() { - match w.validate() { - Ok(_) => w.message = None, - Err((field, msg)) => { - w.focus(field); - w.message = Some(msg); - } - } - } -} - /// An actual edit to `touched` (a committed Enter, or a Port-mode cycle) can /// only confirm or deny a complaint about `touched` itself. A complaint about /// some other, untouched field would be a surprise the user never triggered @@ -1732,6 +1723,29 @@ mod tests { assert_eq!(w.message.as_deref(), Some(CUSTOM_HOST_NEEDS_PORT)); } + #[test] + fn escaping_a_host_editor_error_does_not_surface_or_focus_another_fields_complaint() { + // Host edit -> blank Enter -> Esc, with the model still empty: the Host + // complaint is gone after Esc, and "model is required" must not replace + // it or pull focus off Host. + let mut wiz = Some(ServeWizardState { + advanced_expanded: true, + ..Default::default() + }); + let mut jobs = State::default(); + focus_field(wiz.as_mut().unwrap(), Field::Host); + feed( + &mut wiz, + &mut jobs, + &[KeyCode::Enter, KeyCode::Backspace, KeyCode::Enter], + ); + assert!(wiz.as_ref().unwrap().message.is_some()); + feed(&mut wiz, &mut jobs, &[KeyCode::Esc]); + let w = wiz.as_ref().unwrap(); + assert!(w.message.is_none(), "{:?}", w.message); + assert_eq!(w.current_field(), Field::Host); + } + #[test] fn committing_a_host_edit_that_surfaces_an_unrelated_port_complaint_clears_it_instead() { // An outstanding complaint about the Host row (blank host) is fixed by @@ -1844,7 +1858,7 @@ mod tests { // This only pins the shared constant into the approval body; the CLI's own // plan output proves the phrase actually appears before the resolved - // endpoint, in `serve-auto-port-skips-occupied` (serve-26). + // endpoint, in `serve-auto-port-skips-occupied` (serve-29). #[test] fn approval_shows_exact_argv_and_the_automatic_port_phrase() { let mut w = ServeWizardState { diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 4ee0a6b9a..3d97970b3 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -29,17 +29,6 @@ Feature: Model serving When the user lists running services Then the connection details match the actual server port - # Real CLI coverage for automatic port allocation. A loopback listener holds - # the legacy default while one real managed GPU serve starts; the reported - # endpoint must advance to the next candidate rather than failing late in the - # engine. Runs in the merge queue because it launches a real model server. - @id:serve-auto-port-skips-occupied @requires-gpu @merge-queue - Scenario: serve-05 - Automatic serving skips an occupied default port - Given a managed runtime is active - And the default serve port is occupied - When the user serves a model with automatic port selection - Then the service uses the next automatic port - # vLLM serve + inference (safetensors model). Engine coverage: vLLM. This is the # deliberate vLLM half of a per-engine pair with `serve-lemonade-inference` # below, so it stays pinned to vLLM (the slug names the engine). It is also the @@ -48,7 +37,7 @@ Feature: Model serving # (`serve-default-engine-working-endpoint`, `serve-default-engine-inference`, # and `serve-readiness-contract`) run only in the merge queue (not on MI300X). @id:serve-vllm-inference @requires-gpu @requires-engine:vllm - Scenario: serve-06 - A served model responds to inference requests on vLLM + Scenario: serve-05 - A served model responds to inference requests on vLLM Given a managed runtime is active And a model is being served on GPU When the user sends a chat completion request @@ -63,7 +52,7 @@ Feature: Model serving # ordinary per-PR path, and the longer readiness timeout also gives the first # inference request enough time to complete. @id:serve-large-model-inference @requires-gpu @serve-timeout:2400 @nightly - Scenario: serve-07 - A large platform-specific model serves and responds to inference + Scenario: serve-06 - A large platform-specific model serves and responds to inference Given a managed runtime is active And a large model is being served on GPU When the user sends a chat completion request @@ -74,7 +63,7 @@ Feature: Model serving # lemonade per-PR canary: one real lemonade serve runs on every PR (the # counterpart to the vLLM canary above). @id:serve-lemonade-inference @requires-gpu @requires-engine:lemonade - Scenario: serve-08 - A model served on lemonade responds to inference requests + Scenario: serve-07 - A model served on lemonade responds to inference requests Given a managed runtime is active And a GGUF model is being served on lemonade When the user sends a chat completion request @@ -90,7 +79,7 @@ Feature: Model serving # `serve-lemonade-inference` (cache-shared, no extra download) so this stays a # fast per-PR canary rather than needing the @nightly large-checkpoint path. @id:serve-hf-checkpoint-inference @requires-gpu @requires-engine:lemonade - Scenario: serve-09 - A canonical Hugging Face checkpoint serves and responds to inference + Scenario: serve-08 - A canonical Hugging Face checkpoint serves and responds to inference Given a managed runtime is active And a canonical Hugging Face GGUF checkpoint is being served on lemonade When the user sends a chat completion request @@ -101,15 +90,15 @@ Feature: Model serving # default from the capability probe, so this covers whichever engine the host # would actually pick. @id:serve-default-engine-working-endpoint @requires-gpu @merge-queue - Scenario: serve-10 - Serving a model without specifying an engine produces a working endpoint + Scenario: serve-09 - Serving a model without specifying an engine produces a working endpoint Given a managed runtime is active When the user serves a model without specifying an engine Then an engine is selected automatically And the model is reachable - # The inference half of serve-10. + # The inference half of serve-09. @id:serve-default-engine-inference @requires-gpu @merge-queue - Scenario: serve-11 - A default-engine served model responds to inference requests + Scenario: serve-10 - A default-engine served model responds to inference requests Given a managed runtime is active When the user serves a model without specifying an engine Then the model responds to inference requests @@ -121,7 +110,7 @@ Feature: Model serving # skips it on lemonade-default hosts (Strix Halo), where asserting a vLLM # default would be a guaranteed false failure. @id:serve-vllm-default-on-instinct @requires-gpu @requires-engine:vllm - Scenario: serve-12 - vLLM is the default serving engine on Instinct + Scenario: serve-11 - vLLM is the default serving engine on Instinct Given a managed runtime is active When the user serves a vLLM-capable model without specifying an engine Then vLLM is selected as the default engine @@ -132,7 +121,7 @@ Feature: Model serving # platform that runs it (the merge queue skips MI300X). Readiness is gated on a real inference probe, which is what makes # this contract hold rather than race the model load. @id:serve-readiness-contract @requires-gpu @merge-queue - Scenario: serve-13 - A service reported ready can immediately serve inference + Scenario: serve-12 - A service reported ready can immediately serve inference Given a managed runtime is active And a model is being served on GPU When the CLI reports the service as ready @@ -143,7 +132,7 @@ Feature: Model serving # launched — with an actionable message, never a CPU or device-0 fallback. Runs # on the no-GPU mock host, so it gates every PR (@requires-no-gpu, no GPU needed). @id:serve-no-gpu-fails-fast @requires-no-gpu - Scenario: serve-14 - Serving is refused on a host with no AMD GPU + Scenario: serve-13 - Serving is refused on a host with no AMD GPU When the user serves a model under the GPU-required default Then serving is refused before any engine starts And the user is told no AMD GPU was detected @@ -153,7 +142,7 @@ Feature: Model serving # check runs inside argument parsing, before engine selection or any GPU # pre-flight, so it needs no GPU and no engine and gates every PR (ungated). @id:serve-negative-temperature-rejected - Scenario: serve-15 - Serving with a temperature below zero is refused with a clear reason + Scenario: serve-14 - Serving with a temperature below zero is refused with a clear reason When the user serves a model with a negative sampling temperature Then serving is refused before any engine starts And the CLI explains that temperature cannot be negative @@ -162,7 +151,7 @@ Feature: Model serving # GPU-required serve must treat it as "no GPU" and refuse, not fall back. Runs on # GPU hardware (Strix Halo / Instinct). @id:serve-masked-devices-fail @requires-gpu @requires-os:linux - Scenario: serve-16 - Serving is refused when every GPU is masked from view + Scenario: serve-15 - Serving is refused when every GPU is masked from view When the user serves a model with every GPU masked from view Then serving is refused before any engine starts And the user is told no AMD GPU was detected @@ -175,12 +164,12 @@ Feature: Model serving # # This is the live coverage for an index the host does not have. The *masked* # sub-case — an index that exists but is hidden by an active visibility mask — - # is a different refusal and owns its own scenarios, serve-20 and serve-21. + # is a different refusal and owns its own scenarios, serve-19 and serve-20. # The step here accepts several refusal wordings only because which authority # answers depends on the lane: the visible set when it could be enumerated, # the raw amd-smi count when it could not. @id:serve-absent-gpu-index-rejected @requires-gpu @requires-os:linux - Scenario: serve-17 - Serving pinned to a GPU that does not exist is refused + Scenario: serve-16 - Serving pinned to a GPU that does not exist is refused When the user serves a model pinned to a GPU index that does not exist Then serving is refused before any engine starts And the user is told that GPU index is unavailable @@ -190,7 +179,7 @@ Feature: Model serving # during argument parsing — before any engine or GPU work. No device needed, so # this runs on the mock lane every PR. @id:serve-runtime-and-env-selectors-conflict - Scenario: serve-18 - Selecting both a runtime and an environment at once is refused + Scenario: serve-17 - Selecting both a runtime and an environment at once is refused When the user serves a model selecting both a runtime and an environment Then serving is refused before any engine starts And the user is told the two selectors cannot be combined @@ -201,7 +190,7 @@ Feature: Model serving # deterministic on the blocking no-GPU lane rather than relying on a real 3 GiB # transfer to fail at just the right moment. @id:serve-lemonade-preparation-recovery @requires-no-gpu - Scenario: serve-19 - Repeated Lemonade preparation failure gives the user a recovery path + Scenario: serve-18 - Repeated Lemonade preparation failure gives the user a recovery path Given Lemonade preparation cannot complete When the user serves a model with Lemonade Then serving stops after one automatic retry @@ -216,12 +205,12 @@ Feature: Model serving # `HIP_VISIBLE_DEVICES=0` is in range but masked, exercising the visible-set # rejection directly. It holds on a single-GPU host too, so this stays ungated: # the mask names device 0, which exists there, so the visible set still resolves - # to [0] and ordinal 1 is refused against it. (Contrast serve-21, whose mask + # to [0] and ordinal 1 is refused against it. (Contrast serve-20, whose mask # names a device a single-GPU host does not have — see the note there.) Either # way an honest refusal rather than a remap, which is what this asserts. Runs on # GPU hardware. @id:serve-masked-gpu-index-rejected @requires-gpu @requires-os:linux - Scenario: serve-20 - Serving pinned to a GPU hidden by the visibility mask is refused + Scenario: serve-19 - Serving pinned to a GPU hidden by the visibility mask is refused When the user serves a model pinned to a GPU hidden by the visibility mask Then serving is refused before any engine starts And the user is told the pinned GPU is unavailable @@ -251,11 +240,11 @@ Feature: Model serving # outcome on one GPU turns on whether amd-smi happens to be present, and only a # second device makes the refusal follow from the mask itself. The permissive # unknown-mask fallback is deliberate (an unprobeable host must not be blocked - # from serving) and EAI-7194 does not change it. Sibling scenario serve-20 uses + # from serving) and EAI-7194 does not change it. Sibling scenario serve-19 uses # a `HIP_VISIBLE_DEVICES` mask naming a device that DOES exist, so its visible # set resolves on one GPU too and it stays ungated. @id:serve-rocr-reindexed-gpu-index-rejected @requires-gpu @requires-multi-gpu @requires-os:linux - Scenario: serve-21 - Serving pinned past the ROCR-reindexed visible set is refused + Scenario: serve-20 - Serving pinned past the ROCR-reindexed visible set is refused When the user serves a model pinned past the ROCR-reindexed visible set Then serving is refused before any engine starts And the user is told the pinned GPU is unavailable @@ -264,13 +253,13 @@ Feature: Model serving # alternatives. ROCr applies first and HIP only re-indexes and selects among the # survivors, so `ROCR_VISIBLE_DEVICES=` (hide everything) leaves HIP nothing to # see and `HIP_VISIBLE_DEVICES=0` cannot bring a device back. The GPU-required - # serve must therefore refuse, exactly as serve-16 does for a HIP-only empty + # serve must therefore refuse, exactly as serve-15 does for a HIP-only empty # mask. Before the fix the probe preferred the HIP mask and discarded the ROCR # one entirely, resolved the visible set to [0], and let the serve proceed onto a # device the runtime had already hidden. Runs on GPU hardware: on a no-GPU host # the same refusal fires for having no device at all and would prove nothing. @id:serve-rocr-empty-mask-beats-hip-mask @requires-gpu @requires-os:linux - Scenario: serve-22 - Serving is refused when ROCR hides every GPU a HIP mask names + Scenario: serve-21 - Serving is refused when ROCR hides every GPU a HIP mask names When the user serves a model with ROCR hiding every GPU a HIP mask names Then serving is refused before any engine starts And the user is told no AMD GPU was detected @@ -283,7 +272,7 @@ Feature: Model serving # what it hides and names the two commands that reach it. No GPU and no real # serve: the record is planted, so this runs on every lane. @id:serve-past-attempts-surfaced - Scenario: serve-23 - A failed local server is counted in the default list + Scenario: serve-22 - A failed local server is counted in the default list Given a local server attempt has failed When the user lists running services Then the list reports the attempt and how to look at it @@ -301,7 +290,7 @@ Feature: Model serving # runtime is active — hence the same `a managed runtime is active` precondition # every other real-serve scenario on this lane opens with. @id:serve-vllm-low-vram-oom-guidance @requires-gpu @requires-engine:vllm @requires-os:linux - Scenario: serve-24 - A vLLM serve plan on a nearly-full GPU points at the memory knob + Scenario: serve-23 - A vLLM serve plan on a nearly-full GPU points at the memory knob Given a managed runtime is active And the selected GPU is reported nearly out of VRAM When the user previews a vLLM serve plan pinned to that GPU @@ -327,7 +316,7 @@ Feature: Model serving # Windows detached-spawn primitive), so this scenario cannot reach the # refusal it exercises there. @id:serve-require-api-key-refuses-unauthenticated-reuse @requires-gpu @requires-engine:lemonade @requires-os:linux - Scenario: serve-25 - Reusing a running server without a key refuses a --require-api-key request + Scenario: serve-24 - Reusing a running server without a key refuses a --require-api-key request Given a managed runtime is active And an unauthenticated Lemonade service is already running for that model When the user serves that same model again with --require-api-key @@ -349,16 +338,16 @@ Feature: Model serving # match a narrower wording would make this scenario weaker than the # whole-tree contract in xtask/src/ascii_output_contract.rs that it mirrors. @id:serve-model-list-output-is-ascii - Scenario: serve-26 - The recommended model list prints ASCII-only text + Scenario: serve-25 - The recommended model list prints ASCII-only text When the user lists recommended models Then the model list output contains only ASCII characters # Parse-time refusal (serve-port rewrite): an out-of-range `--port` (0 here; # 65536 and non-numeric values share the same parser and message) is rejected # before any engine or GPU work, so it needs no GPU and gates every PR - # (ungated), like its sibling serve-15. + # (ungated), like its sibling serve-14. @id:serve-invalid-port-rejected - Scenario: serve-27 - Serving with an out-of-range port is refused with a clear reason + Scenario: serve-26 - Serving with an out-of-range port is refused with a clear reason When the user serves a model with an out-of-range port Then serving is refused before any engine starts And the user is told the port must be between 1 and 65535 @@ -375,7 +364,7 @@ Feature: Model serving # apps/rocm/src/service_launch.rs, per AGENTS.md section 3. # Runs in the merge queue, like its automatic-port sibling serve-auto-port-skips-occupied. @id:serve-explicit-port-reserved @requires-gpu @merge-queue - Scenario: serve-28 - Serving on a port another managed service already holds is refused + Scenario: serve-27 - Serving on a port another managed service already holds is refused Given a managed runtime is active And another managed service already reserves the default serve port When the user serves a model explicitly on the reserved port @@ -383,19 +372,30 @@ Feature: Model serving And the user is told to stop the service holding that port # Collision-safe explicit ports, untracked-process case (serve-port rewrite, - # sibling of serve-28). No managed-service record exists here, so + # sibling of serve-27). No managed-service record exists here, so # `resolve_serve_port` never reaches the "reserved by managed service" branch # and instead gets a plain OS-level `AddrInUse` back from the bind probe, # routing to `explicit_bind_error`'s "already in use by another process" - # message. Needs the same `@requires-gpu` as serve-28 for the same reason: + # message. Needs the same `@requires-gpu` as serve-27 for the same reason: # `resolve_serve_port` only runs once `spawn_managed_engine_child` has a # completed model/GPU resolution in hand (GPU indices are selected earlier in # `serve_cmd.rs`, under the launch lock, before any port is ever probed), and # `serve` has no CPU fallback that gets there. @id:serve-explicit-port-in-use-by-untracked-process @requires-gpu @merge-queue - Scenario: serve-29 - Serving on a port an untracked process already holds is refused + Scenario: serve-28 - Serving on a port an untracked process already holds is refused Given a managed runtime is active And the default serve port is occupied When the user serves a model explicitly on the occupied port Then serving is refused before any engine starts And the user is told the port is already in use by another process + + # Real CLI coverage for automatic port allocation. A loopback listener holds + # the legacy default while one real managed GPU serve starts; the reported + # endpoint must advance to the next candidate rather than failing late in the + # engine. Runs in the merge queue because it launches a real model server. + @id:serve-auto-port-skips-occupied @requires-gpu @merge-queue + Scenario: serve-29 - Automatic serving skips an occupied default port + Given a managed runtime is active + And the default serve port is occupied + When the user serves a model with automatic port selection + Then the service uses the next automatic port diff --git a/tests/e2e-cucumber/tests/e2e/dash_steps.rs b/tests/e2e-cucumber/tests/e2e/dash_steps.rs index a81727a24..85a9185e0 100644 --- a/tests/e2e-cucumber/tests/e2e/dash_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/dash_steps.rs @@ -519,7 +519,11 @@ async fn open_serve_wizard(world: &mut E2eWorld) { #[then("the serve wizard shows only the model, Advanced settings, and Launch")] async fn serve_wizard_shows_only_basic_fields(world: &mut E2eWorld) { - let screen = session(world).screen_text(); + let tui = session(world); + tui.wait_for_screen("[ Launch ]", default_timeout()) + .await + .unwrap_or_else(|e| panic!("serve wizard did not finish drawing: {e}")); + let screen = tui.screen_text(); assert!( screen.contains("Model ") && screen.contains("Advanced settings ▸ Automatic") @@ -578,7 +582,11 @@ async fn type_into_serve_wizard(world: &mut E2eWorld, text: String) { #[then("the serve wizard is still open with the host unchanged")] async fn serve_wizard_still_open_host_unchanged(world: &mut E2eWorld) { - let screen = session(world).screen_text(); + let tui = session(world); + tui.wait_until_gone("editing host", default_timeout()) + .await + .unwrap_or_else(|e| panic!("Host edit was not cancelled by Escape: {e}")); + let screen = tui.screen_text(); assert!( screen.contains("Serve a model") && screen.contains("[127.0.0.1]") From 3fd7d537eb5035cbbef48b4ddc2dc130e93d18e6 Mon Sep 17 00:00:00 2001 From: Juho Vainio Date: Mon, 26 Oct 2026 19:58:37 +0200 Subject: [PATCH 15/19] docs(serve): add serve_port to module map; use ASCII in serve copy Add the serve_port.rs entry to docs/architecture.md and refresh the service_launch.rs entry. Replace the em dash in the Serving tab copy and the en dashes in the port-range messages with ASCII hyphens/colons. Rename the loopback-host test so its name states what it pins. Signed-off-by: Juho Vainio --- apps/rocm/src/serve_port.rs | 2 +- crates/rocm-dash-tui/src/ui/serve_wizard.rs | 10 +++++----- crates/rocm-dash-tui/src/ui/tabs/serving.rs | 2 +- docs/architecture.md | 3 ++- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/apps/rocm/src/serve_port.rs b/apps/rocm/src/serve_port.rs index 200147f21..2376337bd 100644 --- a/apps/rocm/src/serve_port.rs +++ b/apps/rocm/src/serve_port.rs @@ -90,7 +90,7 @@ pub(crate) struct ReservedPort { /// silently produce an endpoint nobody asked for. /// /// # Errors -/// Returns a user-facing message for anything outside 1–65535. +/// Returns a user-facing message for anything outside 1-65535. pub(crate) fn parse_serve_port(value: &str) -> Result { match value.trim().parse::() { Ok(0) | Err(_) => Err(format!( diff --git a/crates/rocm-dash-tui/src/ui/serve_wizard.rs b/crates/rocm-dash-tui/src/ui/serve_wizard.rs index e87e4b067..810540ec4 100644 --- a/crates/rocm-dash-tui/src/ui/serve_wizard.rs +++ b/crates/rocm-dash-tui/src/ui/serve_wizard.rs @@ -395,7 +395,7 @@ impl ServeWizardState { let shown = self.port.trim(); return Err(( Field::Port, - format!("port `{shown}` is not a valid 1–65535 value"), + format!("port `{shown}` is not a valid 1-65535 value"), )); }; return Ok(Some(port)); @@ -633,7 +633,7 @@ fn editor_key(w: &mut ServeWizardState, code: KeyCode) { // Mirror the Host rule: an empty custom port is not a // value, so keep the editor open with a plain fix. w.message = Some( - "port cannot be empty; type a 1–65535 value or press Esc to cancel" + "port cannot be empty; type a 1-65535 value or press Esc to cancel" .to_string(), ); return; @@ -1146,7 +1146,7 @@ mod tests { }; let (field, msg) = w.build_args().unwrap_err(); assert_eq!(field, Field::Port, "{bad:?}"); - assert!(msg.contains("1–65535"), "{bad:?}: {msg}"); + assert!(msg.contains("1-65535"), "{bad:?}: {msg}"); } } @@ -1259,7 +1259,7 @@ mod tests { } #[test] - fn is_loopback_host_matches_the_cli_exactly() { + fn is_loopback_host_accepts_only_the_documented_spellings() { for ok in ["127.0.0.1", "localhost", "::1"] { assert!(is_loopback_host(ok), "{ok}"); } @@ -1673,7 +1673,7 @@ mod tests { assert_eq!(w.port, "11435", "form untouched"); let msg = w.message.as_deref().unwrap(); assert!(msg.contains("port cannot be empty"), "{msg}"); - assert!(msg.contains("1–65535"), "{msg}"); + assert!(msg.contains("1-65535"), "{msg}"); } // Typing a real value accepts normally and clears the complaint. for k in typed("9001") { diff --git a/crates/rocm-dash-tui/src/ui/tabs/serving.rs b/crates/rocm-dash-tui/src/ui/tabs/serving.rs index 5a0dbd7d2..11fc01b1f 100644 --- a/crates/rocm-dash-tui/src/ui/tabs/serving.rs +++ b/crates/rocm-dash-tui/src/ui/tabs/serving.rs @@ -26,7 +26,7 @@ pub const VERBS: &[Verb] = &[ steps: &[ "Pick a model", "Leave the rest automatic, or open Advanced settings", - "GPU is required — ROCm never falls back to CPU", + "GPU is required: ROCm never falls back to CPU", "Launch on a local endpoint picked for you and watch it come up", ], cmd: "rocm serve ", diff --git a/docs/architecture.md b/docs/architecture.md index d2e1c9034..9bad11139 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -49,8 +49,9 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) — mechanical relocation, owns private types (Phase 6c, ROCMAI-89): `runtimes()` and its activate/rollback/uninstall/import/adopt handlers, plus runtime manifest validation/registry. `RuntimesCommand` stayed in [`main.rs`](../apps/rocm/src/main.rs) (part of the `Command`/`Cli` clap definition, the same way `EnginesCommand` did in Phase 6a). `RuntimeActivationResult` and `ActiveRuntimeMarker` are `pub(crate)`, read from `main.rs`'s `apply_runtime_update`/`append_update_activate_summary` and from [`storage.rs`](../apps/rocm/src/storage.rs) respectively; `select_runtime_manifest`/`validate_runtime_manifest_for_activation`/`normalize_path_for_compare`/`recover_setup_runtime_registration`/`runtime_usability_status` are also `pub(crate)`, read from [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs), [`storage.rs`](../apps/rocm/src/storage.rs), and [`comfyui.rs`](../apps/rocm/src/comfyui.rs). Calls into the sibling [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) above for the SDK-install-finalization bundle. - [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) — mechanical relocation, owns private types (Phase 6a): `serve()`, plus engine-recipe overrides. - [`serve_summary.rs`](../apps/rocm/src/serve_summary.rs) — full domain extraction. +- [`serve_port.rs`](../apps/rocm/src/serve_port.rs) — full domain extraction: the central `rocm serve` port policy (automatic vs explicit request, reservation by live managed services, OS preflight lease), shared by [`service_launch.rs`](../apps/rocm/src/service_launch.rs) and the CLI restart path. - [`service_hints.rs`](../apps/rocm/src/service_hints.rs) — shared CLI-output component: the `rocm services stop|restart|remove --yes` hint text, shared by [`main.rs`](../apps/rocm/src/main.rs), [`remote/mod.rs`](../apps/rocm/src/remote/mod.rs), [`serve_summary.rs`](../apps/rocm/src/serve_summary.rs), and [`service_launch.rs`](../apps/rocm/src/service_launch.rs) so they cannot go out of sync with each other, as `serve_summary.rs`'s stop row did from the day it was added — #417. -- [`service_launch.rs`](../apps/rocm/src/service_launch.rs) — mechanical relocation, owns private types (Phase 6b, ROCMAI-91): spawning/attaching to a managed service process and streaming its log, plus the endpoint-key/auth checks that gate a public bind — `AttachOutcome`/`RawModeGuard`/`ManagedSpawn`/`AutostartClaim` are its private types; `ManagedLaunchReport` is `pub(crate)`, since [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) reads its fields directly for the interactive summary. Most of this module's functions are `pub(crate)` for the same reason — [`main.rs`](../apps/rocm/src/main.rs), [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), and [`automations.rs`](../apps/rocm/src/automations.rs) all call into it. `ServicesCommand`/`services()` stayed in [`main.rs`](../apps/rocm/src/main.rs) rather than moving alongside — the service-record-management helpers `services()` dispatches to (stop/restart/remove/prune) are entangled with each other; only stop/restart are also reachable from the chat/LLM sandbox tool-calling path, out of scope for Phase 6b. +- [`service_launch.rs`](../apps/rocm/src/service_launch.rs) — mechanical relocation, owns private types (Phase 6b, ROCMAI-91): spawning/attaching to a managed service process and streaming its log, the port allocation transaction and failed-launch cleanup (under the shared launch lock), plus the endpoint-key/auth checks that gate a public bind — `AttachOutcome`/`RawModeGuard`/`ManagedSpawn`/`AutostartClaim` are its private types; `ManagedLaunchReport` is `pub(crate)`, since [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) reads its fields directly for the interactive summary. Most of this module's functions are `pub(crate)` for the same reason — [`main.rs`](../apps/rocm/src/main.rs), [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), and [`automations.rs`](../apps/rocm/src/automations.rs) all call into it. `ServicesCommand`/`services()` stayed in [`main.rs`](../apps/rocm/src/main.rs) rather than moving alongside — the service-record-management helpers `services()` dispatches to (stop/restart/remove/prune) are entangled with each other; only stop/restart are also reachable from the chat/LLM sandbox tool-calling path, out of scope for Phase 6b. - [`storage.rs`](../apps/rocm/src/storage.rs) — full domain extraction. - [`therock.rs`](../apps/rocm/src/therock.rs) — full domain extraction; owns TheRock SDK install/update and the runtime-manifest model/loaders. The `runtimes` subcommand's own command enum, dispatch fn, and most of its domain logic — rendering, activation, and manifest import — stay in [`main.rs`](../apps/rocm/src/main.rs). - [`uninstall.rs`](../apps/rocm/src/uninstall.rs) — mechanical relocation, owns private types: `uninstall()`, plus the installer `PATH` undo and running-executable deletion. The shared plan type and its file-removal helpers stay in [`main.rs`](../apps/rocm/src/main.rs) because [`storage.rs`](../apps/rocm/src/storage.rs) reuses them. From b4d307130801d97a27b468479c41cea87a051961 Mon Sep 17 00:00:00 2001 From: Juho Vainio Date: Tue, 27 Oct 2026 15:02:34 +0200 Subject: [PATCH 16/19] fix(serve): address review round on recovery races and test coverage - rocmd recovery refuses when its own record changed while waiting for the launch lock - roll back against the post-record snapshot on the try_wait failure path - assert the launch lock is held at spawn in start, attached and restart launches - assert every restored field in the rollback test - cover three serve-wizard branches; derive the collapsed summary from validation - add dash-40 and dash-41 scenarios for the staged command and refused custom host - use free ports instead of fixed ones in rocmd restart tests; retry bind in the service_launch release test Signed-off-by: Juho Vainio --- apps/rocm/src/main.rs | 11 ++ apps/rocm/src/service_launch.rs | 112 +++++++++++++++++++- apps/rocmd/src/watchers.rs | 102 +++++++++++++++++- crates/rocm-dash-tui/src/ui/serve_wizard.rs | 84 +++++++++++++-- tests/e2e-cucumber/features/dash.feature | 35 ++++++ tests/e2e-cucumber/tests/e2e/dash_steps.rs | 48 +++++++++ 6 files changed, 377 insertions(+), 15 deletions(-) diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 3b61d03cb..eb3998ef7 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -16100,7 +16100,14 @@ pub(crate) mod tests { record.write()?; let bind_succeeded = std::cell::Cell::new(false); + let lock_still_held = std::cell::Cell::new(false); + let lock_path = paths.managed_launch_lock_path(); let spawn: &RestartSpawnOverride<'_> = &|_command| { + // The launch lock must outlive the spawn: it is what keeps a + // concurrent launch from claiming the port before this record is + // republished. + lock_still_held + .set(rocm_core::FileLock::acquire_bounded(&lock_path, Duration::ZERO).is_err()); // Retry briefly: a sibling test's concurrent fork can hold a copy of // the just-released socket until its exec closes it. let deadline = std::time::Instant::now() + Duration::from_secs(2); @@ -16124,6 +16131,10 @@ pub(crate) mod tests { bind_succeeded.get(), "the leased port must already be bindable (released) by the time the child is spawned" ); + assert!( + lock_still_held.get(), + "the launch lock must still be held when the restart spawns" + ); let rendered = format!("{:#}", result.expect_err("the intercepted spawn must fail")); assert!( rendered.contains("spawn intercepted by test"), diff --git a/apps/rocm/src/service_launch.rs b/apps/rocm/src/service_launch.rs index 4512bb97a..b4fe6f793 100644 --- a/apps/rocm/src/service_launch.rs +++ b/apps/rocm/src/service_launch.rs @@ -623,6 +623,20 @@ fn spawn_managed_engine_child( } } +// Test-only, thread-scoped spawn interceptor for callers that cannot take a +// `spawn_override` argument (`start_managed_service`, `run_attached_service`): +// lets a test observe state at the instant production would spawn the child. +#[cfg(all(test, not(windows)))] +thread_local! { + static TEST_SPAWN_HOOK: std::cell::RefCell>>> = + const { std::cell::RefCell::new(None) }; +} + +#[cfg(all(test, not(windows)))] +fn test_spawn_hook() -> Option>> { + TEST_SPAWN_HOOK.with(|hook| hook.borrow().clone()) +} + #[cfg(not(windows))] type SpawnOverride<'a> = dyn Fn(&mut ProcessCommand) -> io::Result + 'a; @@ -744,6 +758,10 @@ fn launch_managed_engine_child_locked( if let Some(key_file) = endpoint_key_file.as_deref() { command.env(rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV, key_file); } + #[cfg(all(test, not(windows)))] + let hook = test_spawn_hook(); + #[cfg(all(test, not(windows)))] + let spawn_override = spawn_override.or(hook.as_deref()); let mut child = match spawn_override { Some(spawn) => spawn(&mut command), None => command.spawn(), @@ -2042,6 +2060,93 @@ mod tests { Ok(()) } + /// The launch lock must still be held when the child is spawned: it is what + /// serializes allocation against publication. Observed through the spawn + /// hook, since releasing it early changes nothing else a test can see. + #[cfg(not(windows))] + #[test] + fn start_and_attached_launches_hold_the_launch_lock_until_the_spawn() -> Result<()> { + let resolve = ResolveModelResponse { + canonical_model_id: "qwen-canonical".to_owned(), + task: "chat".to_owned(), + source: "hf".to_owned(), + revision: "main".to_owned(), + loader: "llama.cpp".to_owned(), + trust_remote_code: false, + chat_template_mode: "auto".to_owned(), + dtype: "auto".to_owned(), + device_policy: DevicePolicy::GpuPreferred, + estimated_memory: "unknown".to_owned(), + launch_defaults: serde_json::json!({}), + engine_recipe: None, + warnings: Vec::new(), + }; + for attached in [false, true] { + let (root, paths) = test_paths("launch-lock-held-until-spawn"); + paths.ensure()?; + let lock_path = paths.managed_launch_lock_path(); + let lock_was_held = std::rc::Rc::new(std::cell::Cell::new(false)); + let observed = lock_was_held.clone(); + TEST_SPAWN_HOOK.with(|hook| { + *hook.borrow_mut() = Some(std::rc::Rc::new(move |_command| { + observed.set( + rocm_core::FileLock::acquire_bounded(&lock_path, Duration::ZERO).is_err(), + ); + Err(io::Error::other("spawn intercepted by test")) + })); + }); + let launch_lock = rocm_core::FileLock::acquire(paths.managed_launch_lock_path())?; + let result = if attached { + run_attached_service( + &paths, + "lemonade", + "lemonade-lock-attached", + "qwen", + &resolve, + "127.0.0.1", + serve_port::PortRequest::Auto, + &[], + None, + None, + None, + launch_lock, + false, + ) + } else { + start_managed_service( + &paths, + "lemonade", + "lemonade-lock-start", + "qwen", + &resolve, + "127.0.0.1", + serve_port::PortRequest::Auto, + &resolve.device_policy, + &[], + None, + None, + None, + None, + launch_lock, + false, + &mut |_elapsed| {}, + ) + .map(|_| ()) + }; + TEST_SPAWN_HOOK.with(|hook| *hook.borrow_mut() = None); + let _ = fs::remove_dir_all(root); + assert!( + format!("{:#}", result.expect_err("the intercepted spawn fails")) + .contains("spawn intercepted by test") + ); + assert!( + lock_was_held.get(), + "the launch lock must be held at spawn (attached: {attached})" + ); + } + Ok(()) + } + #[test] fn resolve_endpoint_auth_loopback_stays_credential_free() { // Loopback binds never require auth, even if a key is supplied. @@ -3076,7 +3181,12 @@ mod tests { let bind_succeeded = std::cell::Cell::new(false); let spawn_override: &dyn Fn(&mut ProcessCommand) -> io::Result = &|_command| { - bind_succeeded.set(std::net::TcpListener::bind(("127.0.0.1", port)).is_ok()); + // Retry briefly: a sibling test's concurrent fork can hold a copy + // of the just-released socket until its exec closes it. + let deadline = std::time::Instant::now() + Duration::from_secs(2); + while !bind_succeeded.get() && std::time::Instant::now() < deadline { + bind_succeeded.set(std::net::TcpListener::bind(("127.0.0.1", port)).is_ok()); + } // A long-lived no-op child stands in for the engine: `_command` // carries real serve args this test binary cannot execute as // itself (see `managed_launch_cleanup_sleeper_child`). diff --git a/apps/rocmd/src/watchers.rs b/apps/rocmd/src/watchers.rs index 2b7fab96d..19302f7e2 100644 --- a/apps/rocmd/src/watchers.rs +++ b/apps/rocmd/src/watchers.rs @@ -1534,6 +1534,22 @@ fn restart_managed_service_with( ); } + // The same re-read must also prove this service's own record is still the + // one the caller saw: a `rocm services restart` of this service can finish + // while recovery waits for the lock, and writing the stale copy over it + // would orphan the restarted engine or start a second supervisor. + if let Ok(current) = load_service_record(paths, &record.service_id) + && (current.status != record.status + || current.supervisor_pid != record.supervisor_pid + || current.supervisor_start_ticks != record.supervisor_start_ticks + || current.engine_pid != record.engine_pid) + { + anyhow::bail!( + "cannot recover managed service {}: its record changed while recovery was waiting for the launch lock", + record.service_id + ); + } + // On the canonical loopback host, prove the recorded port is genuinely free // before publishing `recovering` or spawning anything: a supervisor that // cannot bind would replace a truthful record with a lie. Custom hosts keep @@ -1649,6 +1665,9 @@ fn restart_managed_service_with( )); } + // The supervisor's identity is on disk now, so the rollback guard must + // compare against this copy, not the one from before it was recorded. + let recorded_snapshot = record.clone(); thread::sleep(Duration::from_millis(200)); let exited = match child.try_wait() { Ok(exited) => exited, @@ -1659,7 +1678,7 @@ fn restart_managed_service_with( return Err(fail_recovery( paths, record, - &recovering_snapshot, + &recorded_snapshot, prior, error, )); @@ -3806,7 +3825,7 @@ mod tests { fs::create_dir_all(paths.services_dir())?; let service_id = "svc-restart-identity"; - let mut record = identity_probe_record(&paths, service_id, 11444); + let mut record = identity_probe_record(&paths, service_id, free_loopback_port()); record.status = "failed".to_owned(); record.write()?; assert_eq!( @@ -3986,7 +4005,7 @@ mod tests { fs::create_dir_all(paths.services_dir())?; let service_id = "svc-restart-drops-engine-pid"; - let mut record = identity_probe_record(&paths, service_id, 11455); + let mut record = identity_probe_record(&paths, service_id, free_loopback_port()); // The shape `rocm serve --background` writes: one PID in both roles. record.supervisor_pid = UNCONFIRMED_STOP_PID; record.engine_pid = Some(UNCONFIRMED_STOP_PID); @@ -4381,6 +4400,68 @@ mod tests { Ok(()) } + /// A `rocm services restart` that finished while recovery waited for the + /// launch lock changed this service's own record; recovery must refuse + /// rather than write its stale copy over it. + #[test] + fn recovery_refuses_when_its_own_record_changed_while_waiting_for_the_lock() -> Result<()> { + let (root, paths) = temp_app_paths("recovery-stale-own-record"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let service_id = "svc-recovery-stale-own-record"; + let mut record = identity_probe_record(&paths, service_id, free_loopback_port()); + record.status = "failed".to_owned(); + record.write()?; + let mut stale = record.clone(); + + // The restart that won the lock: same id, now starting under new pids. + record.status = "starting".to_owned(); + record.supervisor_pid = 4_242_424; + record.write()?; + + let never = |_command: &mut ProcessCommand| -> std::io::Result { + panic!("recovery must not spawn over a restarted record") + }; + let outcome = restart_managed_service_with(&paths, &mut stale, Some(&never)); + let persisted = load_service_record(&paths, service_id); + fs::remove_dir_all(root).ok(); + + let message = format!( + "{:#}", + outcome.expect_err("a changed record refuses recovery") + ); + assert!(message.contains("record changed"), "{message}"); + let persisted = persisted?; + assert_eq!(persisted.status, "starting"); + assert_eq!(persisted.supervisor_pid, 4_242_424); + Ok(()) + } + + /// The `try_wait` failure path rolls back against the snapshot taken after + /// the supervisor was recorded; the pre-record snapshot would mismatch the + /// on-disk pid and leave `recovering` behind. + #[test] + fn rollback_against_the_recorded_snapshot_restores_the_prior_record() -> Result<()> { + let (root, paths) = temp_app_paths("rollback-recorded-snapshot"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let mut prior = identity_probe_record(&paths, "svc-rollback-recorded", 11466); + prior.status = "failed".to_owned(); + let mut recorded = prior.clone(); + recorded.status = "recovering".to_owned(); + recorded.supervisor_pid = 4_242_424; + recorded.write()?; + + rollback_recovering_record(&paths, &recorded, &prior)?; + let persisted = load_service_record(&paths, "svc-rollback-recorded"); + fs::remove_dir_all(root).ok(); + + assert_eq!(persisted?.status, "failed"); + Ok(()) + } + /// The rollback merges onto the live record: a stop marker persisted since /// the first write survives, and the restart's own fields are restored. #[test] @@ -4391,9 +4472,16 @@ mod tests { let mut prior = identity_probe_record(&paths, "svc-rollback-merge", 11464); prior.status = "failed".to_owned(); + // Distinct values, so a restore that is skipped shows up. + prior.restart_count = 7; + prior.engine_pid = Some(4_111_111); + prior.inference_verified_at_unix_ms = Some(5_555); let mut recovering = prior.clone(); recovering.status = "recovering".to_owned(); recovering.supervisor_pid = 0; + recovering.restart_count = 8; + recovering.engine_pid = None; + recovering.inference_verified_at_unix_ms = None; // What a concurrent stop persisted on the live `recovering` record. let mut on_disk = recovering.clone(); on_disk.stop_requested_unix_ms = Some(1_234); @@ -4405,6 +4493,9 @@ mod tests { let persisted = persisted?; assert_eq!(persisted.status, "failed"); + assert_eq!(persisted.restart_count, 7); + assert_eq!(persisted.engine_pid, Some(4_111_111)); + assert_eq!(persisted.inference_verified_at_unix_ms, Some(5_555)); assert_eq!( persisted.stop_requested_unix_ms, Some(1_234), @@ -4463,6 +4554,7 @@ mod tests { fs::create_dir_all(paths.services_dir())?; let service_id = "svc-restart-custom-host"; + let blocked_port = free_loopback_port(); let mut record = ManagedServiceRecord::new( &paths, service_id, @@ -4470,7 +4562,7 @@ mod tests { "a-model", "a-model", "0.0.0.0", - 11460, + blocked_port, "managed", std::process::id(), None, @@ -4480,7 +4572,7 @@ mod tests { record.status = "failed".to_owned(); record.write()?; - let _blocker = std::net::TcpListener::bind(("0.0.0.0", 11460))?; + let _blocker = std::net::TcpListener::bind(("0.0.0.0", blocked_port))?; let outcome = restart_managed_service(&paths, &mut record); let persisted = load_service_record(&paths, service_id); diff --git a/crates/rocm-dash-tui/src/ui/serve_wizard.rs b/crates/rocm-dash-tui/src/ui/serve_wizard.rs index 810540ec4..84e0b1b37 100644 --- a/crates/rocm-dash-tui/src/ui/serve_wizard.rs +++ b/crates/rocm-dash-tui/src/ui/serve_wizard.rs @@ -351,15 +351,10 @@ impl ServeWizardState { if !self.is_customized() { return "Automatic"; } - if self.port_mode == PortMode::Custom && parse_port(&self.port).is_none() { - "Customized · port needs attention" - } else { - let host = self.host.trim(); - if host.is_empty() || !is_loopback_host(host) { - "Customized · host needs attention" - } else { - "Customized" - } + match self.validate_advanced() { + Err((Field::Port, _)) => "Customized · port needs attention", + Err((Field::Host, _)) => "Customized · host needs attention", + _ => "Customized", } } @@ -369,6 +364,12 @@ impl ServeWizardState { if self.model.trim().is_empty() { return Err((Field::Model, "model is required".to_string())); } + self.validate_advanced() + } + + /// The host/port half of [`Self::validate`], independent of the model, so + /// the collapsed summary can report a broken hidden override on its own. + fn validate_advanced(&self) -> Result, (Field, String)> { let host = self.host.trim(); if host.is_empty() { return Err(( @@ -2177,6 +2178,71 @@ mod tests { assert!(!out.contains("Review:"), "no second modal"); } + #[test] + fn foreground_only_form_summarises_as_customized() { + let w = ServeWizardState { + managed: false, + ..Default::default() + }; + assert!(w.is_customized()); + assert_eq!(w.advanced_summary(), "Customized"); + } + + #[test] + fn loopback_name_with_automatic_port_summarises_as_needing_attention() { + let w = ServeWizardState { + model: "m".into(), + host: "localhost".into(), + ..Default::default() + }; + assert!(w.build_args().is_err(), "launch is refused"); + assert_eq!(w.advanced_summary(), "Customized · port needs attention"); + } + + #[test] + fn committing_a_bad_port_restates_the_port_complaint() { + let mut wiz = Some(ServeWizardState { + model: "m".into(), + advanced_expanded: true, + port_mode: PortMode::Custom, + port: "abc".into(), + message: Some("port `abc` is not a valid 1-65535 value".into()), + ..Default::default() + }); + let mut jobs = State::default(); + focus_field(wiz.as_mut().unwrap(), Field::Port); + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + for c in typed("99999") { + on_key(&mut wiz, &mut jobs, &[], c); + } + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + let w = wiz.as_ref().unwrap(); + assert_eq!( + w.message.as_deref(), + Some("port `99999` is not a valid 1-65535 value"), + "the complaint is re-stated for the value just committed" + ); + } + + #[test] + fn open_editor_renders_the_typed_text() { + let mut wiz = Some(ServeWizardState { + advanced_expanded: true, + ..Default::default() + }); + let mut jobs = State::default(); + focus_field(wiz.as_mut().unwrap(), Field::Host); + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + for c in typed("nope") { + on_key(&mut wiz, &mut jobs, &[], c); + } + let out = render(wiz.as_ref().unwrap(), &jobs); + assert!( + out.contains("nope"), + "typed text must be on screen: {out:?}" + ); + } + #[test] fn collapsed_render_reports_a_broken_hidden_port() { let w = ServeWizardState { diff --git a/tests/e2e-cucumber/features/dash.feature b/tests/e2e-cucumber/features/dash.feature index 6191252d6..2e241f16b 100644 --- a/tests/e2e-cucumber/features/dash.feature +++ b/tests/e2e-cucumber/features/dash.feature @@ -665,3 +665,38 @@ Feature: Interactive dashboard Then the serve wizard is closed When the user quits the dashboard Then the dashboard exits successfully + + @id:dash-serve-wizard-automatic-form-stages-bare-serve @requires-os:linux + Scenario: dash-40 - An automatic serve form stages a bare serve command with an automatic port + When the user opens the dashboard with demo data + And the user opens the Observe view + And the user opens the serve wizard + And the user types "demo-model" into the serve wizard + And the user presses Down 2 times in the serve wizard + And the user presses Enter in the serve wizard + Then the serve wizard asks to approve "serve demo-model" with an automatic port + When the user presses Escape + And the user presses Escape + Then the serve wizard is closed + When the user quits the dashboard + Then the dashboard exits successfully + + @id:dash-serve-wizard-refuses-custom-host-with-automatic-port @requires-os:linux + Scenario: dash-41 - A custom host with an automatic port is refused in the form + When the user opens the dashboard with demo data + And the user opens the Observe view + And the user opens the serve wizard + And the user types "demo-model" into the serve wizard + And the user presses Down 1 times in the serve wizard + And the user presses Enter in the serve wizard + And the user presses Down 3 times in the serve wizard + And the user presses Enter in the serve wizard + And the user types "localhost" into the serve wizard + And the user presses Enter in the serve wizard + And the user presses Down 3 times in the serve wizard + And the user presses Enter in the serve wizard + Then the serve wizard refuses to launch and says "set Port to Custom" + When the user presses Escape + Then the serve wizard is closed + When the user quits the dashboard + Then the dashboard exits successfully diff --git a/tests/e2e-cucumber/tests/e2e/dash_steps.rs b/tests/e2e-cucumber/tests/e2e/dash_steps.rs index 85a9185e0..695b7aa51 100644 --- a/tests/e2e-cucumber/tests/e2e/dash_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/dash_steps.rs @@ -595,6 +595,54 @@ async fn serve_wizard_still_open_host_unchanged(world: &mut E2eWorld) { ); } +#[when(expr = "the user presses Down {int} times in the serve wizard")] +async fn press_down_in_serve_wizard(world: &mut E2eWorld, times: u32) { + for _ in 0..times { + session(world) + .send("\x1b[B") + .unwrap_or_else(|e| panic!("failed to move down in the serve wizard: {e}")); + } +} + +#[when("the user presses Enter in the serve wizard")] +async fn press_enter_in_serve_wizard(world: &mut E2eWorld) { + session(world) + .send("\r") + .unwrap_or_else(|e| panic!("failed to press Enter in the serve wizard: {e}")); +} + +// The approval card is what the user is asked to approve, so the staged +// command line is asserted there: an automatic form stages a bare +// `serve `, with no engine or port flag the CLI should be choosing. +#[then(expr = "the serve wizard asks to approve {string} with an automatic port")] +async fn serve_wizard_asks_to_approve_automatic(world: &mut E2eWorld, command: String) { + let tui = session(world); + tui.wait_for_screen( + "Port: automatic; endpoint shown after launch", + default_timeout(), + ) + .await + .unwrap_or_else(|e| panic!("the serve wizard did not stage an approval: {e}")); + let screen = tui.screen_text(); + assert!( + screen.contains(&command) && !screen.contains("--engine") && !screen.contains("--port"), + "expected a bare `{command}` with no engine or port flag:\n{screen}" + ); +} + +#[then(expr = "the serve wizard refuses to launch and says {string}")] +async fn serve_wizard_refuses_to_launch(world: &mut E2eWorld, message: String) { + let tui = session(world); + tui.wait_for_screen(&message, default_timeout()) + .await + .unwrap_or_else(|e| panic!("the serve wizard did not refuse the launch: {e}")); + let screen = tui.screen_text(); + assert!( + screen.contains("Serve a model") && !screen.contains("Port: automatic;"), + "a refused launch must leave the form open and stage nothing:\n{screen}" + ); +} + #[then("the serve wizard is closed")] async fn serve_wizard_closed(world: &mut E2eWorld) { session(world) From 4fa920db0f6e48629eb67a8f3f47b2550eef2afd Mon Sep 17 00:00:00 2001 From: Juho Vainio Date: Tue, 27 Oct 2026 16:38:54 +0200 Subject: [PATCH 17/19] fix(serve): refuse a differing explicit port on a live equivalent service - reject an explicit --port that the already-running equivalent service is not on - drop the fixed :11435 from Dash chat detection copy and pin its absence - skip recovery candidates inside their restart backoff so one refused service cannot starve the rest - add a Dash/CLI loopback host parity test - correct stale cross-references and the failed-launch cleanup doc; soften the restart collision wording Signed-off-by: Juho Vainio --- README.md | 4 + apps/rocm/src/main.rs | 4 +- apps/rocm/src/serve_port.rs | 18 ++++ apps/rocm/src/service_launch.rs | 113 +++++++++++++++++++++-- apps/rocmd/src/watchers.rs | 9 ++ crates/rocm-dash-tui/src/app/mod.rs | 4 +- crates/rocm-dash-tui/src/ui/tabs/chat.rs | 13 ++- 7 files changed, 147 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 2b1abf5a1..20440b395 100644 --- a/README.md +++ b/README.md @@ -837,6 +837,10 @@ different settings. If you previously started a service with `--temperature` flags, or with different ones, stop the existing service first (`rocm services stop --yes`) or match the original flags. +Likewise, an explicit `--port` that differs from the port the running service +is already on is refused rather than silently ignored; omit `--port` to reuse +the running service, or stop it first to serve on the new port. + By default the server runs in the background under ROCm CLI's supervision and prints a deployment summary: a progress indicator while it starts, then a table with the status, the full inference endpoint, the API-qualified model name, and a diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index eb3998ef7..cff4e7eed 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -7309,8 +7309,8 @@ fn restart_internal_managed_service_inner( }) { bail!( - "cannot restart `{service_id}` on {}:{}: {} claimed that port while the restart \ - was waiting for the launch lock", + "cannot restart `{service_id}` on {}:{}: managed service {} is recorded on that \ + port (it may have claimed that port while the restart waited for the launch lock)", stopped_record.host, stopped_record.port, collision.service_id diff --git a/apps/rocm/src/serve_port.rs b/apps/rocm/src/serve_port.rs index 2376337bd..c59703b40 100644 --- a/apps/rocm/src/serve_port.rs +++ b/apps/rocm/src/serve_port.rs @@ -327,6 +327,24 @@ mod tests { } } + /// Dash accepts exactly the hosts the CLI treats as non-public, so a form + /// Dash approves is never refused by the CLI for lacking + /// `--allow-public-bind`. + #[test] + fn dash_loopback_hosts_match_the_clis_non_public_hosts() { + use rocm_dash_tui::ui::serve_wizard::{LOOPBACK_HOSTS, is_loopback_host}; + for host in LOOPBACK_HOSTS { + assert!(!rocm_engine_protocol::is_public_bind_host(host), "{host}"); + } + for host in ["LOCALHOST", "[::1]", "127.0.0.2", "0.0.0.0", "::"] { + assert_eq!( + is_loopback_host(host), + !rocm_engine_protocol::is_public_bind_host(host), + "{host}" + ); + } + } + #[test] fn explicit_loopback_port_outside_the_auto_range_is_retained() { let mut probe = scripted(vec![]); diff --git a/apps/rocm/src/service_launch.rs b/apps/rocm/src/service_launch.rs index b4fe6f793..b961e6f0f 100644 --- a/apps/rocm/src/service_launch.rs +++ b/apps/rocm/src/service_launch.rs @@ -127,9 +127,9 @@ pub(crate) fn resolve_endpoint_auth( } } -/// When an equivalent managed service is already running, the endpoint key -/// serve() freshly stored for this attempt is unused — drop it so it is not -/// orphaned in storage. The already-running service keeps its own key. +/// When an equivalent managed service is already running, or the launch fails +/// before any record exists, the endpoint key serve() freshly stored for this +/// attempt is unused — drop it so it is not orphaned in storage. The already-running service keeps its own key. /// Best-effort and idempotent; a loopback attempt (`freshly_stored == None`) /// is a no-op. pub(crate) fn drop_orphaned_endpoint_key_on_already_running( @@ -399,9 +399,10 @@ const MANAGED_LAUNCH_CLEANUP_GRACE: Duration = Duration::from_secs(2); /// Undo a failed managed launch attempt before the launch lock is released, /// so no failed transaction leaves a live reservation or a stray endpoint key. /// -/// Order matters: a spawned child is terminated and confirmed dead *first*, so -/// nothing is still holding the port or writing to the log by the time cleanup -/// runs. If no child was ever spawned (`record.engine_pid` is still `None`), +/// Order matters: a spawned child is terminated *first*, so it is not still +/// holding the port or writing to the log by the time cleanup runs. The +/// termination outcome is audited but not acted on: the record is marked +/// `failed` (releasing the port) even when death was not confirmed. If no child was ever spawned (`record.engine_pid` is still `None`), /// every artifact this attempt created is removed: there is nothing for /// `rocm services`/`rocm logs` to show. If a child did run, its record and log /// are kept instead (marked `status = "failed"`), since the error that led here @@ -509,6 +510,19 @@ fn spawn_managed_engine_child( resolve.canonical_model_id ); } + // An explicit port is a demand, not a hint: reporting success on a + // different port than the one asked for would silently drop it. + if let Some(port) = port_request.explicit() + && port != existing.port + { + bail!( + "managed service `{}` is already serving engine `{engine}` and model `{}` on port {}, so port {port} would not be used; stop it with `{}` and run the command again to serve on that port", + existing.service_id, + resolve.canonical_model_id, + existing.port, + service_hints::service_stop_hint(&existing.service_id) + ); + } // Reuse cannot satisfy a demand for auth the running server never got. // The engine reads its key once, at launch, from the environment this // function builds below — so a server started without one keeps serving @@ -805,6 +819,10 @@ fn launch_managed_engine_child_locked( Ok(child_pid) } +/// Launch a managed service in the background: run the locked allocation and +/// spawn transaction, release the launch lock once the record is published, +/// then wait for HTTP readiness. +/// /// Drops the endpoint key `serve()` freshly staged for this attempt if the /// launch fails before any `ManagedServiceRecord` exists to own it (port /// reserved/in use, range exhausted, other bind error) — such a failure @@ -1177,7 +1195,7 @@ pub(crate) fn run_attached_service( resolve.engine_recipe.as_ref(), require_api_key, ) - // Mirrors the background path's cleanup in `serve_cmd.rs`: a pre-spawn + // Mirrors `run_attached_service`'s cleanup: a pre-spawn // failure (port reserved/in use, range exhausted, other bind error) returns // before any service record exists, so nothing else clears the key just // stored for this attempt. @@ -1477,6 +1495,81 @@ mod tests { assert_eq!(detach_key_outcome(true, 'z'), None); } + #[test] + fn spawn_managed_engine_child_refuses_an_explicit_port_a_live_equivalent_does_not_serve() + -> Result<()> { + let (root, paths) = test_paths("dup-managed-explicit-port-differs"); + paths.ensure()?; + let mut existing = ManagedServiceRecord::new( + &paths, + "lemonade-qwen-1000", + "lemonade", + "qwen", + "qwen-canonical", + "127.0.0.1", + 11510, + "managed", + std::process::id(), + None, + None, + None, + ); + existing.status = "ready".to_owned(); + existing.engine_pid = Some(std::process::id()); + existing.write()?; + + let resolve = ResolveModelResponse { + canonical_model_id: "qwen-canonical".to_owned(), + task: "chat".to_owned(), + source: "hf".to_owned(), + revision: "main".to_owned(), + loader: "llama.cpp".to_owned(), + trust_remote_code: false, + chat_template_mode: "auto".to_owned(), + dtype: "auto".to_owned(), + device_policy: DevicePolicy::GpuPreferred, + estimated_memory: "unknown".to_owned(), + launch_defaults: serde_json::json!({}), + engine_recipe: None, + warnings: Vec::new(), + }; + let launch = |port_request| { + spawn_managed_engine_child( + &paths, + "lemonade", + "lemonade-qwen-2000", + "qwen", + &resolve, + "127.0.0.1", + port_request, + &resolve.device_policy, + &[], + None, + None, + None, + false, + ) + }; + + let different = launch(serve_port::PortRequest::Explicit(11511)); + // The same port, and no port at all, still reuse the live service. + let same = launch(serve_port::PortRequest::Explicit(11510)); + let auto = launch(serve_port::PortRequest::Auto); + let _ = fs::remove_dir_all(root); + + let Err(error) = different else { + panic!("a port the live service does not serve must be refused") + }; + let message = error.to_string(); + assert!( + message.contains("on port 11510") && message.contains("port 11511 would not be used"), + "{message}" + ); + assert!(matches!(same?, ManagedSpawn::AlreadyRunning(_))); + assert!(matches!(auto?, ManagedSpawn::AlreadyRunning(_))); + Ok(()) + } + #[test] fn spawn_managed_engine_child_blocks_reuse_with_mismatched_recipe() -> Result<()> { // A live service recorded with one recipe (e.g. a tool-call parser flag) @@ -1897,8 +1990,8 @@ mod tests { // ManagedServiceRecord exists for this attempt, so // abandon_failed_managed_launch never runs. Driving the real function // end to end (not a simulated call to - // drop_orphaned_endpoint_key_on_already_running) proves serve_cmd.rs's - // own call site actually wires the cleanup, mirroring how + // drop_orphaned_endpoint_key_on_already_running) proves + // `start_managed_service`'s own `.inspect_err` actually wires the cleanup, mirroring how // attached_launch_failing_before_spawn_leaves_no_orphaned_endpoint_key // drives run_attached_service for the foreground path. let (root, paths) = test_paths("orphaned-key-on-reserved-port"); @@ -1981,7 +2074,7 @@ mod tests { #[test] fn attached_launch_failing_before_spawn_leaves_no_orphaned_endpoint_key() -> Result<()> { - // The background path's cleanup lives in serve_cmd.rs, wrapping + // The background path's cleanup is the `.inspect_err` in // start_managed_service. run_attached_service is the other caller of // spawn_managed_engine_child and needs the identical .inspect_err() // cleanup, not a simulated one: drive the real function end to end. diff --git a/apps/rocmd/src/watchers.rs b/apps/rocmd/src/watchers.rs index 19302f7e2..925e1d40a 100644 --- a/apps/rocmd/src/watchers.rs +++ b/apps/rocmd/src/watchers.rs @@ -1385,6 +1385,15 @@ pub(crate) fn find_recoverable_service( if record.mode != "managed" || stop_requested(&record) { continue; } + // A candidate still inside its restart backoff would only be skipped by + // the caller; returning it here would starve every service after it + // while a refused recovery (e.g. its port was taken) keeps re-stamping. + if record + .last_restart_unix_ms + .is_some_and(|last| now.saturating_sub(last) < SERVER_RECOVER_BACKOFF_MS) + { + continue; + } if let Some(reason) = manifest_service_recovery_reason(&record, now) { return Ok(Some((record, reason))); } diff --git a/crates/rocm-dash-tui/src/app/mod.rs b/crates/rocm-dash-tui/src/app/mod.rs index 394aebe47..586f4ddd8 100644 --- a/crates/rocm-dash-tui/src/app/mod.rs +++ b/crates/rocm-dash-tui/src/app/mod.rs @@ -1109,7 +1109,7 @@ impl AppState { self.chat_detect_dispatch = true; if self.chat_consent == ChatConsent::Accepted { self.chat.push(ChatTurn::agent( - "Detecting a local engine (Lemonade :13305 / vLLM :8000 / rocm serve :11435)…" + "Detecting a local engine (Lemonade :13305 / vLLM :8000 / a rocm serve service)…" .to_string(), )); } @@ -1139,7 +1139,7 @@ impl AppState { self.chat_detect_offer = Some(cfg); } else { self.chat_detect_offer = None; - let msg = "no local engine found (Lemonade :13305 / vLLM :8000 / rocm serve :11435)"; + let msg = "no local engine found (Lemonade :13305 / vLLM :8000 / a rocm serve service)"; if accepted { self.chat.push(ChatTurn::agent(msg.to_string())); } else { diff --git a/crates/rocm-dash-tui/src/ui/tabs/chat.rs b/crates/rocm-dash-tui/src/ui/tabs/chat.rs index 8843763e1..d955c0da1 100644 --- a/crates/rocm-dash-tui/src/ui/tabs/chat.rs +++ b/crates/rocm-dash-tui/src/ui/tabs/chat.rs @@ -62,7 +62,7 @@ fn draw_consent(f: &mut Frame, area: Rect, state: &AppState, theme: &Theme) { ( " Chat — detecting… ", vec![Line::from(Span::styled( - "Probing for a local engine (Lemonade :13305 / vLLM :8000 / rocm serve :11435)…", + "Probing for a local engine (Lemonade :13305 / vLLM :8000 / a rocm serve service)…", Style::default().fg(theme.fg), ))], ) @@ -211,7 +211,7 @@ fn consent_gate_lines<'a>( Style::default().fg(theme.muted), )), Line::from(Span::styled( - " • or run a local endpoint (vLLM :8000, rocm serve :11435)", + " • or run a local endpoint (vLLM :8000, a rocm serve service)", Style::default().fg(theme.muted), )), ], @@ -637,7 +637,10 @@ mod tests { assert!(render_str(&mut s).contains("detect a local engine")); // After a fruitless detect, the message shows. s.set_detect_result(None); - assert!(render_str(&mut s).contains("no local engine found")); + let out = render_str(&mut s); + assert!(out.contains("no local engine found")); + // `rocm serve` leases its port at launch, so no fixed one is promised. + assert!(!out.contains("11435"), "{out:?}"); } #[test] @@ -645,7 +648,9 @@ mod tests { let mut s = AppState::new("t".into(), "default-dark".into()); s.active_tab = crate::app::ActiveTab::Chat; s.request_detect(); - assert!(render_str(&mut s).contains("Probing for a local engine")); + let out = render_str(&mut s); + assert!(out.contains("Probing for a local engine")); + assert!(!out.contains("11435"), "{out:?}"); } #[test] From 61be4b7df9de256571c6b73e3270eef7d5b9b16c Mon Sep 17 00:00:00 2001 From: Juho Vainio Date: Wed, 28 Oct 2026 16:01:50 +0200 Subject: [PATCH 18/19] fix(rocmd): refuse recovery over a pending stop or removed record Recovery's re-check of its own record after taking the launch lock now also refuses when a stop was requested meanwhile or the manifest was deleted, instead of clearing the stop / resurrecting the record. Adds tests for both and for the find_recoverable_service backoff skip, fixes a wrapped error message, a misnamed comment, an architecture.md claim, and a fixed port in a recovery test. Signed-off-by: Juho Vainio --- apps/rocm/src/service_launch.rs | 2 +- apps/rocmd/src/watchers.rs | 131 +++++++++++++++++++++++++++++--- docs/architecture.md | 2 +- 3 files changed, 122 insertions(+), 13 deletions(-) diff --git a/apps/rocm/src/service_launch.rs b/apps/rocm/src/service_launch.rs index 5073ce0a4..9dd9f0679 100644 --- a/apps/rocm/src/service_launch.rs +++ b/apps/rocm/src/service_launch.rs @@ -1197,7 +1197,7 @@ pub(crate) fn run_attached_service( resolve.engine_recipe.as_ref(), require_api_key, ) - // Mirrors `run_attached_service`'s cleanup: a pre-spawn + // Mirrors `start_managed_service`'s cleanup: a pre-spawn // failure (port reserved/in use, range exhausted, other bind error) returns // before any service record exists, so nothing else clears the key just // stored for this attempt. diff --git a/apps/rocmd/src/watchers.rs b/apps/rocmd/src/watchers.rs index 925e1d40a..94bb95ec6 100644 --- a/apps/rocmd/src/watchers.rs +++ b/apps/rocmd/src/watchers.rs @@ -1547,16 +1547,39 @@ fn restart_managed_service_with( // one the caller saw: a `rocm services restart` of this service can finish // while recovery waits for the lock, and writing the stale copy over it // would orphan the restarted engine or start a second supervisor. - if let Ok(current) = load_service_record(paths, &record.service_id) - && (current.status != record.status - || current.supervisor_pid != record.supervisor_pid - || current.supervisor_start_ticks != record.supervisor_start_ticks - || current.engine_pid != record.engine_pid) - { - anyhow::bail!( - "cannot recover managed service {}: its record changed while recovery was waiting for the launch lock", - record.service_id - ); + match load_service_record(paths, &record.service_id) { + Ok(current) => { + // Only a stop newer than the caller's copy counts: restarting a + // record that already carried one deliberately supersedes it. + if stop_requested(¤t) + && current.stop_requested_unix_ms != record.stop_requested_unix_ms + { + anyhow::bail!( + "cannot recover managed service {}: a stop was requested while recovery was \ + waiting for the launch lock", + record.service_id + ); + } + if current.status != record.status + || current.supervisor_pid != record.supervisor_pid + || current.supervisor_start_ticks != record.supervisor_start_ticks + || current.engine_pid != record.engine_pid + { + anyhow::bail!( + "cannot recover managed service {}: its record changed while recovery was \ + waiting for the launch lock", + record.service_id + ); + } + } + Err(_) if !paths.service_manifest_path(&record.service_id).exists() => { + anyhow::bail!( + "cannot recover managed service {}: it was removed while recovery was waiting \ + for the launch lock", + record.service_id + ); + } + Err(error) => return Err(error.context("failed to re-read the service record")), } // On the canonical loopback host, prove the recorded port is genuinely free @@ -3616,7 +3639,7 @@ mod tests { paths.ensure()?; fs::create_dir_all(paths.services_dir())?; - let port = 21_778_u16; + let port = free_loopback_port(); let mut stopped_sibling = ManagedServiceRecord::new( &paths, "lemonade-stopped-sibling", @@ -4447,6 +4470,92 @@ mod tests { Ok(()) } + /// A stop requested while recovery waited for the lock must not be cleared. + #[test] + fn recovery_refuses_when_a_stop_was_requested_while_waiting_for_the_lock() -> Result<()> { + let (root, paths) = temp_app_paths("recovery-stop-requested"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let service_id = "svc-recovery-stop-requested"; + let mut record = identity_probe_record(&paths, service_id, free_loopback_port()); + record.status = "failed".to_owned(); + record.write()?; + let mut stale = record.clone(); + + record.stop_requested_unix_ms = Some(unix_time_millis()); + record.write()?; + + let never = |_command: &mut ProcessCommand| -> std::io::Result { + panic!("recovery must not spawn for a service being stopped") + }; + let outcome = restart_managed_service_with(&paths, &mut stale, Some(&never)); + let persisted = load_service_record(&paths, service_id); + fs::remove_dir_all(root).ok(); + + let message = format!( + "{:#}", + outcome.expect_err("a pending stop refuses recovery") + ); + assert!(message.contains("stop was requested"), "{message}"); + assert!(stop_requested(&persisted?)); + Ok(()) + } + + /// A manifest deleted while recovery waited must not be resurrected. + #[test] + fn recovery_refuses_when_the_record_was_removed_while_waiting_for_the_lock() -> Result<()> { + let (root, paths) = temp_app_paths("recovery-removed-record"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let service_id = "svc-recovery-removed-record"; + let mut record = identity_probe_record(&paths, service_id, free_loopback_port()); + record.status = "failed".to_owned(); + record.write()?; + let mut stale = record; + fs::remove_file(paths.service_manifest_path(service_id))?; + + let never = |_command: &mut ProcessCommand| -> std::io::Result { + panic!("recovery must not spawn for a removed service") + }; + let outcome = restart_managed_service_with(&paths, &mut stale, Some(&never)); + let resurrected = paths.service_manifest_path(service_id).exists(); + fs::remove_dir_all(root).ok(); + + let message = format!( + "{:#}", + outcome.expect_err("a removed record refuses recovery") + ); + assert!(message.contains("was removed"), "{message}"); + assert!(!resurrected, "recovery must not rewrite a deleted manifest"); + Ok(()) + } + + /// A candidate still inside its restart backoff must not starve the ones + /// after it. + #[test] + fn find_recoverable_service_skips_a_candidate_inside_its_backoff() -> Result<()> { + let (root, paths) = temp_app_paths("recoverable-backoff-skip"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let mut a = identity_probe_record(&paths, "svc-backoff-a", free_loopback_port()); + a.status = "failed".to_owned(); + a.last_restart_unix_ms = Some(unix_time_millis()); + a.write()?; + let mut b = identity_probe_record(&paths, "svc-backoff-b", free_loopback_port()); + b.status = "failed".to_owned(); + b.write()?; + + let found = find_recoverable_service(&paths); + fs::remove_dir_all(root).ok(); + + let (record, _reason) = found?.expect("service B is recoverable"); + assert_eq!(record.service_id, "svc-backoff-b"); + Ok(()) + } + /// The `try_wait` failure path rolls back against the snapshot taken after /// the supervisor was recorded; the pre-record snapshot would mismatch the /// on-disk pid and leave `recovering` behind. diff --git a/docs/architecture.md b/docs/architecture.md index bfcf22855..3aa873511 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -52,7 +52,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) — mechanical relocation, owns private types (Phase 6c, ROCMAI-89): `runtimes()` and its activate/rollback/uninstall/import/adopt handlers, plus runtime manifest validation/registry. `RuntimesCommand` stayed in [`main.rs`](../apps/rocm/src/main.rs) (part of the `Command`/`Cli` clap definition, the same way `EnginesCommand` did in Phase 6a). `RuntimeActivationResult` and `ActiveRuntimeMarker` are `pub(crate)`, read from `main.rs`'s `apply_runtime_update`/`append_update_activate_summary` and from [`storage.rs`](../apps/rocm/src/storage.rs) respectively; `select_runtime_manifest`/`validate_runtime_manifest_for_activation`/`normalize_path_for_compare`/`recover_setup_runtime_registration`/`runtime_usability_status` are also `pub(crate)`, read from [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs), [`storage.rs`](../apps/rocm/src/storage.rs), and [`comfyui.rs`](../apps/rocm/src/comfyui.rs). Calls into the sibling [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) above for the SDK-install-finalization bundle. - [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) — mechanical relocation, owns private types (Phase 6a): `serve()`, plus engine-recipe overrides. - [`serve_summary.rs`](../apps/rocm/src/serve_summary.rs) — full domain extraction. -- [`serve_port.rs`](../apps/rocm/src/serve_port.rs) — full domain extraction: the central `rocm serve` port policy (automatic vs explicit request, reservation by live managed services, OS preflight lease), shared by [`service_launch.rs`](../apps/rocm/src/service_launch.rs) and the CLI restart path. +- [`serve_port.rs`](../apps/rocm/src/serve_port.rs) — full domain extraction: the central `rocm serve` port policy (automatic vs explicit request, reservation by live managed services, OS preflight lease), used by [`service_launch.rs`](../apps/rocm/src/service_launch.rs); the CLI restart path shares the `rocm-core` lease primitives (`lease_loopback_port_or_bind_error`, `is_canonical_loopback_host`) rather than this module. - [`service_hints.rs`](../apps/rocm/src/service_hints.rs) — shared CLI-output component: the `rocm services stop|restart|remove --yes` hint text, shared by [`main.rs`](../apps/rocm/src/main.rs), [`remote/mod.rs`](../apps/rocm/src/remote/mod.rs), [`serve_summary.rs`](../apps/rocm/src/serve_summary.rs), and [`service_launch.rs`](../apps/rocm/src/service_launch.rs) so they cannot go out of sync with each other, as `serve_summary.rs`'s stop row did from the day it was added — #417. - [`service_launch.rs`](../apps/rocm/src/service_launch.rs) — mechanical relocation, owns private types (Phase 6b, ROCMAI-91): spawning/attaching to a managed service process and streaming its log, the port allocation transaction and failed-launch cleanup (under the shared launch lock), plus the endpoint-key/auth checks that gate a public bind — `AttachOutcome`/`RawModeGuard`/`ManagedSpawn`/`AutostartClaim` are its private types; `ManagedLaunchReport` is `pub(crate)`, since [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) reads its fields directly for the interactive summary. Most of this module's functions are `pub(crate)` for the same reason — [`main.rs`](../apps/rocm/src/main.rs), [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), and [`automations.rs`](../apps/rocm/src/automations.rs) all call into it. `ServicesCommand`/`services()` stayed in [`main.rs`](../apps/rocm/src/main.rs) rather than moving alongside — the service-record-management helpers `services()` dispatches to (stop/restart/remove/prune) are entangled with each other; only stop/restart are also reachable from the chat/LLM sandbox tool-calling path, out of scope for Phase 6b. - [`storage.rs`](../apps/rocm/src/storage.rs) — full domain extraction. From a8daaee083c5f8404a40252c05586ed0fe8527b0 Mon Sep 17 00:00:00 2001 From: Juho Vainio Date: Fri, 30 Oct 2026 13:13:49 +0200 Subject: [PATCH 19/19] Address review: pin range top, blocks_port, restart lock, and refusal scenarios - serve_port: test the last port of the automatic window and count probes; return an error instead of unreachable! for custom-host automatic requests. - Tests that fail when their target is broken: blocks_port port comparison (rocm-core and restart), restart holding the launch lock until the record names the child, rocmd recovery lock bound, backoff skip, supervisor kill timing, engine-only Customized summary, stop hint asserted with its outcome. - chat: apply the serve host/port rule to the rocm_command route. - Remediation hints for public-host respawn now include --port. - e2e: dash-41 waits between Escapes; add serve-32, serve-33, dash-43. - Fix stale comments and the manual-testing reserved-port example. Signed-off-by: Juho Vainio --- apps/rocm/src/chat.rs | 30 +++++ apps/rocm/src/main.rs | 6 +- apps/rocm/src/managed_service_status.rs | 2 +- apps/rocm/src/serve_port.rs | 25 +++- apps/rocm/src/service_launch.rs | 33 ++--- apps/rocm/src/services_text.rs | 127 +++++++++++++++++- apps/rocmd/src/common.rs | 2 +- apps/rocmd/src/watchers.rs | 70 +++++++++- crates/rocm-core/src/service_record.rs | 16 +++ crates/rocm-dash-tui/src/ui/serve_wizard.rs | 12 +- docs/architecture.md | 2 +- docs/manual-testing.md | 8 +- tests/e2e-cucumber/features/dash.feature | 23 +++- .../features/model_serving.feature | 22 +++ tests/e2e-cucumber/tests/e2e/dash_steps.rs | 19 +++ tests/e2e-cucumber/tests/e2e/serving_steps.rs | 115 +++++++++++++++- 16 files changed, 474 insertions(+), 38 deletions(-) diff --git a/apps/rocm/src/chat.rs b/apps/rocm/src/chat.rs index 45e2c312a..3782fe462 100644 --- a/apps/rocm/src/chat.rs +++ b/apps/rocm/src/chat.rs @@ -2756,6 +2756,16 @@ pub(crate) fn validate_chat_rocm_command_safety(args: &[String]) -> Result<()> { { bail!("local assistant cannot request non-local host `{host}`"); } + // The same rule `rocm serve` applies, so an approval card is never offered + // for a command the CLI then refuses. + let port = chat_cli_arg_value(args, "--port") + .map(crate::serve_port::parse_serve_port) + .transpose() + .map_err(anyhow::Error::msg)?; + crate::serve_port::validate_port_request( + chat_cli_arg_value(args, "--host").unwrap_or(rocm_core::DEFAULT_LOCAL_HOST), + crate::serve_port::PortRequest::from_flag(port), + )?; if chat_cli_has_flag(args, "--foreground") || !chat_cli_has_flag(args, "--managed") { bail!("local assistant must request managed serving with --managed"); } @@ -6070,6 +6080,26 @@ model recipes .to_string(); assert!(error.contains("public network binding")); + let no_port_on_custom_host = providers::ChatToolCall { + id: None, + name: "rocm_command".to_owned(), + arguments: serde_json::json!({ + "args": ["serve", "qwen", "--host", "localhost", "--managed"] + }), + }; + let error = validate_chat_tool_call(&no_port_on_custom_host) + .unwrap_err() + .to_string(); + assert!(error.contains("explicit `--port"), "{error}"); + validate_chat_tool_call(&providers::ChatToolCall { + id: None, + name: "rocm_command".to_owned(), + arguments: serde_json::json!({ + "args": ["serve", "qwen", "--host", "localhost", "--port", "8000", "--managed"] + }), + }) + .expect("an explicit port on a loopback host is allowed"); + let foreground = providers::ChatToolCall { id: None, name: "rocm_command".to_owned(), diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 627c09ed7..b0ef9b91d 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -4631,9 +4631,9 @@ fn apply_service_prune_plan( /// **No nested acquire.** `FileLock::acquire` blocks with no `try_` variant, so /// a second acquire on this same lock path while this process already holds /// it would hang forever. Nothing reachable from either phase re-acquires -/// `managed_launch_lock_path`: the only other production call site on this -/// same path is `select_gpu_indices_under_launch_lock`, which is only ever -/// reached from `serve`, and `serve` never runs prune in-process or as a +/// `managed_launch_lock_path`: the other production acquirers of this +/// same path are `select_gpu_indices_under_launch_lock` (reached only from +/// `serve`) and `rocm services restart`, and neither runs prune in-process or as a /// subprocess (test code acquires this path directly too, but that is test /// setup, not something reachable from this function's own call graph). /// diff --git a/apps/rocm/src/managed_service_status.rs b/apps/rocm/src/managed_service_status.rs index 74e8b2865..40a6d5771 100644 --- a/apps/rocm/src/managed_service_status.rs +++ b/apps/rocm/src/managed_service_status.rs @@ -14,7 +14,7 @@ //! registry (`load_managed_services`, `refresh_managed_service_runtime_liveness`, //! `recorded_service_pids`, `settle_pending_stop_key_cleanup`), the idempotency //! guard that finds an already-live service for a given engine+model -//! (`existing_live_managed_service`), the "is this record live" classifier +//! (`existing_live_managed_service_in`), the "is this record live" classifier //! (`managed_service_is_live`), and the dashboard sidebar's ready/starting/past- //! attempts counts and one-line status text //! (`ManagedServiceSidebarCounts`/`managed_service_sidebar_counts`/ diff --git a/apps/rocm/src/serve_port.rs b/apps/rocm/src/serve_port.rs index c59703b40..7bc8a638b 100644 --- a/apps/rocm/src/serve_port.rs +++ b/apps/rocm/src/serve_port.rs @@ -151,9 +151,9 @@ pub(crate) fn resolve_serve_port( // or failure stays engine-owned. Automatic is refused here as well as in // `serve()`, because this is the shared choke point every caller reaches // — and it is refused by returning an error, never by panicking. + validate_port_request(host, request)?; let Some(port) = request.explicit() else { - validate_port_request(host, request)?; - unreachable!("custom-host automatic requests are rejected above"); + bail!("automatic port selection is not available on host {host}"); }; return Ok(LoopbackPortLease::engine_owned(port)); } @@ -492,13 +492,30 @@ mod tests { #[test] fn automatic_exhaustion_reports_the_exact_scanned_range() { - let mut probe = |_port: u16| -> io::Result { + let probe = |_port: u16| -> io::Result { Err(io::Error::new(io::ErrorKind::AddrInUse, "occupied")) }; - let error = resolve_serve_port(DEFAULT_LOCAL_HOST, PortRequest::Auto, &[], &mut probe) + let mut probes = 0usize; + let mut counting = |port: u16| { + probes += 1; + probe(port) + }; + let error = resolve_serve_port(DEFAULT_LOCAL_HOST, PortRequest::Auto, &[], &mut counting) .expect_err("an exhausted range must fail"); let message = format!("{error:#}"); assert!(message.contains("11435-11535"), "{message}"); + assert_eq!(probes, 101, "every port in the inclusive window is probed"); + } + + #[test] + fn automatic_reaches_the_last_port_of_the_window() { + let occupied = (AUTO_PORT_FIRST..AUTO_PORT_LAST) + .map(|port| (port, io::ErrorKind::AddrInUse)) + .collect(); + let mut probe = scripted(occupied); + let lease = resolve_serve_port(DEFAULT_LOCAL_HOST, PortRequest::Auto, &[], &mut probe) + .expect("the final port of the window is still tried"); + assert_eq!(lease.port(), AUTO_PORT_LAST); } #[test] diff --git a/apps/rocm/src/service_launch.rs b/apps/rocm/src/service_launch.rs index 60b0d802b..a5889f1ef 100644 --- a/apps/rocm/src/service_launch.rs +++ b/apps/rocm/src/service_launch.rs @@ -213,7 +213,7 @@ pub(crate) fn ensure_public_service_has_endpoint_key( "managed service is bound to the public host `{host}` but has no endpoint API key, \ so restarting it would reopen it without authentication. The key is dropped when a \ service stops and cannot be recovered. Launch it again with \ - `rocm serve --host {host} --allow-public-bind` (add `--api-key `, or set \ + `rocm serve --host {host} --port --allow-public-bind` (add `--api-key `, or set \ ROCM_SERVE_API_KEY, to choose the key instead of generating one)." ); } @@ -337,9 +337,9 @@ enum ManagedPortDecision { /// service the OS preflight still guards its port (the engine holds it), and /// refusing to serve because the services directory could not be listed would /// be a worse failure than launching. That guard does not cover a `starting` -/// record: its port is released back to the OS at the end of -/// [`spawn_managed_engine_child`] (see `lease.release()`) before the new -/// engine child has actually bound it, so a concurrent launch that hits this +/// record: its port is released back to the OS in +/// `launch_managed_engine_child_locked` (see `lease.release()`), right before the +/// spawn, and stays unbound until the new engine child has actually bound it, so a concurrent launch that hits this /// read failure in that narrow window can probe the same port successfully. /// Accepted as a known gap — narrower than the general services-dir read /// failure case, and only reachable by a second, near-simultaneous I/O @@ -2878,8 +2878,13 @@ mod tests { serve_port::PortRequest::Explicit(11_435), &mut always_free, ); + // The refusal names the service AND the command that frees the port; + // the stop below is that command's own implementation, and the grant + // after it is the outcome, so advice and result are pinned together. + let stop_hint = service_hints::service_stop_hint("lemonade-a-1"); assert!( - matches!(still_reserved, Err(ref error) if error.to_string().contains("already reserved by managed service lemonade-a-1")), + matches!(still_reserved, Err(ref error) if error.to_string().contains("already reserved by managed service lemonade-a-1") + && error.to_string().contains(&stop_hint)), "unexpected result while still running: {still_reserved:?}" ); @@ -3051,16 +3056,14 @@ mod tests { } #[test] - fn select_gpu_indices_under_launch_lock_blocks_for_the_full_hold_and_then_sees_the_holders_write() - -> Result<()> { - // The standing gap this closes: nothing exercised the real lock-acquisition - // function production callers use (`select_gpu_indices_under_launch_lock`) - // against a lock another thread actually holds, so a refactor that dropped - // the lock early (or stopped threading it through) would not fail any test. - // A real end-to-end spawn test is avoided here per the fix-forward plan: - // `spawn_managed_engine_child` is still exercised for real, but only via its - // `AlreadyLive` short-circuit (a pre-seeded duplicate record), so no child - // process is ever launched. + fn select_gpu_indices_under_launch_lock_blocks_until_the_holder_releases() -> Result<()> { + // Exercises the real lock-acquisition function production callers use + // (`select_gpu_indices_under_launch_lock`) against a lock another thread + // actually holds, so a refactor that dropped the lock early would fail + // here. `spawn_managed_engine_child` is then run for real, but only via + // its `AlreadyLive` short-circuit (a pre-seeded duplicate record), so no + // child process is ever launched. The holder writes nothing: this does + // not show a write made under the lock being observed. use std::sync::{Arc, Barrier}; use std::time::Instant; diff --git a/apps/rocm/src/services_text.rs b/apps/rocm/src/services_text.rs index b7a8d580c..477474518 100644 --- a/apps/rocm/src/services_text.rs +++ b/apps/rocm/src/services_text.rs @@ -997,6 +997,16 @@ fn restart_internal_managed_service_with( /// Replaces the child `spawn()` of a restart (non-Windows only; Windows spawns /// through `rocm_core::spawn_detached_no_inherit`). Test-only seam, so a test /// can observe the leased port at the exact instant production would spawn. +/// How long a restart waits for the new child to serve. +#[allow(clippy::missing_const_for_fn)] // not const under `cfg(test)`, where a test shortens it +fn restart_readiness_timeout() -> Duration { + #[cfg(test)] + if let Some(timeout) = tests::READINESS_TIMEOUT_OVERRIDE.with(std::cell::Cell::get) { + return timeout; + } + Duration::from_secs(45) +} + type RestartSpawnOverride<'a> = dyn Fn(&mut std::process::Command) -> io::Result + 'a; @@ -1179,7 +1189,7 @@ fn restart_internal_managed_service_inner( started.port, started.served_model_name(), endpoint_api_key.as_deref(), - Duration::from_secs(45), + restart_readiness_timeout(), ); record_restart_readiness(paths, &started, readiness) } @@ -2802,6 +2812,121 @@ mod tests { Ok(()) } + thread_local! { + /// Lets a test that lets a restart reach its readiness wait shorten it. + pub(super) static READINESS_TIMEOUT_OVERRIDE: std::cell::Cell> = + const { std::cell::Cell::new(None) }; + } + + /// The restart holds the launch lock until its record names the new child. + /// A waiter blocked on the lock since the spawn must therefore read a record + /// that already carries the child's pid the moment it gets in; releasing + /// the lock right after the spawn lets it read the stale record instead. + #[cfg(not(windows))] + #[test] + fn restart_holds_the_launch_lock_until_the_record_names_the_new_child() -> Result<()> { + let (root, paths) = test_paths("restart-lock-until-published"); + paths.ensure()?; + let service_id = "svc-restart-lock-until-published"; + let record = stop_test_record(&paths, service_id); + record.write()?; + + READINESS_TIMEOUT_OVERRIDE.with(|cell| cell.set(Some(Duration::ZERO))); + let (observed_tx, observed_rx) = std::sync::mpsc::channel(); + let lock_path = paths.managed_launch_lock_path(); + let manifest_path = paths.service_manifest_path(service_id); + let spawn: &RestartSpawnOverride<'_> = &|command| { + let child = std::process::Command::new("sleep") + .arg("30") + .spawn() + .map_err(|error| io::Error::other(format!("{error} ({command:?})")))?; + let (lock_path, manifest_path, tx) = ( + lock_path.clone(), + manifest_path.clone(), + observed_tx.clone(), + ); + thread::spawn(move || { + // Blocks until the restart drops the lock. + let _lock = rocm_core::FileLock::acquire(&lock_path).expect("acquire"); + let record: ManagedServiceRecord = + serde_json::from_slice(&fs::read(manifest_path).expect("read")).expect("parse"); + let _ = tx.send(record.supervisor_pid); + }); + Ok(child) + }; + let result = restart_internal_managed_service_inner( + &paths, + service_id, + None, + |_| (Vec::new(), true), + Some(spawn), + ); + READINESS_TIMEOUT_OVERRIDE.with(|cell| cell.set(None)); + let started = result?; + let observed = observed_rx.recv_timeout(Duration::from_secs(10)); + // Reap the stand-in engine. + let _ = std::process::Command::new("kill") + .arg(started.supervisor_pid.to_string()) + .status(); + let _ = fs::remove_dir_all(root); + + assert_eq!( + observed.expect("the waiter must get the lock after the restart"), + started.supervisor_pid, + "the lock was released before the record named the restarted child" + ); + Ok(()) + } + + #[cfg(not(windows))] + #[test] + fn restart_is_not_blocked_by_a_live_record_on_another_port() -> Result<()> { + // `blocks_port` compares ports. A live service elsewhere must not trip + // the "claimed that port" refusal: if it ignored the port, every + // restart would be refused while any other service ran. + let (root, paths) = test_paths("restart-live-other-port"); + paths.ensure()?; + let service_id = "svc-restart-live-other-port"; + let record = stop_test_record(&paths, service_id); + let host = record.host.clone(); + let port = record.port; + record.write()?; + + let mut other = ManagedServiceRecord::new( + &paths, + "svc-live-other-port", + "lemonade", + "other-model", + "other-model-canonical", + &host, + port.wrapping_add(1), + "managed", + 0, + None, + None, + None, + ); + other.status = "starting".to_owned(); + other.write()?; + + let spawn: &RestartSpawnOverride<'_> = + &|_command| Err(io::Error::other("spawn intercepted by test")); + let result = restart_internal_managed_service_inner( + &paths, + service_id, + None, + |_| (Vec::new(), true), + Some(spawn), + ); + let _ = fs::remove_dir_all(root); + let rendered = format!("{:#}", result.expect_err("the intercepted spawn must fail")); + assert!( + rendered.contains("spawn intercepted by test"), + "a live service on another port must not block the restart: {rendered}" + ); + Ok(()) + } + /// The value following `flag` in an argv, or `None` when the flag is absent. fn argv_flag_value<'a>(args: &'a [String], flag: &str) -> Option<&'a str> { let index = args.iter().position(|arg| arg == flag)?; diff --git a/apps/rocmd/src/common.rs b/apps/rocmd/src/common.rs index 87dfa11eb..d2862cb4f 100644 --- a/apps/rocmd/src/common.rs +++ b/apps/rocmd/src/common.rs @@ -341,7 +341,7 @@ pub(crate) fn ensure_public_service_has_endpoint_key( bail!( "refusing to respawn a service bound to the public host `{host}` without an endpoint \ API key: it would come back up without authentication. Relaunch it with \ - `rocm serve --host {host} --allow-public-bind` to issue a new key." + `rocm serve --host {host} --port --allow-public-bind` to issue a new key." ); } Ok(()) diff --git a/apps/rocmd/src/watchers.rs b/apps/rocmd/src/watchers.rs index 68f1db094..540056ab3 100644 --- a/apps/rocmd/src/watchers.rs +++ b/apps/rocmd/src/watchers.rs @@ -1568,10 +1568,24 @@ fn restart_managed_service_with( record: &mut ManagedServiceRecord, spawn_override: Option<&SpawnOverride<'_>>, ) -> Result<()> { - let _launch_lock = rocm_core::FileLock::acquire_bounded( - paths.managed_launch_lock_path(), + restart_managed_service_bounded( + paths, + record, + spawn_override, rocm_core::MANAGED_LAUNCH_LOCK_TIMEOUT, - )?; + ) +} + +/// [`restart_managed_service_with`] with the launch-lock wait injectable, so a +/// test can prove the wait is bounded without waiting out the real bound. +fn restart_managed_service_bounded( + paths: &AppPaths, + record: &mut ManagedServiceRecord, + spawn_override: Option<&SpawnOverride<'_>>, + lock_timeout: Duration, +) -> Result<()> { + let _launch_lock = + rocm_core::FileLock::acquire_bounded(paths.managed_launch_lock_path(), lock_timeout)?; // Re-check against the disk, now that the lock excludes a concurrent // `rocm serve` publish: the caller's `record` was read before this lock was @@ -4481,7 +4495,15 @@ mod tests { .expect("spawn sleeper child"); let child_pid = child.id(); + let started = std::time::Instant::now(); terminate_recovery_supervisor(&mut child); + // The sleeper sleeps for two minutes by itself: only a kill returns + // well before that, so waiting it out must not pass. + assert!( + started.elapsed() < Duration::from_secs(30), + "terminate_recovery_supervisor took {:?}: the child was waited out, not killed", + started.elapsed() + ); // The helper reaps, so the pid must be gone rather than a zombie. assert!( @@ -4857,6 +4879,36 @@ mod tests { Ok(()) } + /// Recovery waits a bounded time for the launch lock, then gives up instead + /// of queuing behind an interactive launch. + #[test] + fn recovery_gives_up_when_the_launch_lock_stays_held_past_its_bound() -> Result<()> { + let (root, paths) = temp_app_paths("recovery-lock-bound"); + paths.ensure()?; + let mut record = identity_probe_record(&paths, "svc-lock-bound", free_loopback_port()); + record.status = "failed".to_owned(); + record.write()?; + + let guard = rocm_core::FileLock::acquire(paths.managed_launch_lock_path())?; + let holder = thread::spawn(move || { + thread::sleep(Duration::from_secs(3)); + drop(guard); + }); + let started = std::time::Instant::now(); + let outcome = + restart_managed_service_bounded(&paths, &mut record, None, Duration::from_millis(100)); + let waited = started.elapsed(); + holder.join().expect("holder thread"); + fs::remove_dir_all(root).ok(); + + assert!(outcome.is_err(), "a held lock must defer recovery"); + assert!( + waited < Duration::from_secs(2), + "recovery waited {waited:?} for a lock held for 3s: the wait is not bounded" + ); + Ok(()) + } + /// A candidate still inside its restart backoff must not starve the ones /// after it. #[test] @@ -4865,12 +4917,17 @@ mod tests { paths.ensure()?; fs::create_dir_all(paths.services_dir())?; + let b_created = unix_time_millis() - 10_000; let mut a = identity_probe_record(&paths, "svc-backoff-a", free_loopback_port()); a.status = "failed".to_owned(); a.last_restart_unix_ms = Some(unix_time_millis()); + // Records load newest first: A must come first, or the skip is not what + // gets B returned. + a.created_at_unix_ms = b_created + 1_000; a.write()?; let mut b = identity_probe_record(&paths, "svc-backoff-b", free_loopback_port()); b.status = "failed".to_owned(); + b.created_at_unix_ms = b_created; b.write()?; let found = find_recoverable_service(&paths); @@ -4881,9 +4938,10 @@ mod tests { Ok(()) } - /// The `try_wait` failure path rolls back against the snapshot taken after - /// the supervisor was recorded; the pre-record snapshot would mismatch the - /// on-disk pid and leave `recovering` behind. + /// The rollback restores the prior record when handed the snapshot taken + /// after the supervisor was recorded, which is the one the `try_wait` + /// failure arm passes. That arm cannot be driven from here, so this covers + /// the rollback's guard, not the choice of snapshot at that call site. #[test] fn rollback_against_the_recorded_snapshot_restores_the_prior_record() -> Result<()> { let (root, paths) = temp_app_paths("rollback-recorded-snapshot"); diff --git a/crates/rocm-core/src/service_record.rs b/crates/rocm-core/src/service_record.rs index 1f3b94d1a..c5022d70b 100644 --- a/crates/rocm-core/src/service_record.rs +++ b/crates/rocm-core/src/service_record.rs @@ -452,6 +452,22 @@ mod tests { assert!(record.last_restart_unix_ms.is_some()); } + #[test] + fn blocks_port_needs_a_live_record_on_that_exact_port() { + let mut record = probe_test_record(11435); + record.status = "running".to_owned(); + assert!(record.blocks_port(11435)); + assert!( + !record.blocks_port(11436), + "a different port is not blocked" + ); + record.status = "stopped".to_owned(); + assert!( + !record.blocks_port(11435), + "a stopped record blocks nothing" + ); + } + /// Writes `state` as the record's engine state file and runs the refresh. fn refresh_from_state(name: &str, state: &serde_json::Value) -> ManagedServiceRecord { let dir = std::env::temp_dir().join(format!( diff --git a/crates/rocm-dash-tui/src/ui/serve_wizard.rs b/crates/rocm-dash-tui/src/ui/serve_wizard.rs index 2d80aae6c..a2b4e84ea 100644 --- a/crates/rocm-dash-tui/src/ui/serve_wizard.rs +++ b/crates/rocm-dash-tui/src/ui/serve_wizard.rs @@ -1858,7 +1858,7 @@ mod tests { // This only pins the shared constant into the approval body; the CLI's own // plan output proves the phrase actually appears before the resolved - // endpoint, in `serve-auto-port-skips-occupied` (serve-29). + // endpoint, in `serve-auto-port-skips-occupied` (serve-31). #[test] fn approval_shows_exact_argv_and_the_automatic_port_phrase() { let mut w = ServeWizardState { @@ -2177,6 +2177,16 @@ mod tests { assert!(!out.contains("Review:"), "no second modal"); } + #[test] + fn engine_only_override_summarises_as_customized() { + let w = ServeWizardState { + engine_idx: ENGINE_AUTO + 1, + ..Default::default() + }; + assert!(w.is_customized()); + assert_eq!(w.advanced_summary(), "Customized"); + } + #[test] fn foreground_only_form_summarises_as_customized() { let w = ServeWizardState { diff --git a/docs/architecture.md b/docs/architecture.md index 55662e561..25dff63c7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -47,7 +47,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`freeform_planner.rs`](../apps/rocm/src/freeform_planner.rs) — full domain extraction (Phase 6e, ROCMAI-87). Owns the freeform natural-language planner cluster's types (`StructuredRequestPlan`, `PlannerIntent`, `PlannedToolCall`, `FreeformPlanAction`, `ProviderPlannerResponse`, `ProviderPlannerToolCall`, all module-private except `FreeformPlanAction` and the fields of `StructuredRequestPlan`/`PlannedToolCall` that `main.rs` reads directly), the deterministic plan builder (`build_freeform_plan*`), the provider-assisted path (`resolve_freeform_plan_with_provider`, `build_provider_planner_prompt`, `provider_planner_response_to_plan`, `validate_provider_planner_tool_call`), and plan rendering (`render_structured_request_plan`). [`main.rs`](../apps/rocm/src/main.rs) calls `build_freeform_plan_with_context`/`render_structured_request_plan` directly for the dispatch/review flow (`main.rs`'s only other call into this module, `render_launch_summary` → `host_default_engine`, is unrelated: bare `rocm`'s non-interactive launch summary), while [`freeform_execution.rs`](../apps/rocm/src/freeform_execution.rs) above calls `plan_next_action`/`format_structured_tool_call` for the execution flow and reads `FreeformPlanAction`/`StructuredRequestPlan` by type for its execution-validation functions — those live there, not here, because they decide whether a plan's next action is safe to *execute*, a dispatch-adjacent concern distinct from *building* the plan; `main.rs`'s `run_freeform` builds a plan exactly once via `build_freeform_plan_with_context` and threads that same value through `execute_freeform_next_action` to `prepare_freeform_execution` rather than rebuilding it (ROCMAI-725: building a plan can call a configured cloud `planner_provider`, which is not guaranteed deterministic, so a second independent build could silently disagree with the first). [`chat.rs`](../apps/rocm/src/chat.rs)'s `natural_language_plan` MCP tool gets the same single-build guarantee from `plan_freeform_request_with_context`, which builds once and returns both the rendered text and the next action; `chat.rs` also calls `format_structured_tool_call` directly, like [`comfyui.rs`](../apps/rocm/src/comfyui.rs) does, and its `validate_chat_rocm_command_safety` reads `serve_args_request_cpu_device` directly as a no-CPU-fallback safety check on provider-proposed `serve` args. `host_default_engine`, `build_freeform_plan_with_context`, `render_structured_request_plan`, `plan_next_action`, `plan_freeform_request_with_context`, `provider_planner_response_to_plan`, `format_structured_tool_call`, and `serve_args_request_cpu_device` are `pub(crate)` for these reasons; `freeform_plan_next_action` and `render_freeform_plan` are test-only (`#[cfg(test)]`) and `FreeformPlanAction` was already `pub(crate)` before the split. `parse_freeform_invocation`/`should_treat_as_freeform` (deciding whether an invocation is freeform at all) stay in `main.rs`, a dispatch-level concern outside both this module and `freeform_execution.rs`. The cluster's regression tests live in `freeform_planner.rs`'s own `mod tests`; the handful of `main.rs` tests that exercise the invocation classifier/clap-suggestion-avoidance stay in `main.rs`'s shared `mod tests`, since neither is specific to this module or to `freeform_execution.rs` (and neither touches `freeform_planner::` at all), and the ROCMAI-725 regression tests proving the single-build guarantee live in `providers.rs`'s `mod tests` (reusing its mock local-provider server), reaching this module's items through the qualified `freeform_planner::` path like any other external caller. - [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs) — mechanical relocation, owns private types (Phase 6f, ROCMAI-214): `--device`/`--gpu` parsing (`parse_device_policy`, `parse_gpu_selection`, `parse_gpu_indices_arg`), the managed-launch-lock GPU selection (`select_gpu_indices_under_launch_lock`, `validate_pinned_gpu_index`, `effective_gpu_count`, `auto_select_gpu_indices`, `select_auto_gpu_index`, `busy_gpu_indices`), per-GPU VRAM telemetry (the `GpuVramUsage` type, `gpu_vram_usage`, `simulated_low_vram_usage`, `gpu_vram_usage_amd_smi`, `gpu_vram_usage_sysfs`, `read_drm_vram_usage`, `read_sysfs_u64`, `drm_device_is_amd`, `parse_gpu_vram_usage`), and the serve-plan low-VRAM warning (`gpu_low_memory_warning`, `vram_capacity_is_meaningful`, `serve_gpu_low_memory_warning`), plus the best-effort `amd-smi`-based GPU count (`detect_gpu_count`). `const fn device_policy_name` sits immediately after this cluster's last function in the original file but stayed in `main.rs` rather than moving — it already had callers outside the cluster (`engine_ipc.rs`, `service_launch.rs`, `serve_cmd.rs`) plus its own test. `GpuVramUsage` (and its fields), `gpu_vram_usage`, `parse_device_policy`, `parse_gpu_indices_arg`, `serve_gpu_low_memory_warning`, and `vram_capacity_is_meaningful` are `pub(crate)` and read from [`main.rs`](../apps/rocm/src/main.rs) (`classify_accelerator_memory`, `host_accelerator_memory`, the `dispatch()` `Command::EngineServeHttp` arm, and `collect_serve_notes` respectively); `detect_gpu_count`, `gpu_vram_usage`, `parse_device_policy`, `parse_gpu_selection`, `select_gpu_indices_under_launch_lock`, `serve_gpu_low_memory_warning`, and `validate_pinned_gpu_index` are also `pub(crate)` and read from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs); `parse_device_policy` is also read from [`engine_ipc.rs`](../apps/rocm/src/engine_ipc.rs)'s `run_builtin_engine_serve_http` and, as of Phase 6f (ROCMAI-209), from [`services_text.rs`](../apps/rocm/src/services_text.rs)'s `restart_child_argv` (moved out of `main.rs` in that same phase). `effective_gpu_count`, `auto_select_gpu_indices`, `select_auto_gpu_index`, `simulated_low_vram_usage`, `gpu_vram_usage_amd_smi`, `gpu_vram_usage_sysfs` (both the Linux and non-Linux `cfg` variants), `read_drm_vram_usage`, `read_sysfs_u64`, `drm_device_is_amd`, `parse_gpu_vram_usage`, `gpu_low_memory_warning`, and `busy_gpu_indices` have no callers outside this module and stay private. The cluster's regression tests live in `gpu_policy.rs`'s own `mod tests`; two small test-only fixture builders (`vram`, `host_gpu`) are duplicated rather than shared, since `main.rs`'s own `serve_notes_pair_low_vram_with_the_vllm_utilization_hint` (testing `collect_serve_notes`, which stayed) still needs them too. - [`logging.rs`](../apps/rocm/src/logging.rs) — mechanical relocation, dispatch-adjacent, no owned types. -- [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs) — mechanical relocation, owns private types (Phase 6f, ROCMAI-211): managed-service liveness classification (`managed_service_is_live`, `refresh_managed_service_runtime_liveness`, `existing_live_managed_service`'s idempotency guard) and the dashboard-sidebar status/counts helpers derived from it (`ManagedServiceSidebarCounts`/`managed_service_sidebar_counts`/`local_server_sidebar_status`), plus the engine display-name helper `friendly_engine_label` sidebar rendering also uses. `ManagedServiceSidebarCounts` is `pub(crate)` to match the visibility of the `pub(crate)` functions that return/take it; `friendly_engine_label`, `existing_live_managed_service`, `recorded_service_pids`, `refresh_managed_service_runtime_liveness`, `local_server_sidebar_status`, `managed_service_sidebar_counts`, and `load_managed_services` are `pub(crate)` for the same reason — each has a confirmed caller outside this module. [`main.rs`](../apps/rocm/src/main.rs) calls into it from the local-servers view, the duplicate-managed-launch guard, and service-record removal; [`chat.rs`](../apps/rocm/src/chat.rs) calls into it for the deterministic services-tool summary; [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs) calls into it when reconciling live servers against a runtime activation; [`service_launch.rs`](../apps/rocm/src/service_launch.rs) calls into it for the same duplicate-launch guard `main.rs` uses before spawning a new managed service. +- [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs) — mechanical relocation, owns private types (Phase 6f, ROCMAI-211): managed-service liveness classification (`managed_service_is_live`, `refresh_managed_service_runtime_liveness`, `existing_live_managed_service_in`'s idempotency guard) and the dashboard-sidebar status/counts helpers derived from it (`ManagedServiceSidebarCounts`/`managed_service_sidebar_counts`/`local_server_sidebar_status`), plus the engine display-name helper `friendly_engine_label` sidebar rendering also uses. `ManagedServiceSidebarCounts` is `pub(crate)` to match the visibility of the `pub(crate)` functions that return/take it; `friendly_engine_label`, `existing_live_managed_service_in`, `recorded_service_pids`, `refresh_managed_service_runtime_liveness`, `local_server_sidebar_status`, `managed_service_sidebar_counts`, and `load_managed_services` are `pub(crate)` for the same reason — each has a confirmed caller outside this module. [`main.rs`](../apps/rocm/src/main.rs) calls into it from the local-servers view, the duplicate-managed-launch guard, and service-record removal; [`chat.rs`](../apps/rocm/src/chat.rs) calls into it for the deterministic services-tool summary; [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs) calls into it when reconciling live servers against a runtime activation; [`service_launch.rs`](../apps/rocm/src/service_launch.rs) calls into it for the same duplicate-launch guard `main.rs` uses before spawning a new managed service. - [`provider_keys.rs`](../apps/rocm/src/provider_keys.rs) — full domain extraction. - [`providers.rs`](../apps/rocm/src/providers.rs) — full domain extraction (no dedicated command enum; invoked from the existing chat/config command flows). - [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) — mechanical relocation, owns private types (part of [`main.rs`](../apps/rocm/src/main.rs)'s Phase 6c split, ROCMAI-89, alongside [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) below): settling an engine onto a runtime after an SDK install — torch alignment, engine-dependency checks, and device checks. `SdkInstallConsents` is `pub(crate)` and read from `main.rs`'s `install()`; `ensure_openmpi_for_vllm`/`ensure_libatomic_for_torch`/`ensure_libnuma_for_torch`/`settle_engine_install` are also `pub(crate)` and read from [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs). diff --git a/docs/manual-testing.md b/docs/manual-testing.md index 58dfe6003..e7cf2f660 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -333,11 +333,13 @@ Expected result: - `rocm services` shows the launched service bound to the port it picked, and the deployment summary's endpoint matches it. -With that service still running, try to launch a second one pinned to the -same port: +With that service still running, try to launch a service for a different +model pinned to the same port. (The same model and engine would be reported as +already running instead, since an equivalent live service is reused before any +port check.) ```powershell -rocm serve qwen --engine lemonade --device gpu_required --managed --port +rocm serve --engine lemonade --device gpu_required --managed --port ``` Expected result: diff --git a/tests/e2e-cucumber/features/dash.feature b/tests/e2e-cucumber/features/dash.feature index 9d8e4f171..37732aeb0 100644 --- a/tests/e2e-cucumber/features/dash.feature +++ b/tests/e2e-cucumber/features/dash.feature @@ -704,7 +704,8 @@ Feature: Interactive dashboard And the user presses Enter in the serve wizard Then the serve wizard asks to approve "serve demo-model" with an automatic port When the user presses Escape - And the user presses Escape + Then the serve wizard approval closes and the wizard stays open + When the user presses Escape Then the serve wizard is closed When the user quits the dashboard Then the dashboard exits successfully @@ -728,3 +729,23 @@ Feature: Interactive dashboard Then the serve wizard is closed When the user quits the dashboard Then the dashboard exits successfully + + @id:dash-serve-wizard-refuses-public-host @requires-os:linux + Scenario: dash-43 - A public host is refused in the form with the CLI pointer + When the user opens the dashboard with demo data + And the user opens the Observe view + And the user opens the serve wizard + And the user types "demo-model" into the serve wizard + And the user presses Down 1 times in the serve wizard + And the user presses Enter in the serve wizard + And the user presses Down 3 times in the serve wizard + And the user presses Enter in the serve wizard + And the user types "0.0.0.0" into the serve wizard + And the user presses Enter in the serve wizard + And the user presses Down 3 times in the serve wizard + And the user presses Enter in the serve wizard + Then the serve wizard refuses to launch and says "Dash serves on loopback only" + When the user presses Escape + Then the serve wizard is closed + When the user quits the dashboard + Then the dashboard exits successfully diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index faf06fb37..65b717bfc 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -323,6 +323,28 @@ Feature: Model serving Then the CLI refuses and names the exact command to stop the existing service And running that command actually stops the service + # An explicit --port is a demand: when an equivalent service (same engine and + # model) is already live on a different port, serve refuses instead of + # reporting success on the old port. Same planted record and lane as serve-24. + @id:serve-explicit-port-refused-for-live-equivalent @requires-gpu @requires-engine:lemonade @requires-os:linux + Scenario: serve-32 - An explicit port that differs from a running equivalent service is refused + Given a managed runtime is active + And an unauthenticated Lemonade service is already running for that model + When the user serves that same model again on an explicit port other than the running one + Then the CLI refuses and names the port the running service uses + And the CLI names the exact command to stop the existing service + + # `rocm services restart` takes the launch lock and refuses a port another + # live managed service now holds, before it spawns anything. No GPU: the + # refusal comes from the on-disk records. + @id:services-restart-refused-when-port-claimed @requires-os:linux + Scenario: serve-33 - Restarting a service whose port another service holds is refused + Given a managed runtime is active + And another managed service already reserves the default serve port + And a stopped managed service is recorded on the default serve port + When the user restarts the stopped service + Then the restart is refused naming the service that holds the port + # ROCMAI-543: a Windows console on the OEM code page (437) renders a UTF-8 em # dash, and other non-ASCII punctuation, as mojibake (e.g. "ΓÇö") instead of # transliterating it. `rocm model` is the flagship instance of that bug class; diff --git a/tests/e2e-cucumber/tests/e2e/dash_steps.rs b/tests/e2e-cucumber/tests/e2e/dash_steps.rs index 19ed910b3..cc4cb0317 100644 --- a/tests/e2e-cucumber/tests/e2e/dash_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/dash_steps.rs @@ -709,6 +709,25 @@ async fn serve_wizard_asks_to_approve_automatic(world: &mut E2eWorld, command: S ); } +#[then("the serve wizard approval closes and the wizard stays open")] +async fn serve_wizard_approval_closes_wizard_stays(world: &mut E2eWorld) { + // Waiting for the approval to clear between two Escapes keeps them from + // coalescing into one in the same PTY read, and shows the first one closed + // only the approval. + let tui = session(world); + tui.wait_until_gone( + "Port: automatic; endpoint shown after launch", + default_timeout(), + ) + .await + .unwrap_or_else(|e| panic!("the serve approval did not close: {e}")); + let screen = tui.screen_text(); + assert!( + screen.contains("Serve a model"), + "Escape on the approval closed the whole wizard:\n{screen}" + ); +} + #[then(expr = "the serve wizard refuses to launch and says {string}")] async fn serve_wizard_refuses_to_launch(world: &mut E2eWorld, message: String) { let tui = session(world); diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index fbc9fc379..22e4d809e 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -419,7 +419,7 @@ async fn occupy_default_serve_port(world: &mut E2eWorld) { /// behind the record, the endpoint probe would then come back `Unreachable` /// and the record gets demoted to `"stopped"` *before* the explicit-port /// collision check ever sees it (`refresh_managed_service_runtime_liveness` -/// in `apps/rocm/src/main.rs`). Backing the record with a `MockServer` on +/// in `apps/rocm/src/managed_service_status.rs`). Backing the record with a `MockServer` on /// `SERVE_PORT` makes `/v1/models` list `other-model`, so the probe reports /// `Listing`/`Serving` and the record is left `"ready"` instead — the /// scenario needs the collision detected, so the port has to be reserved for @@ -430,6 +430,7 @@ async fn occupy_default_serve_port(world: &mut E2eWorld) { /// doc comment for the same hazard). #[given("another managed service already reserves the default serve port")] async fn reserve_default_serve_port(world: &mut E2eWorld) { + ensure_serve_port_free().await; let root = world.isolated_root.as_ref().expect("no isolated root"); let services = root.path().join("data").join("services"); let mock = MockServer::start_on_port("other-model", SERVE_PORT).await; @@ -1930,6 +1931,118 @@ async fn user_serves_requiring_api_key(world: &mut E2eWorld) { world.cli_rc = Some(rc); } +#[when("the user serves that same model again on an explicit port other than the running one")] +async fn user_serves_other_explicit_port(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm( + world, + &[ + "serve", + REUSE_REFUSAL_MODEL, + "--engine", + REUSE_REFUSAL_ENGINE, + "--managed", + "--port", + "11500", + ], + ); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +#[then("the CLI refuses and names the port the running service uses")] +async fn assert_refused_naming_running_port(world: &mut E2eWorld) { + let rc = world.cli_rc.expect("no serve rc recorded"); + let output = serve_output(world); + assert!( + rc != 0, + "an explicit port the running service does not use must be refused:\n{output}" + ); + assert!( + output.contains(REUSE_REFUSAL_SERVICE_ID) + && output.contains("on port 9") + && output.contains("port 11500 would not be used"), + "the refusal must name the running service and both ports:\n{output}" + ); +} + +#[then("the CLI names the exact command to stop the existing service")] +async fn assert_names_stop_command(world: &mut E2eWorld) { + let output = serve_output(world); + assert!( + output.contains(&format!( + "rocm services stop {REUSE_REFUSAL_SERVICE_ID} --yes" + )), + "the refusal must name the way out, directly runnable with --yes: {output}" + ); +} + +/// A second record, `e2e-stopped`, on the default serve port: the service the +/// restart scenario tries to bring back. Cloned from the `e2e-mock` record +/// `reserve_default_serve_port` planted, so its shape is the one the CLI reads. +#[given("a stopped managed service is recorded on the default serve port")] +async fn plant_stopped_service_on_default_port(world: &mut E2eWorld) { + let services = world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .join("data") + .join("services"); + let mut record: serde_json::Value = serde_json::from_slice( + &std::fs::read(services.join("e2e-mock.json")).expect("read e2e-mock record"), + ) + .expect("parse e2e-mock record"); + record["service_id"] = "e2e-stopped".into(); + record["status"] = "stopped".into(); + record["manifest_path"] = services.join("e2e-stopped.json").to_string_lossy().into(); + record["log_path"] = services.join("e2e-stopped.log").to_string_lossy().into(); + record["engine_state_path"] = services + .join("e2e-stopped.state.json") + .to_string_lossy() + .into(); + std::fs::write( + services.join("e2e-stopped.json"), + serde_json::to_vec_pretty(&record).expect("serialize record"), + ) + .expect("write stopped record"); +} + +#[when("the user restarts the stopped service")] +async fn user_restarts_stopped_service(world: &mut E2eWorld) { + let (stdout, stderr, rc) = + crate::run_rocm(world, &["services", "restart", "e2e-stopped", "--yes"]); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +#[then("the restart is refused naming the service that holds the port")] +async fn assert_restart_refused_naming_holder(world: &mut E2eWorld) { + let rc = world.cli_rc.expect("no restart rc recorded"); + let output = serve_output(world); + assert!( + rc != 0, + "the restart must be refused, but exited 0:\n{output}" + ); + assert!( + output.contains("e2e-mock") && output.contains("recorded on that port"), + "the refusal must name the service holding the port:\n{output}" + ); + // Nothing was spawned: the stopped record is still stopped. + let root = world + .isolated_root + .as_ref() + .expect("no isolated root") + .path(); + let record: serde_json::Value = serde_json::from_slice( + &std::fs::read(root.join("data").join("services").join("e2e-stopped.json")) + .expect("read stopped record"), + ) + .expect("parse stopped record"); + assert_eq!(record["status"], "stopped", "{record}"); +} + #[then("the CLI refuses and names the exact command to stop the existing service")] async fn assert_refused_with_stop_hint(world: &mut E2eWorld) { let rc = world.cli_rc.expect("no serve rc recorded");